diff options
Diffstat (limited to 'tool')
218 files changed, 15378 insertions, 3135 deletions
diff --git a/tool/bundler/dev_gems.rb b/tool/bundler/dev_gems.rb index 0e5e681f73..1422cfc7a5 100644 --- a/tool/bundler/dev_gems.rb +++ b/tool/bundler/dev_gems.rb @@ -2,18 +2,19 @@ source "https://rubygems.org" -gem "rdoc", "6.2.0" # 6.2.1 is required > Ruby 2.3 gem "test-unit", "~> 3.0" -gem "rake", "~> 13.0" +gem "rake", "~> 13.1" +gem "rb_sys" gem "webrick", "~> 1.6" -gem "parallel_tests", "~> 2.29" -gem "parallel", "1.19.2" # 1.20+ is required > Ruby 2.3 -gem "rspec-core", "~> 3.8" -gem "rspec-expectations", "~> 3.8" -gem "rspec-mocks", "~> 3.11.1" -gem "uri", "~> 0.10.1" +gem "turbo_tests", "~> 2.2.3" +gem "parallel_tests", "< 3.9.0" +gem "parallel", "~> 1.19" +gem "rspec-core", "~> 3.12" +gem "rspec-expectations", "~> 3.12" +gem "rspec-mocks", "~> 3.12" +gem "uri", "~> 0.13.0" group :doc do - gem "ronn", "~> 0.7.3", :platform => :ruby + gem "nronn", "~> 0.11.1", platform: :ruby end diff --git a/tool/bundler/dev_gems.rb.lock b/tool/bundler/dev_gems.rb.lock deleted file mode 100644 index feb3cc22fd..0000000000 --- a/tool/bundler/dev_gems.rb.lock +++ /dev/null @@ -1,56 +0,0 @@ -GEM - remote: https://rubygems.org/ - specs: - diff-lcs (1.5.0) - hpricot (0.8.6) - mustache (1.1.1) - parallel (1.19.2) - parallel_tests (2.32.0) - parallel - power_assert (2.0.1) - rake (13.0.6) - rdiscount (2.2.0.2) - rdoc (6.2.0) - ronn (0.7.3) - hpricot (>= 0.8.2) - mustache (>= 0.7.0) - rdiscount (>= 1.5.8) - rspec-core (3.11.0) - rspec-support (~> 3.11.0) - rspec-expectations (3.11.0) - diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.11.0) - rspec-mocks (3.11.1) - diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.11.0) - rspec-support (3.11.0) - test-unit (3.5.3) - power_assert - uri (0.10.1) - webrick (1.7.0) - -PLATFORMS - java - ruby - universal-java-11 - universal-java-18 - x64-mingw-ucrt - x64-mingw32 - x86_64-darwin-20 - x86_64-linux - -DEPENDENCIES - parallel (= 1.19.2) - parallel_tests (~> 2.29) - rake (~> 13.0) - rdoc (= 6.2.0) - ronn (~> 0.7.3) - rspec-core (~> 3.8) - rspec-expectations (~> 3.8) - rspec-mocks (~> 3.11.1) - test-unit (~> 3.0) - uri (~> 0.10.1) - webrick (~> 1.6) - -BUNDLED WITH - 2.4.0.dev diff --git a/tool/bundler/rubocop_gems.rb b/tool/bundler/rubocop_gems.rb index 84cb226330..4d0b21060a 100644 --- a/tool/bundler/rubocop_gems.rb +++ b/tool/bundler/rubocop_gems.rb @@ -2,10 +2,11 @@ source "https://rubygems.org" -gem "rubocop", "~> 1.7" +gem "rubocop", ">= 1.52.1", "< 2" gem "minitest" gem "rake" gem "rake-compiler" gem "rspec" gem "test-unit" +gem "rb_sys" diff --git a/tool/bundler/rubocop_gems.rb.lock b/tool/bundler/rubocop_gems.rb.lock deleted file mode 100644 index 13e541bcfa..0000000000 --- a/tool/bundler/rubocop_gems.rb.lock +++ /dev/null @@ -1,65 +0,0 @@ -GEM - remote: https://rubygems.org/ - specs: - ast (2.4.2) - diff-lcs (1.5.0) - minitest (5.15.0) - parallel (1.21.0) - parser (3.1.0.0) - ast (~> 2.4.1) - power_assert (2.0.1) - rainbow (3.1.1) - rake (13.0.6) - rake-compiler (1.1.7) - rake - regexp_parser (2.2.0) - rexml (3.2.5) - rspec (3.10.0) - rspec-core (~> 3.10.0) - rspec-expectations (~> 3.10.0) - rspec-mocks (~> 3.10.0) - rspec-core (3.10.1) - rspec-support (~> 3.10.0) - rspec-expectations (3.10.1) - diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.10.0) - rspec-mocks (3.10.2) - diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.10.0) - rspec-support (3.10.3) - rubocop (1.24.1) - parallel (~> 1.10) - parser (>= 3.0.0.0) - rainbow (>= 2.2.2, < 4.0) - regexp_parser (>= 1.8, < 3.0) - rexml - rubocop-ast (>= 1.15.1, < 2.0) - ruby-progressbar (~> 1.7) - unicode-display_width (>= 1.4.0, < 3.0) - rubocop-ast (1.15.1) - parser (>= 3.0.1.1) - ruby-progressbar (1.11.0) - test-unit (3.5.3) - power_assert - unicode-display_width (2.1.0) - -PLATFORMS - arm64-darwin-20 - arm64-darwin-21 - universal-java-11 - universal-java-18 - x64-mingw-ucrt - x86_64-darwin-19 - x86_64-darwin-20 - x86_64-linux - -DEPENDENCIES - minitest - rake - rake-compiler - rspec - rubocop (~> 1.7) - test-unit - -BUNDLED WITH - 2.4.0.dev diff --git a/tool/bundler/standard_gems.rb b/tool/bundler/standard_gems.rb index 1cd189742d..20c1ecd827 100644 --- a/tool/bundler/standard_gems.rb +++ b/tool/bundler/standard_gems.rb @@ -9,3 +9,4 @@ gem "rake" gem "rake-compiler" gem "rspec" gem "test-unit" +gem "rb_sys" diff --git a/tool/bundler/standard_gems.rb.lock b/tool/bundler/standard_gems.rb.lock deleted file mode 100644 index df346ad7ce..0000000000 --- a/tool/bundler/standard_gems.rb.lock +++ /dev/null @@ -1,71 +0,0 @@ -GEM - remote: https://rubygems.org/ - specs: - ast (2.4.2) - diff-lcs (1.5.0) - minitest (5.15.0) - parallel (1.21.0) - parser (3.1.0.0) - ast (~> 2.4.1) - power_assert (2.0.1) - rainbow (3.1.1) - rake (13.0.6) - rake-compiler (1.1.7) - rake - regexp_parser (2.2.0) - rexml (3.2.5) - rspec (3.10.0) - rspec-core (~> 3.10.0) - rspec-expectations (~> 3.10.0) - rspec-mocks (~> 3.10.0) - rspec-core (3.10.1) - rspec-support (~> 3.10.0) - rspec-expectations (3.10.1) - diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.10.0) - rspec-mocks (3.10.2) - diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.10.0) - rspec-support (3.10.3) - rubocop (1.24.1) - parallel (~> 1.10) - parser (>= 3.0.0.0) - rainbow (>= 2.2.2, < 4.0) - regexp_parser (>= 1.8, < 3.0) - rexml - rubocop-ast (>= 1.15.1, < 2.0) - ruby-progressbar (~> 1.7) - unicode-display_width (>= 1.4.0, < 3.0) - rubocop-ast (1.15.1) - parser (>= 3.0.1.1) - rubocop-performance (1.13.1) - rubocop (>= 1.7.0, < 2.0) - rubocop-ast (>= 0.4.0) - ruby-progressbar (1.11.0) - standard (1.6.0) - rubocop (= 1.24.1) - rubocop-performance (= 1.13.1) - test-unit (3.5.3) - power_assert - unicode-display_width (2.1.0) - -PLATFORMS - arm64-darwin-20 - arm64-darwin-21 - universal-java-11 - universal-java-18 - x64-mingw-ucrt - x86_64-darwin-19 - x86_64-darwin-20 - x86_64-linux - -DEPENDENCIES - minitest - rake - rake-compiler - rspec - standard (~> 1.0) - test-unit - -BUNDLED WITH - 2.4.0.dev diff --git a/tool/bundler/test_gems.rb b/tool/bundler/test_gems.rb index 215d23183e..32cb6b34ee 100644 --- a/tool/bundler/test_gems.rb +++ b/tool/bundler/test_gems.rb @@ -2,11 +2,13 @@ source "https://rubygems.org" -gem "rack", "2.0.8" +gem "rack", "~> 2.0" +gem "base64" gem "webrick", "1.7.0" gem "rack-test", "~> 1.1" -gem "artifice", "~> 0.6.0" -gem "compact_index", "~> 0.13.0" -gem "sinatra", "~> 2.0" -gem "rake", "13.0.1" +gem "compact_index", "~> 0.15.0" +gem "sinatra", "~> 3.0" +gem "rake", "~> 13.1" gem "builder", "~> 3.2" +gem "rb_sys" +gem "rubygems-generate_index", "~> 1.1" diff --git a/tool/bundler/test_gems.rb.lock b/tool/bundler/test_gems.rb.lock deleted file mode 100644 index 8e2f0768a6..0000000000 --- a/tool/bundler/test_gems.rb.lock +++ /dev/null @@ -1,46 +0,0 @@ -GEM - remote: https://rubygems.org/ - specs: - artifice (0.6) - rack-test - builder (3.2.4) - compact_index (0.13.0) - mustermann (1.1.1) - ruby2_keywords (~> 0.0.1) - rack (2.0.8) - rack-protection (2.0.8.1) - rack - rack-test (1.1.0) - rack (>= 1.0, < 3) - rake (13.0.1) - ruby2_keywords (0.0.5) - sinatra (2.0.8.1) - mustermann (~> 1.0) - rack (~> 2.0) - rack-protection (= 2.0.8.1) - tilt (~> 2.0) - tilt (2.0.10) - webrick (1.7.0) - -PLATFORMS - java - ruby - universal-java-11 - universal-java-18 - x64-mingw-ucrt - x64-mingw32 - x86_64-darwin-20 - x86_64-linux - -DEPENDENCIES - artifice (~> 0.6.0) - builder (~> 3.2) - compact_index (~> 0.13.0) - rack (= 2.0.8) - rack-test (~> 1.1) - rake (= 13.0.1) - sinatra (~> 2.0) - webrick (= 1.7.0) - -BUNDLED WITH - 2.4.0.dev diff --git a/tool/bundler/vendor_gems.rb b/tool/bundler/vendor_gems.rb new file mode 100644 index 0000000000..f02d02656d --- /dev/null +++ b/tool/bundler/vendor_gems.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +gem "fileutils", "1.7.2" +gem "molinillo", github: "cocoapods/molinillo" +gem "net-http", "0.4.0" +gem "net-http-persistent", "4.0.2" +gem "net-protocol", "0.2.2" +gem "optparse", "0.4.0" +gem "pub_grub", github: "jhawthorn/pub_grub" +gem "resolv", "0.4.0" +gem "timeout", "0.4.1" +gem "thor", "1.3.0" +gem "tsort", "0.2.0" diff --git a/tool/checksum.rb b/tool/checksum.rb index bcc60ee14a..8f2d1d97d0 100755 --- a/tool/checksum.rb +++ b/tool/checksum.rb @@ -36,9 +36,7 @@ class Checksum end def update! - open(@checksum, "wb") {|f| - f.puts("src=\"#{@source}\", len=#{@len}, checksum=#{@sum}") - } + File.binwrite(@checksum, "src=\"#{@source}\", len=#{@len}, checksum=#{@sum}") end def update diff --git a/tool/ci_functions.sh b/tool/ci_functions.sh deleted file mode 100644 index 7066bbe4ec..0000000000 --- a/tool/ci_functions.sh +++ /dev/null @@ -1,29 +0,0 @@ -# -*- BASH -*- -# Manage functions used on a CI. -# Run `. tool/ci_functions.sh` to use it. - -# Create options with patterns `-n !/name1/ -n !/name2/ ..` to exclude the test -# method names by the method names `name1 name2 ..`. -# See `ruby tool/test/runner.rb --help` `-n` option. -function ci_to_excluded_test_opts { - local tests_str="${1}" - # Use the backward matching `!/name$/`, as the perfect matching doesn't work. - # https://bugs.ruby-lang.org/issues/16936 - ruby <<EOF - opts = "${tests_str}".split.map { |test| "-n \!/#{test}\$$/" } - puts opts.join(' ') -EOF - return 0 -} - -# Create options with patterns `-n name1 -n name2 ..` to include the test -# method names by the method names `name1 name2 ..`. -# See `ruby tool/test/runner.rb --help` `-n` option. -function ci_to_included_test_opts { - local tests_str="${1}" - ruby <<EOF - opts = "${tests_str}".split.map { |test| "-n #{test}" } - puts opts.join(' ') -EOF - return 0 -} diff --git a/tool/darwin-ar b/tool/darwin-ar new file mode 100755 index 0000000000..8b25425cfe --- /dev/null +++ b/tool/darwin-ar @@ -0,0 +1,6 @@ +#!/bin/bash +export LANG=C LC_ALL=C # Suppress localication +exec 2> >(exec grep -v \ + -e ' no symbols$' \ + >&2) +exec "$@" diff --git a/tool/darwin-cc b/tool/darwin-cc index 6eee96e435..42637022a4 100755 --- a/tool/darwin-cc +++ b/tool/darwin-cc @@ -2,5 +2,8 @@ exec 2> >(exec grep -v \ -e '^ld: warning: The [a-z0-9_][a-z0-9_]* architecture is deprecated for macOS' \ -e '^ld: warning: text-based stub file /System/Library/Frameworks/' \ + -e '^ld: warning: ignoring duplicate libraries:' \ + -e "warning: '\.debug_macinfo' is not currently supported:" \ + -e "note: while processing" \ >&2) exec "$@" diff --git a/tool/downloader.rb b/tool/downloader.rb index d3a9f75637..3a91ea0b93 100644 --- a/tool/downloader.rb +++ b/tool/downloader.rb @@ -36,6 +36,12 @@ else end class Downloader + def self.find(dlname) + constants.find do |name| + return const_get(name) if dlname.casecmp(name.to_s) == 0 + end + end + def self.https=(https) @@https = https end @@ -48,13 +54,18 @@ class Downloader @@https end + def self.get_option(argv, options) + false + end + class GNU < self def self.download(name, *rest) if https? begin super("https://cdn.jsdelivr.net/gh/gcc-mirror/gcc@master/#{name}", name, *rest) rescue => e - STDERR.puts "Download failed (#{e.message}), try another URL" + m1, m2 = e.message.split("\n", 2) + STDERR.puts "Download failed (#{m1}), try another URL\n#{m2}" super("https://raw.githubusercontent.com/gcc-mirror/gcc/master/#{name}", name, *rest) end else @@ -68,6 +79,9 @@ class Downloader require 'rubygems' options = options.dup options[:ssl_ca_cert] = Dir.glob(File.expand_path("../lib/rubygems/ssl_certs/**/*.pem", File.dirname(__FILE__))) + if Gem::Version.new(name[/-\K[^-]*(?=\.gem\z)/]).prerelease? + options[:ignore_http_client_errors] = true + end super("https://rubygems.org/downloads/#{name}", name, dir, since, options) end end @@ -78,6 +92,21 @@ class Downloader INDEX = {} # cache index file information across files in the same directory UNICODE_PUBLIC = "https://www.unicode.org/Public/" + def self.get_option(argv, options) + case argv[0] + when '--unicode-beta' + options[:unicode_beta] = argv[1] + argv.shift(2) + true + when /\A--unicode-beta=(.*)/m + options[:unicode_beta] = $1 + argv.shift + true + else + super + end + end + def self.download(name, dir = nil, since = true, options = {}) options = options.dup unicode_beta = options.delete(:unicode_beta) @@ -173,7 +202,6 @@ class Downloader options = options.dup url = URI(url) dryrun = options.delete(:dryrun) - options.delete(:unicode_beta) # just to be on the safe side for gems and gcc if name file = Pathname.new(under(dir, name)) @@ -212,6 +240,7 @@ class Downloader $stdout.flush end mtime = nil + ignore_http_client_errors = options.delete(:ignore_http_client_errors) options = options.merge(http_options(file, since.nil? ? true : since)) begin data = with_retry(10) do @@ -222,12 +251,18 @@ class Downloader data end rescue OpenURI::HTTPError => http_error - if http_error.message =~ /^304 / # 304 Not Modified + case http_error.message + when /^304 / # 304 Not Modified if $VERBOSE $stdout.puts "#{name} not modified" $stdout.flush end return file.to_path + when /^40/ # Net::HTTPClientError: 403 Forbidden, 404 Not Found + if ignore_http_client_errors + puts "Ignore #{url}: #{http_error.message}" + return file.to_path + end end raise rescue Timeout::Error @@ -245,6 +280,7 @@ class Downloader end dest = (cache_save && cache && !cache.exist? ? cache : file) dest.parent.mkpath + dest.unlink if dest.symlink? && !dest.exist? dest.open("wb", 0600) do |f| f.write(data) f.chmod(mode_for(data)) @@ -288,7 +324,16 @@ class Downloader return true if cache.eql?(file) if /cygwin/ !~ RUBY_PLATFORM or /winsymlink:nativestrict/ =~ ENV['CYGWIN'] begin - file.make_symlink(cache.relative_path_from(file.parent)) + link = cache.relative_path_from(file.parent) + rescue ArgumentError + abs = cache.expand_path + link = abs.relative_path_from(file.parent.expand_path) + if link.to_s.count("/") > abs.to_s.count("/") + link = abs + end + end + begin + file.make_symlink(link) rescue SystemCallError else if verbose @@ -351,45 +396,73 @@ Downloader.https = https.freeze if $0 == __FILE__ since = true options = {} + dl = nil + (args = []).singleton_class.__send__(:define_method, :downloader?) do |arg| + !dl and args.empty? and (dl = Downloader.find(arg)) + end until ARGV.empty? + if ARGV[0] == '--' + ARGV.shift + break if ARGV.empty? + ARGV.shift if args.downloader? ARGV[0] + args.concat(ARGV) + break + end + + if dl and dl.get_option(ARGV, options) + # the downloader dealt with the arguments, and should be removed + # from ARGV. + next + end + case ARGV[0] - when '-d' + when '-d', '--destdir' + ## -d, --destdir DIRECTORY Download into the directory destdir = ARGV[1] ARGV.shift - when '-p' - # strip directory names from the name to download, and add the - # prefix instead. + when '-p', '--prefix' + ## -p, --prefix Strip directory names from the name to download, + ## and add the prefix instead. prefix = ARGV[1] ARGV.shift - when '-e' + when '-e', '--exist', '--non-existent-only' + ## -e, --exist, --non-existent-only Skip already existent files. since = nil - when '-a' + when '-a', '--always' + ## -a, --always Download all files. since = false - when '-n', '--dryrun' + when '-u', '--update', '--if-modified' + ## -u, --update, --if-modified Download newer files only. + since = true + when '-n', '--dry-run', '--dryrun' + ## -n, --dry-run Do not download actually. options[:dryrun] = true when '--cache-dir' + ## --cache-dir DIRECTORY Cache downloaded files in the directory. options[:cache_dir] = ARGV[1] ARGV.shift - when '--unicode-beta' - options[:unicode_beta] = ARGV[1] - ARGV.shift when /\A--cache-dir=(.*)/m options[:cache_dir] = $1 + when /\A--help\z/ + ## --help Print this message + puts "Usage: #$0 [options] relative-url..." + File.foreach(__FILE__) do |line| + line.sub!(/^ *## /, "") or next + break if line.chomp!.empty? + opt, desc = line.split(/ {2,}/, 2) + printf " %-28s %s\n", opt, desc + end + exit when /\A-/ abort "#{$0}: unknown option #{ARGV[0]}" else - break + args << ARGV[0] unless args.downloader? ARGV[0] end ARGV.shift end - dl = Downloader.constants.find do |name| - ARGV[0].casecmp(name.to_s) == 0 - end unless ARGV.empty? $VERBOSE = true if dl - dl = Downloader.const_get(dl) - ARGV.shift - ARGV.each do |name| + args.each do |name| dir = destdir if prefix name = name.sub(/\A\.\//, '') @@ -409,7 +482,7 @@ if $0 == __FILE__ dl.download(name, dir, since, options) end else - abort "usage: #{$0} url name" unless ARGV.size == 2 - Downloader.download(ARGV[0], ARGV[1], destdir, since, options) + abort "usage: #{$0} url name" unless args.size == 2 + Downloader.download(args[0], args[1], destdir, since, options) end end diff --git a/tool/dummy-rake-compiler/rake/extensiontask.rb b/tool/dummy-rake-compiler/rake/extensiontask.rb deleted file mode 100644 index 62b7ff8018..0000000000 --- a/tool/dummy-rake-compiler/rake/extensiontask.rb +++ /dev/null @@ -1,9 +0,0 @@ -module Rake - class ExtensionTask < TaskLib - def initialize(...) - task :compile do - puts "Dummy `compile` task defined in #{__FILE__}" - end - end - end -end diff --git a/tool/enc-case-folding.rb b/tool/enc-case-folding.rb new file mode 100755 index 0000000000..82fec7b625 --- /dev/null +++ b/tool/enc-case-folding.rb @@ -0,0 +1,416 @@ +#!/usr/bin/ruby +require 'stringio' + +# Usage (for case folding only): +# $ wget http://www.unicode.org/Public/UNIDATA/CaseFolding.txt +# $ ruby enc-case-folding.rb CaseFolding.txt -o casefold.h +# or (for case folding and case mapping): +# $ wget http://www.unicode.org/Public/UNIDATA/CaseFolding.txt +# $ wget http://www.unicode.org/Public/UNIDATA/UnicodeData.txt +# $ wget http://www.unicode.org/Public/UNIDATA/SpecialCasing.txt +# $ ruby enc-case-folding.rb -m . -o casefold.h +# using -d or --debug will include UTF-8 characters in comments for debugging + +class CaseFolding + module Util + module_function + + def hex_seq(v) + v.map { |i| "0x%04x" % i }.join(", ") + end + + def print_table_1(dest, type, mapping_data, data) + for k, v in data = data.sort + sk = (Array === k and k.length > 1) ? "{#{hex_seq(k)}}" : ("0x%04x" % k) + if type=='CaseUnfold_11' and v.length>1 + # reorder CaseUnfold_11 entries to avoid special treatment for U+03B9/U+03BC/U+A64B + item = mapping_data.map("%04X" % k[0]) + upper = item.upper if item + v = v.sort_by { |i| ("%04X"%i) == upper ? 0 : 1 } + end + ck = @debug ? ' /* ' + Array(k).pack("U*") + ' */' : '' + cv = @debug ? ' /* ' + Array(v).map{|c|[c].pack("U*")}.join(", ") + ' */' : '' + dest.print(" {#{sk}#{ck}, {#{v.length}#{mapping_data.flags(k, type, v)}, {#{hex_seq(v)}#{cv}}}},\n") + end + data + end + + def print_table(dest, type, mapping_data, data) + dest.print("static const #{type}_Type #{type}_Table[] = {\n") + i = 0 + ret = data.inject([]) do |a, (n, d)| + dest.print("#define #{n} (*(#{type}_Type (*)[#{d.size}])(#{type}_Table+#{i}))\n") + i += d.size + a.concat(print_table_1(dest, type, mapping_data, d)) + end + dest.print("};\n\n") + ret + end + end + + include Util + + attr_reader :fold, :fold_locale, :unfold, :unfold_locale, :version + + def load(filename) + pattern = /([0-9A-F]{4,6}); ([CFT]); ([0-9A-F]{4,6})(?: ([0-9A-F]{4,6}))?(?: ([0-9A-F]{4,6}))?;/ + + @fold = fold = {} + @unfold = unfold = [{}, {}, {}] + @debug = false + @version = nil + turkic = [] + + File.foreach(filename, mode: "rb") do |line| + @version ||= line[/-([0-9.]+).txt/, 1] + next unless res = pattern.match(line) + ch_from = res[1].to_i(16) + + if res[2] == 'T' + # Turkic case folding + turkic << ch_from + next + end + + # store folding data + ch_to = res[3..6].inject([]) do |a, i| + break a unless i + a << i.to_i(16) + end + fold[ch_from] = ch_to + + # store unfolding data + i = ch_to.length - 1 + (unfold[i][ch_to] ||= []) << ch_from + end + + # move locale dependent data to (un)fold_locale + @fold_locale = fold_locale = {} + @unfold_locale = unfold_locale = [{}, {}] + for ch_from in turkic + key = fold[ch_from] + i = key.length - 1 + unfold_locale[i][i == 0 ? key[0] : key] = unfold[i].delete(key) + fold_locale[ch_from] = fold.delete(ch_from) + end + self + end + + def range_check(code) + "#{code} <= MAX_CODE_VALUE && #{code} >= MIN_CODE_VALUE" + end + + def lookup_hash(key, type, data) + hash = "onigenc_unicode_#{key}_hash" + lookup = "onigenc_unicode_#{key}_lookup" + arity = Array(data[0][0]).size + gperf = %W"gperf -7 -k#{[*1..(arity*3)].join(',')} -F,-1 -c -j1 -i1 -t -T -E -C -H #{hash} -N #{lookup} -n" + argname = arity > 1 ? "codes" : "code" + argdecl = "const OnigCodePoint #{arity > 1 ? "*": ""}#{argname}" + n = 7 + m = (1 << n) - 1 + min, max = data.map {|c, *|c}.flatten.minmax + src = IO.popen(gperf, "r+") {|f| + f << "short\n%%\n" + data.each_with_index {|(k, _), i| + k = Array(k) + ks = k.map {|j| [(j >> n*2) & m, (j >> n) & m, (j) & m]}.flatten.map {|c| "\\x%.2x" % c}.join("") + f.printf "\"%s\", ::::/*%s*/ %d\n", ks, k.map {|c| "0x%.4x" % c}.join(","), i + } + f << "%%\n" + f.close_write + f.read + } + src.sub!(/^(#{hash})\s*\(.*?\).*?\n\{\n(.*)^\}/m) { + name = $1 + body = $2 + body.gsub!(/\(unsigned char\)str\[(\d+)\]/, "bits_#{arity > 1 ? 'at' : 'of'}(#{argname}, \\1)") + "#{name}(#{argdecl})\n{\n#{body}}" + } + src.sub!(/const short *\*\n^(#{lookup})\s*\(.*?\).*?\n\{\n(.*)^\}/m) { + name = $1 + body = $2 + body.sub!(/\benum\s+\{(\n[ \t]+)/, "\\&MIN_CODE_VALUE = 0x#{min.to_s(16)},\\1""MAX_CODE_VALUE = 0x#{max.to_s(16)},\\1") + body.gsub!(/(#{hash})\s*\(.*?\)/, "\\1(#{argname})") + body.gsub!(/\{"",-1}/, "-1") + body.gsub!(/\{"(?:[^"]|\\")+", *::::(.*)\}/, '\1') + body.sub!(/(\s+if\s)\(len\b.*\)/) do + "#$1(" << + (arity > 1 ? (0...arity).map {|i| range_check("#{argname}[#{i}]")}.join(" &&\n ") : range_check(argname)) << + ")" + end + v = nil + body.sub!(/(if\s*\(.*MAX_HASH_VALUE.*\)\n([ \t]*))\{(.*?)\n\2\}/m) { + pre = $1 + indent = $2 + s = $3 + s.sub!(/const char *\* *(\w+)( *= *wordlist\[\w+\]).\w+/, 'short \1 = wordlist[key]') + v = $1 + s.sub!(/\bif *\(.*\)/, "if (#{v} >= 0 && code#{arity}_equal(#{argname}, #{key}_Table[#{v}].from))") + "#{pre}{#{s}\n#{indent}}" + } + body.sub!(/\b(return\s+&)([^;]+);/, '\1'"#{key}_Table[#{v}].to;") + "static const #{type} *\n#{name}(#{argdecl})\n{\n#{body}}" + } + src + end + + def display(dest, mapping_data) + # print the header + dest.print("/* DO NOT EDIT THIS FILE. */\n") + dest.print("/* Generated by enc-case-folding.rb */\n\n") + + versions = version.scan(/\d+/) + dest.print("#if defined ONIG_UNICODE_VERSION_STRING && !( \\\n") + %w[MAJOR MINOR TEENY].zip(versions) do |n, v| + dest.print(" ONIG_UNICODE_VERSION_#{n} == #{v} && \\\n") + end + dest.print(" 1)\n") + dest.print("# error ONIG_UNICODE_VERSION_STRING mismatch\n") + dest.print("#endif\n") + dest.print("#define ONIG_UNICODE_VERSION_STRING #{version.dump}\n") + %w[MAJOR MINOR TEENY].zip(versions) do |n, v| + dest.print("#define ONIG_UNICODE_VERSION_#{n} #{v}\n") + end + dest.print("\n") + + # print folding data + + # CaseFold + CaseFold_Locale + name = "CaseFold_11" + data = print_table(dest, name, mapping_data, "CaseFold"=>fold, "CaseFold_Locale"=>fold_locale) + dest.print lookup_hash(name, "CodePointList3", data) + + # print unfolding data + + # CaseUnfold_11 + CaseUnfold_11_Locale + name = "CaseUnfold_11" + data = print_table(dest, name, mapping_data, name=>unfold[0], "#{name}_Locale"=>unfold_locale[0]) + dest.print lookup_hash(name, "CodePointList3", data) + + # CaseUnfold_12 + CaseUnfold_12_Locale + name = "CaseUnfold_12" + data = print_table(dest, name, mapping_data, name=>unfold[1], "#{name}_Locale"=>unfold_locale[1]) + dest.print lookup_hash(name, "CodePointList2", data) + + # CaseUnfold_13 + name = "CaseUnfold_13" + data = print_table(dest, name, mapping_data, name=>unfold[2]) + dest.print lookup_hash(name, "CodePointList2", data) + + # TitleCase + dest.print mapping_data.specials_output + end + + def debug! + @debug = true + end + + def self.load(*args) + new.load(*args) + end +end + +class MapItem + attr_accessor :upper, :lower, :title, :code + + def initialize(code, upper, lower, title) + @code = code + @upper = upper unless upper == '' + @lower = lower unless lower == '' + @title = title unless title == '' + end +end + +class CaseMapping + attr_reader :filename, :version + + def initialize(mapping_directory) + @mappings = {} + @specials = [] + @specials_length = 0 + @version = nil + File.foreach(File.join(mapping_directory, 'UnicodeData.txt'), mode: "rb") do |line| + next if line =~ /^</ + code, _, _, _, _, _, _, _, _, _, _, _, upper, lower, title = line.chomp.split ';' + unless upper and lower and title and (upper+lower+title)=='' + @mappings[code] = MapItem.new(code, upper, lower, title) + end + end + + @filename = File.join(mapping_directory, 'SpecialCasing.txt') + File.foreach(@filename, mode: "rb") do |line| + @version ||= line[/-([0-9.]+).txt/, 1] + line.chomp! + line, comment = line.split(/ *#/) + next if not line or line == '' + code, lower, title, upper, conditions = line.split(/ *; */) + unless conditions + item = @mappings[code] + item.lower = lower + item.title = title + item.upper = upper + end + end + end + + def map (from) + @mappings[from] + end + + def flags(from, type, to) + # types: CaseFold_11, CaseUnfold_11, CaseUnfold_12, CaseUnfold_13 + flags = "" + from = Array(from).map {|i| "%04X" % i}.join(" ") + to = Array(to).map {|i| "%04X" % i}.join(" ") + item = map(from) + specials = [] + case type + when 'CaseFold_11' + flags += '|F' + if item + flags += '|U' if to==item.upper + flags += '|D' if to==item.lower + unless item.upper == item.title + if item.code == item.title + flags += '|IT' + swap = case item.code + when '01C5' then '0064 017D' + when '01C8' then '006C 004A' + when '01CB' then '006E 004A' + when '01F2' then '0064 005A' + else # Greek + to.split(' ').first + ' 0399' + end + specials << swap + else + flags += '|ST' + specials << item.title + end + end + unless item.lower.nil? or item.lower==from or item.lower==to + specials << item.lower + flags += '|SL' + end + unless item.upper.nil? or item.upper==from or item.upper==to + specials << item.upper + flags += '|SU' + end + end + when 'CaseUnfold_11' + to = to.split(/ /) + if item + case to.first + when item.upper then flags += '|U' + when item.lower then flags += '|D' + else + raise "Unpredicted case 0 in enc/unicode/case_folding.rb. Please contact https://bugs.ruby-lang.org/." + end + unless item.upper == item.title + if item.code == item.title + flags += '|IT' # was unpredicted case 1 + elsif item.title==to[1] + flags += '|ST' + else + raise "Unpredicted case 2 in enc/unicode/case_folding.rb. Please contact https://bugs.ruby-lang.org/." + end + end + end + end + unless specials.empty? + flags += "|I(#{@specials_length})" + @specials_length += specials.map { |s| s.split(/ /).length }.reduce(:+) + @specials << specials + end + flags + end + + def debug! + @debug = true + end + + def specials_output + "static const OnigCodePoint CaseMappingSpecials[] = {\n" + + @specials.map do |sps| + ' ' + sps.map do |sp| + chars = sp.split(/ /) + ct = ' /* ' + Array(chars).map{|c|[c.to_i(16)].pack("U*")}.join(", ") + ' */' if @debug + " L(#{chars.length})|#{chars.map {|c| "0x"+c }.join(', ')}#{ct}," + end.join + "\n" + end.join + "};\n" + end + + def self.load(*args) + new(*args) + end +end + +class CaseMappingDummy + def flags(from, type, to) + "" + end + + def titlecase_output() '' end + def debug!() end +end + +if $0 == __FILE__ + require 'optparse' + dest = nil + mapping_directory = nil + mapping_data = nil + debug = false + fold_1 = false + ARGV.options do |opt| + opt.banner << " [INPUT]" + opt.on("--output-file=FILE", "-o", "output to the FILE instead of STDOUT") {|output| + dest = (output unless output == '-') + } + opt.on('--mapping-data-directory=DIRECTORY', '-m', 'data DIRECTORY of mapping files') { |directory| + mapping_directory = directory + } + opt.on('--debug', '-d') { + debug = true + } + opt.parse! + abort(opt.to_s) if ARGV.size > 1 + end + if mapping_directory + if ARGV[0] + warn "Either specify directory or individual file, but not both." + exit + end + filename = File.join(mapping_directory, 'CaseFolding.txt') + mapping_data = CaseMapping.load(mapping_directory) + end + filename ||= ARGV[0] || 'CaseFolding.txt' + data = CaseFolding.load(filename) + if mapping_data and data.version != mapping_data.version + abort "Unicode data version mismatch\n" \ + " #{filename} = #{data.version}\n" \ + " #{mapping_data.filename} = #{mapping_data.version}" + end + mapping_data ||= CaseMappingDummy.new + + if debug + data.debug! + mapping_data.debug! + end + f = StringIO.new + begin + data.display(f, mapping_data) + rescue Errno::ENOENT => e + raise unless /gperf/ =~ e.message + warn e.message + abort unless dest + File.utime(nil, nil, dest) # assume existing file is OK + exit + else + s = f.string + end + if dest + File.binwrite(dest, s) + else + STDOUT.print(s) + end +end diff --git a/tool/enc-emoji-citrus-gen.rb b/tool/enc-emoji-citrus-gen.rb index da9c8a6b62..0b37e48d3f 100644 --- a/tool/enc-emoji-citrus-gen.rb +++ b/tool/enc-emoji-citrus-gen.rb @@ -71,7 +71,7 @@ end def generate_to_ucs(params, pairs) pairs.sort_by! {|u, c| c } name = "EMOJI_#{params[:name]}%UCS" - open("#{name}.src", "w") do |io| + File.open("#{name}.src", "w") do |io| io.print header(params.merge(name: name.tr('%', '/'))) io.puts io.puts "BEGIN_MAP" @@ -83,7 +83,7 @@ end def generate_from_ucs(params, pairs) pairs.sort_by! {|u, c| u } name = "UCS%EMOJI_#{params[:name]}" - open("#{name}.src", "w") do |io| + File.open("#{name}.src", "w") do |io| io.print header(params.merge(name: name.tr('%', '/'))) io.puts io.puts "BEGIN_MAP" diff --git a/tool/enc-unicode.rb b/tool/enc-unicode.rb index 93f6e869f8..9d49f427bb 100755 --- a/tool/enc-unicode.rb +++ b/tool/enc-unicode.rb @@ -5,14 +5,30 @@ # # To use this, get UnicodeData.txt, Scripts.txt, PropList.txt, # PropertyAliases.txt, PropertyValueAliases.txt, DerivedCoreProperties.txt, -# DerivedAge.txt and Blocks.txt from unicode.org. +# DerivedAge.txt, Blocks.txt, emoji/emoji-data.txt, +# auxiliary/GraphemeBreakProperty.txt from unicode.org # (http://unicode.org/Public/UNIDATA/) And run following command. -# ruby1.9 tool/enc-unicode.rb data_dir > enc/unicode/name2ctype.kwd +# tool/enc-unicode.rb data_dir emoji_data_dir > enc/unicode/name2ctype.kwd # You can get source file for gperf. After this, simply make ruby. - -if ARGV[0] == "--header" - header = true - ARGV.shift +# Or directly run: +# tool/enc-unicode.rb --header data_dir emoji_data_dir > enc/unicode/<VERSION>/name2ctype.h + +while arg = ARGV.shift + case arg + when "--" + break + when "--header" + header = true + when "--diff" + diff = ARGV.shift or abort "#{$0}: --diff=DIFF-COMMAND" + when /\A--diff=(.+)/m + diff = $1 + when /\A-/ + abort "#{$0}: unknown option #{arg}" + else + ARGV.unshift(arg) + break + end end unless ARGV.size == 2 abort "Usage: #{$0} data_directory emoji_data_directory" @@ -59,7 +75,7 @@ def parse_unicode_data(file) data = {'Any' => (0x0000..0x10ffff).to_a, 'Assigned' => [], 'ASCII' => (0..0x007F).to_a, 'NEWLINE' => [0x0a], 'Cn' => []} beg_cp = nil - IO.foreach(file) do |line| + File.foreach(file) do |line| fields = line.split(';') cp = fields[0].to_i(16) @@ -150,7 +166,7 @@ def parse_scripts(data, categories) categories[current] = file[:title] (names[file[:title]] ||= []) << current cps = [] - elsif /^([0-9a-fA-F]+)(?:\.\.([0-9a-fA-F]+))?\s*;\s*(\w+)/ =~ line + elsif /^(\h+)(?:\.\.(\h+))?\s*;\s*(\w+)/ =~ line current = $3 $2 ? cps.concat(($1.to_i(16)..$2.to_i(16)).to_a) : cps.push($1.to_i(16)) end @@ -205,7 +221,7 @@ def parse_age(data) ages << current last_constname = constname cps = [] - elsif /^([0-9a-fA-F]+)(?:\.\.([0-9a-fA-F]+))?\s*;\s*(\d+\.\d+)/ =~ line + elsif /^(\h+)(?:\.\.(\h+))?\s*;\s*(\d+\.\d+)/ =~ line current = $3 $2 ? cps.concat(($1.to_i(16)..$2.to_i(16)).to_a) : cps.push($1.to_i(16)) end @@ -224,7 +240,7 @@ def parse_GraphemeBreakProperty(data) make_const(constname, cps, "Grapheme_Cluster_Break=#{current}") ages << current cps = [] - elsif /^([0-9a-fA-F]+)(?:\.\.([0-9a-fA-F]+))?\s*;\s*(\w+)/ =~ line + elsif /^(\h+)(?:\.\.(\h+))?\s*;\s*(\w+)/ =~ line current = $3 $2 ? cps.concat(($1.to_i(16)..$2.to_i(16)).to_a) : cps.push($1.to_i(16)) end @@ -236,7 +252,7 @@ def parse_block(data) cps = [] blocks = [] data_foreach('Blocks.txt') do |line| - if /^([0-9a-fA-F]+)\.\.([0-9a-fA-F]+);\s*(.*)/ =~ line + if /^(\h+)\.\.(\h+);\s*(.*)/ =~ line cps = ($1.to_i(16)..$2.to_i(16)).to_a constname = constantize_blockname($3) data[constname] = cps @@ -253,23 +269,12 @@ def parse_block(data) blocks << constname end -# shim for Ruby 1.8 -unless {}.respond_to?(:key) - class Hash - alias key index - end -end - $const_cache = {} # make_const(property, pairs, name): Prints a 'static const' structure for a # given property, group of paired codepoints, and a human-friendly name for # the group def make_const(prop, data, name) - if name.empty? - puts "\n/* '#{prop}' */" - else - puts "\n/* '#{prop}': #{name} */" - end + puts "\n/* '#{prop}': #{name} */" # comment used to generate documentation if origprop = $const_cache.key(data) puts "#define CR_#{prop} CR_#{origprop}" else @@ -311,18 +316,19 @@ end def data_foreach(name, &block) fn = get_file(name) warn "Reading #{name}" - if /^emoji/ =~ name - sep = "" - pat = /^# #{Regexp.quote(File.basename(name))}.*^# Version: ([\d.]+)/m - type = :Emoji - else - sep = "\n" - pat = /^# #{File.basename(name).sub(/\./, '-([\\d.]+)\\.')}/ - type = :Unicode - end File.open(fn, 'rb') do |f| - line = f.gets(sep) - unless version = line[pat, 1] + if /^emoji/ =~ name + line = f.gets("") + # Headers till Emoji 13 or 15 + version = line[/^# #{Regexp.quote(File.basename(name))}.*(?:^# Version:|Emoji Version) ([\d.]+)/m, 1] + type = :Emoji + else + # Headers since Emoji 14 or other Unicode data + line = f.gets("\n") + type = :Unicode + end + version ||= line[/^# #{File.basename(name).sub(/\./, '-([\\d.]+)\\.')}/, 1] + unless version raise ArgumentError, <<-ERROR #{name}: no #{type} version #{line.gsub(/^/, '> ')} @@ -330,7 +336,7 @@ def data_foreach(name, &block) end if !(v = $versions[type]) $versions[type] = version - elsif v != version + elsif v != version and "#{v}.0" != version raise ArgumentError, <<-ERROR #{name}: #{type} version mismatch: #{version} to #{v} #{line.gsub(/^/, '> ')} @@ -420,8 +426,6 @@ define_posix_props(data) POSIX_NAMES.each do |name| if name == 'XPosixPunct' make_const(name, data[name], "[[:Punct:]]") - elsif name == 'Punct' - make_const(name, data[name], "") else make_const(name, data[name], "[[:#{name}:]]") end @@ -464,11 +468,7 @@ struct uniname2ctype_struct { }; #define uniname2ctype_offset(str) offsetof(struct uniname2ctype_pool_t, uniname2ctype_pool_##str) -static const struct uniname2ctype_struct *uniname2ctype_p( -#if !(/*ANSI*/+0) /* if ANSI, old style not to conflict with generated prototype */ - const char *, unsigned int -#endif -); +static const struct uniname2ctype_struct *uniname2ctype_p(register const char *str, register size_t len); %} struct uniname2ctype_struct; %% @@ -547,6 +547,47 @@ output.restore if header require 'tempfile' + def diff_args(diff) + ok = IO.popen([diff, "-DDIFF_TEST", IO::NULL, "-"], "r+") do |f| + f.puts "Test for diffutils 3.8" + f.close_write + /^#if/ =~ f.read + end + if ok + proc {|macro, *inputs| + [diff, "-D#{macro}", *inputs] + } + else + IO.popen([diff, "--old-group-format=%<", "--new-group-format=%>", IO::NULL, IO::NULL], err: %i[child out], &:read) + unless $?.success? + abort "#{$0}: #{diff} -D does not work" + end + warn "Avoiding diffutils 3.8 bug#61193" + proc {|macro, *inputs| + [diff] + [ + "--old-group-format=" \ + "#ifndef @\n" \ + "%<" \ + "#endif /* ! @ */\n", + + "--new-group-format=" \ + "#ifdef @\n" \ + "%>" \ + "#endif /* @ */\n", + + "--changed-group-format=" \ + "#ifndef @\n" \ + "%<" \ + "#else /* @ */\n" \ + "%>" \ + "#endif /* @ */\n" + ].map {|opt| opt.gsub(/@/) {macro}} + inputs + } + end + end + + ifdef = diff_args(diff || "diff") + NAME2CTYPE = %w[gperf -7 -c -j1 -i1 -t -C -P -T -H uniname2ctype_hash -Q uniname2ctype_pool -N uniname2ctype_p] fds = [] @@ -556,9 +597,8 @@ if header IO.popen([*NAME2CTYPE, out: tmp], "w") {|f| output.show(f, *syms)} end while syms.pop fds.each(&:close) - ff = nil - IO.popen(%W[diff -DUSE_UNICODE_AGE_PROPERTIES #{fds[1].path} #{fds[0].path}], "r") {|age| - IO.popen(%W[diff -DUSE_UNICODE_PROPERTIES #{fds[2].path} -], "r", in: age) {|f| + IO.popen(ifdef["USE_UNICODE_AGE_PROPERTIES", fds[1].path, fds[0].path], "r") {|age| + IO.popen(ifdef["USE_UNICODE_PROPERTIES", fds[2].path, "-"], "r", in: age) {|f| ansi = false f.each {|line| if /ANSI-C code produced by gperf/ =~ line @@ -567,7 +607,7 @@ if header line.sub!(/\/\*ANSI\*\//, '1') if ansi line.gsub!(/\(int\)\((?:long|size_t)\)&\(\(struct uniname2ctype_pool_t \*\)0\)->uniname2ctype_pool_(str\d+),\s+/, 'uniname2ctype_offset(\1), ') - if ff = (!ff ? /^(uniname2ctype_hash) /=~line : /^\}/!~line) # no line can match both, exclusive flip-flop + if line.start_with?("uniname2ctype_hash\s") ... line.start_with?("}") line.sub!(/^( *(?:register\s+)?(.*\S)\s+hval\s*=\s*)(?=len;)/, '\1(\2)') end puts line diff --git a/tool/expand-config.rb b/tool/expand-config.rb index 81ffa6cb98..ac0ffbfd41 100755 --- a/tool/expand-config.rb +++ b/tool/expand-config.rb @@ -7,7 +7,17 @@ config.sub!(/^(\s*)RUBY_VERSION\b.*(\sor\s*)$/, '\1true\2') rbconfig = Module.new {module_eval(config, conffile)}::RbConfig config = $expand ? rbconfig::CONFIG : rbconfig::MAKEFILE_CONFIG config["RUBY_RELEASE_DATE"] ||= - File.read(File.expand_path("../../version.h", __FILE__))[/^\s*#\s*define\s+RUBY_RELEASE_DATE\s+"(.*)"/, 1] + [ + ["revision.h"], + ["../../revision.h", __FILE__], + ["../../version.h", __FILE__], + ].find do |hdr, dir| + hdr = File.expand_path(hdr, dir) if dir + if date = File.read(hdr)[/^\s*#\s*define\s+RUBY_RELEASE_DATE(?:TIME)?\s+"([0-9-]*)/, 1] + break date + end +rescue +end while /\A(\w+)=(.*)/ =~ ARGV[0] config[$1] = $2 @@ -16,7 +26,7 @@ while /\A(\w+)=(.*)/ =~ ARGV[0] end if $output - output = open($output, "wb", $mode &&= $mode.oct) + output = File.open($output, "wb", $mode &&= $mode.oct) output.chmod($mode) if $mode else output = STDOUT diff --git a/tool/extlibs.rb b/tool/extlibs.rb index b482258a2c..887cac61eb 100755 --- a/tool/extlibs.rb +++ b/tool/extlibs.rb @@ -185,7 +185,7 @@ class ExtLibs extracted = false dest = File.dirname(list) url = chksums = nil - IO.foreach(list) do |line| + File.foreach(list) do |line| line.sub!(/\s*#.*/, '') if /^(\w+)\s*=\s*(.*)/ =~ line vars[$1] = vars.expand($2) diff --git a/tool/fake.rb b/tool/fake.rb index 91dfb041c4..0366144531 100644 --- a/tool/fake.rb +++ b/tool/fake.rb @@ -9,6 +9,15 @@ class File end end +[[libpathenv, "."], [preloadenv, libruby_so]].each do |env, path| + env or next + e = ENV[env] or next + e = e.split(File::PATH_SEPARATOR) + path = File.realpath(path, builddir) rescue next + e.delete(path) or next + ENV[env] = (e.join(File::PATH_SEPARATOR) unless e.empty?) +end + static = !!(defined?($static) && $static) $:.unshift(builddir) posthook = proc do diff --git a/tool/fetch-bundled_gems.rb b/tool/fetch-bundled_gems.rb index 8d04892b70..f0d3c3cb89 100755 --- a/tool/fetch-bundled_gems.rb +++ b/tool/fetch-bundled_gems.rb @@ -1,6 +1,9 @@ #!ruby -an BEGIN { require 'fileutils' + require_relative 'lib/colorize' + + color = Colorize.new dir = ARGV.shift ARGF.eof? @@ -10,22 +13,34 @@ BEGIN { n, v, u, r = $F +next unless n next if n =~ /^#/ if File.directory?(n) - puts "updating #{n} ..." - system("git", "fetch", chdir: n) or abort + puts "updating #{color.notice(n)} ..." + system("git", "fetch", "--all", chdir: n) or abort else - puts "retrieving #{n} ..." + puts "retrieving #{color.notice(n)} ..." system(*%W"git clone #{u} #{n}") or abort end + if r - puts "fetching #{r} ..." + puts "fetching #{color.notice(r)} ..." system("git", "fetch", "origin", r, chdir: n) or abort end + c = r || "v#{v}" checkout = %w"git -c advice.detachedHead=false checkout" -puts "checking out #{c} (v=#{v}, r=#{r}) ..." +print %[checking out #{color.notice(c)} (v=#{color.info(v)}] +print %[, r=#{color.info(r)}] if r +puts ") ..." unless system(*checkout, c, "--", chdir: n) abort if r or !system(*checkout, v, "--", chdir: n) end + +if r + unless File.exist? "#{n}/#{n}.gemspec" + require_relative "lib/bundled_gem" + BundledGem.dummy_gemspec("#{n}/#{n}.gemspec") + end +end diff --git a/tool/file2lastrev.rb b/tool/file2lastrev.rb index 3d8c69357d..6200e78a56 100755 --- a/tool/file2lastrev.rb +++ b/tool/file2lastrev.rb @@ -8,50 +8,47 @@ require 'optparse' # this file run with BASERUBY, which may be older than 1.9, so no # require_relative require File.expand_path('../lib/vcs', __FILE__) +require File.expand_path('../lib/output', __FILE__) Program = $0 -@output = nil -def self.output=(output) - if @output and @output != output +@format = nil +def self.format=(format) + if @format and @format != format raise "you can specify only one of --changed, --revision.h and --doxygen" end - @output = output + @format = format end @suppress_not_found = false @limit = 20 +@output = Output.new -format = '%Y-%m-%dT%H:%M:%S%z' +time_format = '%Y-%m-%dT%H:%M:%S%z' vcs = nil +create_only = false OptionParser.new {|opts| opts.banner << " paths..." vcs_options = VCS.define_options(opts) - new_vcs = proc do |path| - begin - vcs = VCS.detect(path, vcs_options, opts.new) - rescue VCS::NotFoundError => e - abort "#{File.basename(Program)}: #{e.message}" unless @suppress_not_found - opts.remove - end - nil - end + opts.new {@output.def_options(opts)} + srcdir = nil opts.new opts.on("--srcdir=PATH", "use PATH as source directory") do |path| - abort "#{File.basename(Program)}: srcdir is already set" if vcs - new_vcs[path] + abort "#{File.basename(Program)}: srcdir is already set" if srcdir + srcdir = path + @output.vpath.add(srcdir) end opts.on("--changed", "changed rev") do - self.output = :changed + self.format = :changed end opts.on("--revision.h", "RUBY_REVISION macro") do - self.output = :revision_h + self.format = :revision_h end opts.on("--doxygen", "Doxygen format") do - self.output = :doxygen + self.format = :doxygen end opts.on("--modified[=FORMAT]", "modified time") do |fmt| - self.output = :modified - format = fmt if fmt + self.format = :modified + time_format = fmt if fmt end opts.on("--limit=NUM", "limit branch name length (#@limit)", Integer) do |n| @limit = n @@ -60,44 +57,27 @@ OptionParser.new {|opts| @suppress_not_found = true end opts.order! rescue abort "#{File.basename(Program)}: #{$!}\n#{opts}" - if vcs - vcs.set_options(vcs_options) # options after --srcdir - else - new_vcs["."] + begin + vcs = VCS.detect(srcdir || ".", vcs_options, opts.new) + rescue VCS::NotFoundError => e + abort "#{File.basename(Program)}: #{e.message}" unless @suppress_not_found + opts.remove + (vcs = VCS::Null.new(nil)).set_options(vcs_options) + if @format == :revision_h + create_only = true # don't overwrite existing revision.h when .git doesn't exist + end end } -exit unless vcs -@output = - case @output +formatter = + case @format when :changed, nil Proc.new {|last, changed| - changed + changed || "" } when :revision_h Proc.new {|last, changed, modified, branch, title| - short = vcs.short_revision(last) - if /[^\x00-\x7f]/ =~ title and title.respond_to?(:force_encoding) - title = title.dup.force_encoding("US-ASCII") - end - [ - "#define RUBY_REVISION #{short.inspect}", - ("#define RUBY_FULL_REVISION #{last.inspect}" unless short == last), - if branch - e = '..' - limit = @limit - name = branch.sub(/\A(.{#{limit-e.size}}).{#{e.size+1},}/o) {$1+e} - name = name.dump.sub(/\\#/, '#') - "#define RUBY_BRANCH_NAME #{name}" - end, - if title - title = title.dump.sub(/\\#/, '#') - "#define RUBY_LAST_COMMIT_TITLE #{title}" - end, - if modified - modified.utc.strftime('#define RUBY_RELEASE_DATETIME "%FT%TZ"') - end, - ].compact + vcs.revision_header(last, modified, modified, branch, title, limit: @limit).join("\n") } when :doxygen Proc.new {|last, changed| @@ -105,18 +85,19 @@ exit unless vcs } when :modified Proc.new {|last, changed, modified| - modified.strftime(format) + modified.strftime(time_format) } else - raise "unknown output format `#{@output}'" + raise "unknown output format `#{@format}'" end ok = true (ARGV.empty? ? [nil] : ARGV).each do |arg| begin - puts @output[*vcs.get_revisions(arg)] + data = formatter[*vcs.get_revisions(arg)] + data.sub!(/(?<!\A|\n)\z/, "\n") + @output.write(data, overwrite: true, create_only: create_only) rescue => e - next if @suppress_not_found and VCS::NotFoundError === e warn "#{File.basename(Program)}: #{e.message}" ok = false end diff --git a/tool/format-release b/tool/format-release index 7b0f8e78e7..737148e0ce 100755 --- a/tool/format-release +++ b/tool/format-release @@ -235,11 +235,11 @@ eom diff.each_with_index do |line, index| case index when 0 - line.sub!(/\A--- (.*)\t(\d+-\d+-\d+ [0-9:.]+ [\-+]\d+)\Z/) do + line.sub!(/\A--- (.*)\t(\d+-\d+-\d+ [0-9:.]+(?: [\-+]\d+)?)\Z/) do "--- a/#{filename}\t#{$2}" end when 1 - line.sub!(/\A\+\+\+ (.*)\t(\d+-\d+-\d+ [0-9:.]+ [\-+]\d+)\Z/) do + line.sub!(/\A\+\+\+ (.*)\t(\d+-\d+-\d+ [0-9:.]+(?: [\-+]\d+)?)\Z/) do "+++ b/#{filename}\t#{$2}" end end diff --git a/tool/gen-github-release.rb b/tool/gen-github-release.rb new file mode 100755 index 0000000000..cdb66080d9 --- /dev/null +++ b/tool/gen-github-release.rb @@ -0,0 +1,66 @@ +#!/usr/bin/env ruby + +if ARGV.size < 2 + puts "Usage: #{$0} <from version tag> <to version tag> [--no-dry-run]" + puts " : if --no-dry-run is specified, it will create a release on GitHub" + exit 1 +end + +require "bundler/inline" + +gemfile do + source "https://rubygems.org" + gem "octokit" + gem "faraday-retry" + gem "nokogiri" +end + +require "open-uri" + +Octokit.configure do |c| + c.access_token = ENV['GITHUB_TOKEN'] + c.auto_paginate = true + c.per_page = 100 +end + +client = Octokit::Client.new + +note = "## What's Changed\n\n" + +notes = [] + +diff = client.compare("ruby/ruby", ARGV[0], ARGV[1]) +diff[:commits].each do |c| + if c[:commit][:message] =~ /\[(Backport|Feature|Bug) #(\d*)\]/ + url = "https://bugs.ruby-lang.org/issues/#{$2}" + title = Nokogiri::HTML(URI.open(url)).title + title.gsub!(/ - Ruby master - Ruby Issue Tracking System/, "") + elsif c[:commit][:message] =~ /\(#(\d*)\)/ + url = "https://github.com/ruby/ruby/pull/#{$1}" + title = Nokogiri::HTML(URI.open(url)).title + title.gsub!(/ · ruby\/ruby · GitHub/, "") + else + next + end + notes << "* [#{title}](#{url})" +rescue OpenURI::HTTPError + puts "Error: #{url}" +end + +notes.uniq! + +note << notes.join("\n") + +note << "\n\n" +note << "Note: This list is automatically generated by tool/gen-github-release.rb. Because of this, some commits may be missing.\n\n" +note << "## Full Changelog\n\n" +note << "https://github.com/ruby/ruby/compare/#{ARGV[0]}...#{ARGV[1]}\n\n" + +if ARGV[2] == "--no-dry-run" + name = ARGV[1].gsub(/^v/, "").gsub(/_/, ".") + prerelease = ARGV[1].match?(/rc|preview/) ? true : false + client.create_release("ruby/ruby", ARGV[1], name: name, body: note, make_latest: "false", prerelease: prerelease) + puts "Created a release: https://github.com/ruby/ruby/releases/tag/#{ARGV[1]}" +else + puts note +end diff --git a/tool/gen-mailmap.rb b/tool/gen-mailmap.rb index 27b7abf8de..0cdedf1e7b 100755 --- a/tool/gen-mailmap.rb +++ b/tool/gen-mailmap.rb @@ -3,7 +3,7 @@ require "open-uri" require "yaml" -EMAIL_YML_URL = "https://cdn.jsdelivr.net/gh/ruby/ruby-commit-hook/config/email.yml" +EMAIL_YML_URL = "https://cdn.jsdelivr.net/gh/ruby/git.ruby-lang.org/config/email.yml" email_yml = URI(EMAIL_YML_URL).read.sub(/\A(?:#.*\n)+/, "").gsub(/^# +(.+)$/) { $1 + ": []" } @@ -13,7 +13,7 @@ YAML.load(DATA.read).each do |name, mails| email[name] |= mails end -open(File.join(__dir__, "../.mailmap"), "w") do |f| +File.open(File.join(__dir__, "../.mailmap"), "w") do |f| email.each do |name, mails| canonical = "#{ name }@ruby-lang.org" mails.delete(canonical) diff --git a/tool/generic_erb.rb b/tool/generic_erb.rb index 6af995fc13..9326309ff6 100644 --- a/tool/generic_erb.rb +++ b/tool/generic_erb.rb @@ -5,57 +5,31 @@ require 'erb' require 'optparse' -require_relative 'lib/vpath' -require_relative 'lib/colorize' +require_relative 'lib/output' -vpath = VPath.new -timestamp = nil -output = nil -ifchange = nil +out = Output.new source = false -color = nil templates = [] ARGV.options do |o| - o.on('-t', '--timestamp[=PATH]') {|v| timestamp = v || true} o.on('-i', '--input=PATH') {|v| template << v} - o.on('-o', '--output=PATH') {|v| output = v} - o.on('-c', '--[no-]if-change') {|v| ifchange = v} o.on('-x', '--source') {source = true} - o.on('--color') {color = true} - vpath.def_options(o) + out.def_options(o) o.order!(ARGV) templates << (ARGV.shift or abort o.to_s) if templates.empty? end -color = Colorize.new(color) -unchanged = color.pass("unchanged") -updated = color.fail("updated") + +# Used in prelude.c.tmpl and unicode_norm_gen.tmpl +output = out.path +vpath = out.vpath + +# A hack to prevent "unused variable" warnings +output, vpath = output, vpath result = templates.map do |template| - if ERB.instance_method(:initialize).parameters.assoc(:key) # Ruby 2.6+ - erb = ERB.new(File.read(template), trim_mode: '%-') - else - erb = ERB.new(File.read(template), nil, '%-') - end + erb = ERB.new(File.read(template), trim_mode: '%-') erb.filename = template source ? erb.src : proc{erb.result(binding)}.call end result = result.size == 1 ? result[0] : result.join("") -if output - if ifchange and (vpath.open(output, "rb") {|f| f.read} rescue nil) == result - puts "#{output} #{unchanged}" - else - open(output, "wb") {|f| f.print result} - puts "#{output} #{updated}" - end - if timestamp - if timestamp == true - dir, base = File.split(output) - timestamp = File.join(dir, ".time." + base) - end - File.open(timestamp, 'a') {} - File.utime(nil, nil, timestamp) - end -else - print result -end +out.write(result) diff --git a/tool/id2token.rb b/tool/id2token.rb index d12ea9c08e..cf73095842 100755 --- a/tool/id2token.rb +++ b/tool/id2token.rb @@ -5,18 +5,15 @@ BEGIN { require 'optparse' - require_relative 'lib/vpath' - vpath = VPath.new - header = nil opt = OptionParser.new do |o| - vpath.def_options(o) - header = o.order!(ARGV).shift + o.order!(ARGV) end or abort opt.opt_s TOKENS = {} - h = vpath.read(header) rescue abort("#{header} not found in #{vpath.inspect}") - h.scan(/^#define\s+RUBY_TOKEN_(\w+)\s+(\d+)/) do |token, id| + defs = File.join(File.dirname(File.dirname(__FILE__)), "defs/id.def") + ids = eval(File.read(defs), nil, defs) + ids[:token_op].each do |_id, _op, token, id| TOKENS[token] = id end diff --git a/tool/leaked-globals b/tool/leaked-globals index d95f3794e8..4aa2aff996 100755 --- a/tool/leaked-globals +++ b/tool/leaked-globals @@ -1,23 +1,33 @@ #!/usr/bin/ruby require_relative 'lib/colorize' +require 'shellwords' until ARGV.empty? case ARGV[0] - when /\ASYMBOL_PREFIX=(.*)/ + when /\A SYMBOL_PREFIX=(.*)/x SYMBOL_PREFIX = $1 - when /\ANM=(.*)/ # may be multiple words - NM = $1 - when /\APLATFORM=(.+)?/ + when /\A NM=(.*)/x # may be multiple words + NM = $1.shellsplit + when /\A PLATFORM=(.+)?/x platform = $1 + when /\A SOEXT=(.+)?/x + soext = $1 + when /\A SYMBOLS_IN_EMPTYLIB=(.*)/x + SYMBOLS_IN_EMPTYLIB = $1.split(" ") + when /\A EXTSTATIC=(.+)?/x + EXTSTATIC = true else break end ARGV.shift end +SYMBOLS_IN_EMPTYLIB ||= nil +EXTSTATIC ||= false config = ARGV.shift count = 0 col = Colorize.new + config_code = File.read(config) REPLACE = config_code.scan(/\bAC_(?:REPLACE|CHECK)_FUNCS?\((\w+)/).flatten # REPLACE << 'memcmp' if /\bAC_FUNC_MEMCMP\b/ =~ config_code @@ -29,27 +39,66 @@ if platform and !platform.empty? else REPLACE.concat( h .gsub(%r[/\*.*?\*/]m, " ") # delete block comments - .gsub(%r[//.*], ' ') # delete oneline comments + .gsub(%r[//.*], " ") # delete oneline comments .gsub(/^\s*#.*(?:\\\n.*)*/, "") # delete preprocessor directives + .gsub(/(?:\A|;)\K\s*typedef\s.*?;/m, "") .scan(/\b((?!rb_|DEPRECATED|_)\w+)\s*\(.*\);/) .flatten) end end missing = File.dirname(config) + "/missing/" ARGV.reject! do |n| - unless (src = Dir.glob(missing + File.basename(n, ".*") + ".[cS]")).empty? - puts "Ignore #{n} because of #{src.map {|s| File.basename(s)}.join(', ')} under missing" + base = File.basename(n, ".*") + next true if REPLACE.include?(base) + unless (src = Dir.glob(missing + base + ".[cS]")).empty? + puts "Ignore #{col.skip(n)} because of #{src.map {|s| File.basename(s)}.join(', ')} under missing" true end end + +# darwin's ld64 seems to require exception handling personality functions to be +# extern, so we allow the Rust one. +REPLACE.push("rust_eh_personality") if RUBY_PLATFORM.include?("darwin") + print "Checking leaked global symbols..." STDOUT.flush -IO.foreach("|#{NM} -Pgp #{ARGV.join(' ')}") do |line| +if soext + soext = /\.#{soext}(?:$|\.)/ + if EXTSTATIC + ARGV.delete_if {|n| soext =~ n} + elsif ARGV.size == 1 + so = soext =~ ARGV.first + end +end + +Pipe = Struct.new(:command) do + def open(&block) IO.popen(command, &block) end + def each(&block) open {|f| f.each(&block)} end +end + +Pipe.new(NM + ARGV).each do |line| + line.chomp! + next so = nil if line.empty? + if so.nil? and line.chomp!(":") + so = soext =~ line || false + next + end n, t, = line.split next unless /[A-TV-Z]/ =~ t next unless n.sub!(/^#{SYMBOL_PREFIX}/o, "") next if n.include?(".") - next if /\A(?:Init_|InitVM_|RUBY_|ruby_|rb_|[Oo]nig|dln_|mjit_|coroutine_)/ =~ n + next if !so and n.start_with?("___asan_") + next if !so and n.start_with?("__odr_asan_") + case n + when /\A(?:Init_|InitVM_|pm_|[Oo]nig|dln_|coroutine_)/ + next + when /\Aruby_static_id_/ + next unless so + when /\A(?:RUBY_|ruby_|rb_)/ + next unless so and /_(threadptr|ec)_/ =~ n + when *SYMBOLS_IN_EMPTYLIB + next + end next if REPLACE.include?(n) puts col.fail("leaked") if count.zero? count += 1 diff --git a/tool/lib/_tmpdir.rb b/tool/lib/_tmpdir.rb new file mode 100644 index 0000000000..fd429dab37 --- /dev/null +++ b/tool/lib/_tmpdir.rb @@ -0,0 +1,100 @@ +template = "rubytest." + +# This path is only for tests. +# Assume the directory by these environment variables are safe. +base = [ENV["TMPDIR"], ENV["TMP"], "/tmp"].find do |tmp| + next unless tmp and tmp.size <= 50 and File.directory?(tmp) + # On macOS, the default TMPDIR is very long, inspite of UNIX socket + # path length is limited. + # + # Also Rubygems creates its own temporary directory per tests, and + # some tests copy the full path of gemhome there. In that caes, the + # path contains both temporary names twice, and can exceed path name + # limit very easily. + tmp +end +begin + tmpdir = File.join(base, template + Random.new_seed.to_s(36)[-6..-1]) + Dir.mkdir(tmpdir, 0o700) +rescue Errno::EEXIST + retry +end +# warn "tmpdir(#{tmpdir.size}) = #{tmpdir}" + +pid = $$ +END { + if pid == $$ + begin + Dir.rmdir(tmpdir) + rescue Errno::ENOENT + rescue Errno::ENOTEMPTY + require_relative "colorize" + colorize = Colorize.new + ls = Struct.new(:colorize) do + def mode_inspect(m, s) + [ + (m & 0o4 == 0 ? ?- : ?r), + (m & 0o2 == 0 ? ?- : ?w), + (m & 0o1 == 0 ? (s ? s.upcase : ?-) : (s || ?x)), + ] + end + def decorate_path(path, st) + case + when st.directory? + color = "bold;blue" + type = "/" + when st.symlink? + color = "bold;cyan" + # type = "@" + when st.executable? + color = "bold;green" + type = "*" + when path.end_with?(".gem") + color = "green" + end + colorize.decorate(path, color) + (type || "") + end + def list_tree(parent, indent = "", &block) + children = Dir.children(parent).map do |child| + [child, path = File.join(parent, child), File.lstat(path)] + end + nlink_width = children.map {|child, path, st| st.nlink}.max.to_s.size + size_width = children.map {|child, path, st| st.size}.max.to_s.size + + children.each do |child, path, st| + m = st.mode + m = [ + (st.file? ? ?- : st.ftype[0]), + mode_inspect(m >> 6, (?s unless m & 04000 == 0)), + mode_inspect(m >> 3, (?s unless m & 02000 == 0)), + mode_inspect(m, (?t unless m & 01000 == 0)), + ].join("") + warn sprintf("%s* %s %*d %*d %s % s%s", + indent, m, nlink_width, st.nlink, size_width, st.size, + st.mtime.to_s, decorate_path(child, st), + (" -> " + decorate_path(File.readlink(path), File.stat(path)) if + st.symlink?)) + if st.directory? + list_tree(File.join(parent, child), indent + " ", &block) + end + yield path, st if block + end + end + end.new(colorize) + warn colorize.notice("Children under ")+colorize.fail(tmpdir)+":" + Dir.chdir(tmpdir) do + ls.list_tree(".") do |path, st| + if st.directory? + Dir.rmdir(path) + else + File.unlink(path) + end + end + end + require "fileutils" + FileUtils.rm_rf(tmpdir) + end + end +} + +ENV["TMPDIR"] = ENV["SPEC_TEMP_DIR"] = ENV["GEM_TEST_TMPDIR"] = tmpdir diff --git a/tool/lib/bundled_gem.rb b/tool/lib/bundled_gem.rb index 38c331183d..3ba27f6d64 100644 --- a/tool/lib/bundled_gem.rb +++ b/tool/lib/bundled_gem.rb @@ -6,12 +6,40 @@ require 'rubygems/package' # unpack bundled gem files. module BundledGem + DEFAULT_GEMS_DEPENDENCIES = [ + "net-protocol", # net-ftp + "time", # net-ftp + "singleton", # prime + "ipaddr", # rinda + "forwardable" # prime, rinda + ] + module_function def unpack(file, *rest) pkg = Gem::Package.new(file) prepare_test(pkg.spec, *rest) {|dir| pkg.extract_files(dir)} puts "Unpacked #{file}" + rescue Gem::Package::FormatError, Errno::ENOENT + puts "Try with hash version of bundled gems instead of #{file}. We don't use this gem with release version of Ruby." + if file =~ /^gems\/(\w+)-/ + file = Dir.glob("gems/#{$1}-*.gem").first + end + retry + end + + def build(gemspec, version, outdir = ".", validation: true) + outdir = File.expand_path(outdir) + gemdir, gemfile = File.split(gemspec) + Dir.chdir(gemdir) do + spec = Gem::Specification.load(gemfile) + abort "Failed to load #{gemspec}" unless spec + output = File.join(outdir, spec.file_name) + FileUtils.rm_rf(output) + package = Gem::Package.new(output) + package.spec = spec + package.build(validation == false) + end end def copy(path, *rest) @@ -35,6 +63,9 @@ module BundledGem gem_dir = File.join(dir, "gems", target) yield gem_dir spec_dir = spec.extensions.empty? ? "specifications" : File.join("gems", target) + if spec.extensions.empty? + spec.dependencies.reject! {|dep| DEFAULT_GEMS_DEPENDENCIES.include?(dep.name)} + end File.binwrite(File.join(dir, spec_dir, "#{target}.gemspec"), spec.to_ruby) unless spec.extensions.empty? spec.dependencies.clear @@ -52,4 +83,36 @@ module BundledGem end FileUtils.rm_rf(Dir.glob("#{gem_dir}/.git*")) end + + def dummy_gemspec(gemspec) + return if File.exist?(gemspec) + gemdir, gemfile = File.split(gemspec) + Dir.chdir(gemdir) do + spec = Gem::Specification.new do |s| + s.name = gemfile.chomp(".gemspec") + s.version = File.read("lib/#{s.name}.rb")[/VERSION = "(.+?)"/, 1] + s.authors = ["DUMMY"] + s.email = ["dummy@ruby-lang.org"] + s.files = Dir.glob("{lib,ext}/**/*").select {|f| File.file?(f)} + s.licenses = ["Ruby"] + s.description = "DO NOT USE; dummy gemspec only for test" + s.summary = "(dummy gemspec)" + end + File.write(gemfile, spec.to_ruby) + end + end + + def checkout(gemdir, repo, rev, git: $git) + return unless rev or !git or git.empty? + unless File.exist?("#{gemdir}/.git") + puts "Cloning #{repo}" + command = "#{git} clone #{repo} #{gemdir}" + system(command) or raise "failed: #{command}" + end + puts "Update #{File.basename(gemdir)} to #{rev}" + command = "#{git} fetch origin #{rev}" + system(command, chdir: gemdir) or raise "failed: #{command}" + command = "#{git} checkout --detach #{rev}" + system(command, chdir: gemdir) or raise "failed: #{command}" + end end diff --git a/tool/lib/colorize.rb b/tool/lib/colorize.rb index 11b878d318..0904312119 100644 --- a/tool/lib/colorize.rb +++ b/tool/lib/colorize.rb @@ -6,8 +6,8 @@ class Colorize # Colorize.new(color: color, colors_file: colors_file) def initialize(color = nil, opts = ((_, color = color, nil)[0] if Hash === color)) @colors = @reset = nil - @color = (opts[:color] if opts) - if color or (color == nil && STDOUT.tty?) + @color = opts && opts[:color] || color + if color or (color == nil && coloring?) if (%w[smso so].any? {|attr| /\A\e\[.*m\z/ =~ IO.popen("tput #{attr}", "r", :err => IO::NULL, &:read)} rescue nil) @beg = "\e[" colors = (colors = ENV['TEST_COLORS']) ? Hash[colors.scan(/(\w+)=([^:\n]*)/)] : {} @@ -27,21 +27,48 @@ class Colorize end DEFAULTS = { - "pass"=>"32", "fail"=>"31;1", "skip"=>"33;1", + # color names "black"=>"30", "red"=>"31", "green"=>"32", "yellow"=>"33", "blue"=>"34", "magenta"=>"35", "cyan"=>"36", "white"=>"37", "bold"=>"1", "underline"=>"4", "reverse"=>"7", + "bright_black"=>"90", "bright_red"=>"91", "bright_green"=>"92", "bright_yellow"=>"93", + "bright_blue"=>"94", "bright_magenta"=>"95", "bright_cyan"=>"96", "bright_white"=>"97", + + # abstract decorations + "pass"=>"green", "fail"=>"red;bold", "skip"=>"yellow;bold", + "note"=>"bright_yellow", "notice"=>"bright_yellow", "info"=>"bright_magenta", } + def coloring? + STDOUT.tty? && (!(nc = ENV['NO_COLOR']) || nc.empty?) + end + # colorize.decorate(str, name = color_name) def decorate(str, name = @color) - if @colors and color = (@colors[name] || DEFAULTS[name]) + if coloring? and color = resolve_color(name) "#{@beg}#{color}m#{str}#{@reset}" else str end end + def resolve_color(color = @color, seen = {}, colors = nil) + return unless @colors + color.to_s.gsub(/\b[a-z][\w ]+/) do |n| + n.gsub!(/\W+/, "_") + n.downcase! + c = seen[n] and next c + if colors + c = colors[n] + elsif (c = (tbl = @colors)[n] || (tbl = DEFAULTS)[n]) + colors = tbl + else + next n + end + seen[n] = resolve_color(c, seen, colors) + end + end + DEFAULTS.each_key do |name| define_method(name) {|str| decorate(str, name) diff --git a/tool/lib/core_assertions.rb b/tool/lib/core_assertions.rb index 67373139ca..b456a55b34 100644 --- a/tool/lib/core_assertions.rb +++ b/tool/lib/core_assertions.rb @@ -1,6 +1,43 @@ # frozen_string_literal: true module Test + + class << self + ## + # Filter object for backtraces. + + attr_accessor :backtrace_filter + end + + class BacktraceFilter # :nodoc: + def filter bt + return ["No backtrace"] unless bt + + new_bt = [] + pattern = %r[/(?:lib\/test/|core_assertions\.rb:)] + + unless $DEBUG then + bt.each do |line| + break if pattern.match?(line) + new_bt << line + end + + new_bt = bt.reject { |line| pattern.match?(line) } if new_bt.empty? + new_bt = bt.dup if new_bt.empty? + else + new_bt = bt.dup + end + + new_bt + end + end + + self.backtrace_filter = BacktraceFilter.new + + def self.filter_backtrace bt # :nodoc: + backtrace_filter.filter bt + end + module Unit module Assertions def assert_raises(*exp, &b) @@ -37,6 +74,11 @@ module Test module CoreAssertions require_relative 'envutil' require 'pp' + begin + require '-test-/asan' + rescue LoadError + end + nil.pretty_inspect def mu_pp(obj) #:nodoc: @@ -111,8 +153,13 @@ module Test end def assert_no_memory_leak(args, prepare, code, message=nil, limit: 2.0, rss: false, **opt) - # TODO: consider choosing some appropriate limit for MJIT and stop skipping this once it does not randomly fail + # TODO: consider choosing some appropriate limit for RJIT and stop skipping this once it does not randomly fail + pend 'assert_no_memory_leak may consider RJIT memory usage as leak' if defined?(RubyVM::RJIT) && RubyVM::RJIT.enabled? + # For previous versions which implemented MJIT pend 'assert_no_memory_leak may consider MJIT memory usage as leak' if defined?(RubyVM::MJIT) && RubyVM::MJIT.enabled? + # ASAN has the same problem - its shadow memory greatly increases memory usage + # (plus asan has better ways to detect memory leaks than this assertion) + pend 'assert_no_memory_leak may consider ASAN memory usage as leak' if defined?(Test::ASAN) && Test::ASAN.enabled? require_relative 'memory_status' raise Test::Unit::PendedError, "unsupported platform" unless defined?(Memory::Status) @@ -248,7 +295,11 @@ module Test at_exit { out.puts "#{token}<error>", [Marshal.dump($!)].pack('m'), "#{token}</error>", "#{token}assertions=#{self._assertions}" } - Test::Unit::Runner.class_variable_set(:@@stop_auto_run, true) if defined?(Test::Unit::Runner) + if defined?(Test::Unit::Runner) + Test::Unit::Runner.class_variable_set(:@@stop_auto_run, true) + elsif defined?(Test::Unit::AutoRunner) + Test::Unit::AutoRunner.need_auto_run = false + end end def assert_separately(args, file = nil, line = nil, src, ignore_stderr: nil, **opt) @@ -268,7 +319,7 @@ module Test src = <<eom # -*- coding: #{line += __LINE__; src.encoding}; -*- BEGIN { - require "test/unit";include Test::Unit::Assertions;include Test::Unit::CoreAssertions;require #{__FILE__.dump} + require "test/unit";include Test::Unit::Assertions;require #{__FILE__.dump};include Test::Unit::CoreAssertions separated_runner #{token_dump}, #{res_c&.fileno || 'nil'} } #{line -= __LINE__; src} @@ -535,11 +586,11 @@ eom refute_respond_to(obj, meth, msg) end - # pattern_list is an array which contains regexp and :*. + # pattern_list is an array which contains regexp, string and :*. # :* means any sequence. # # pattern_list is anchored. - # Use [:*, regexp, :*] for non-anchored match. + # Use [:*, regexp/string, :*] for non-anchored match. def assert_pattern_list(pattern_list, actual, message=nil) rest = actual anchored = true @@ -548,11 +599,13 @@ eom anchored = false else if anchored - match = /\A#{pattern}/.match(rest) + match = rest.rindex(pattern, 0) else - match = pattern.match(rest) + match = rest.index(pattern) end - unless match + if match + post_match = $~ ? $~.post_match : rest[match+pattern.size..-1] + else msg = message(msg) { expect_msg = "Expected #{mu_pp pattern}\n" if /\n[^\n]/ =~ rest @@ -569,7 +622,7 @@ eom } assert false, msg end - rest = match.post_match + rest = post_match anchored = true end } @@ -695,7 +748,7 @@ eom msg = "exceptions on #{errs.length} threads:\n" + errs.map {|t, err| "#{t.inspect}:\n" + - err.full_message(highlight: false, order: :top) + (err.respond_to?(:full_message) ? err.full_message(highlight: false, order: :top) : err.message) }.join("\n---\n") if message msg = "#{message}\n#{msg}" @@ -730,6 +783,62 @@ eom end alias all_assertions_foreach assert_all_assertions_foreach + %w[ + CLOCK_THREAD_CPUTIME_ID CLOCK_PROCESS_CPUTIME_ID + CLOCK_MONOTONIC + ].find do |c| + if Process.const_defined?(c) + [c.to_sym, Process.const_get(c)].find do |clk| + begin + Process.clock_gettime(clk) + rescue + # Constants may be defined but not implemented, e.g., mingw. + else + PERFORMANCE_CLOCK = clk + end + end + end + end + + # Expect +seq+ to respond to +first+ and +each+ methods, e.g., + # Array, Range, Enumerator::ArithmeticSequence and other + # Enumerable-s, and each elements should be size factors. + # + # :yield: each elements of +seq+. + def assert_linear_performance(seq, rehearsal: nil, pre: ->(n) {n}) + pend "No PERFORMANCE_CLOCK found" unless defined?(PERFORMANCE_CLOCK) + + # Timeout testing generally doesn't work when RJIT compilation happens. + rjit_enabled = defined?(RubyVM::RJIT) && RubyVM::RJIT.enabled? + measure = proc do |arg, message| + st = Process.clock_gettime(PERFORMANCE_CLOCK) + yield(*arg) + t = (Process.clock_gettime(PERFORMANCE_CLOCK) - st) + assert_operator 0, :<=, t, message unless rjit_enabled + t + end + + first = seq.first + *arg = pre.call(first) + times = (0..(rehearsal || (2 * first))).map do + measure[arg, "rehearsal"].nonzero? + end + times.compact! + tmin, tmax = times.minmax + tbase = 10 ** Math.log10(tmax * ([(tmax / tmin), 2].max ** 2)).ceil + info = "(tmin: #{tmin}, tmax: #{tmax}, tbase: #{tbase})" + + seq.each do |i| + next if i == first + t = tbase * i.fdiv(first) + *arg = pre.call(i) + message = "[#{i}]: in #{t}s #{info}" + Timeout.timeout(t, Timeout::Error, message) do + measure[arg, message] + end + end + end + def diff(exp, act) require 'pp' q = PP.new(+"") diff --git a/tool/lib/envutil.rb b/tool/lib/envutil.rb index e21305c9ef..642965047f 100644 --- a/tool/lib/envutil.rb +++ b/tool/lib/envutil.rb @@ -15,23 +15,22 @@ end module EnvUtil def rubybin if ruby = ENV["RUBY"] - return ruby - end - ruby = "ruby" - exeext = RbConfig::CONFIG["EXEEXT"] - rubyexe = (ruby + exeext if exeext and !exeext.empty?) - 3.times do - if File.exist? ruby and File.executable? ruby and !File.directory? ruby - return File.expand_path(ruby) - end - if rubyexe and File.exist? rubyexe and File.executable? rubyexe - return File.expand_path(rubyexe) - end - ruby = File.join("..", ruby) - end - if defined?(RbConfig.ruby) + ruby + elsif defined?(RbConfig.ruby) RbConfig.ruby else + ruby = "ruby" + exeext = RbConfig::CONFIG["EXEEXT"] + rubyexe = (ruby + exeext if exeext and !exeext.empty?) + 3.times do + if File.exist? ruby and File.executable? ruby and !File.directory? ruby + return File.expand_path(ruby) + end + if rubyexe and File.exist? rubyexe and File.executable? rubyexe + return File.expand_path(rubyexe) + end + ruby = File.join("..", ruby) + end "ruby" end end @@ -53,7 +52,14 @@ module EnvUtil @original_internal_encoding = Encoding.default_internal @original_external_encoding = Encoding.default_external @original_verbose = $VERBOSE - @original_warning = defined?(Warning.[]) ? %i[deprecated experimental].to_h {|i| [i, Warning[i]]} : nil + @original_warning = + if defined?(Warning.categories) + Warning.categories.to_h {|i| [i, Warning[i]]} + elsif defined?(Warning.[]) # 2.7+ + %i[deprecated experimental performance].to_h do |i| + [i, begin Warning[i]; rescue ArgumentError; end] + end.compact + end end end @@ -155,7 +161,7 @@ module EnvUtil # remain env %w(ASAN_OPTIONS RUBY_ON_BUG).each{|name| - child_env[name] = ENV[name] if ENV[name] + child_env[name] = ENV[name] if !child_env.key?(name) and ENV.key?(name) } args = [args] if args.kind_of?(String) @@ -246,6 +252,24 @@ module EnvUtil end module_function :under_gc_stress + def under_gc_compact_stress(val = :empty, &block) + raise "compaction doesn't work well on s390x. Omit the test in the caller." if RUBY_PLATFORM =~ /s390x/ # https://github.com/ruby/ruby/pull/5077 + auto_compact = GC.auto_compact + GC.auto_compact = val + under_gc_stress(&block) + ensure + GC.auto_compact = auto_compact + end + module_function :under_gc_compact_stress + + def without_gc + prev_disabled = GC.disable + yield + ensure + GC.enable unless prev_disabled + end + module_function :without_gc + def with_default_external(enc) suppress_warning { Encoding.default_external = enc } yield @@ -297,16 +321,24 @@ module EnvUtil cmd = @ruby_install_name if "ruby-runner#{RbConfig::CONFIG["EXEEXT"]}" == cmd path = DIAGNOSTIC_REPORTS_PATH timeformat = DIAGNOSTIC_REPORTS_TIMEFORMAT - pat = "#{path}/#{cmd}_#{now.strftime(timeformat)}[-_]*.crash" + pat = "#{path}/#{cmd}_#{now.strftime(timeformat)}[-_]*.{crash,ips}" first = true 30.times do first ? (first = false) : sleep(0.1) Dir.glob(pat) do |name| log = File.read(name) rescue next - if /\AProcess:\s+#{cmd} \[#{pid}\]$/ =~ log - File.unlink(name) - File.unlink("#{path}/.#{File.basename(name)}.plist") rescue nil - return log + case name + when /\.crash\z/ + if /\AProcess:\s+#{cmd} \[#{pid}\]$/ =~ log + File.unlink(name) + File.unlink("#{path}/.#{File.basename(name)}.plist") rescue nil + return log + end + when /\.ips\z/ + if /^ *"pid" *: *#{pid},/ =~ log + File.unlink(name) + return log + end end end end diff --git a/tool/lib/iseq_loader_checker.rb b/tool/lib/iseq_loader_checker.rb index 3f07b3a999..73784f8450 100644 --- a/tool/lib/iseq_loader_checker.rb +++ b/tool/lib/iseq_loader_checker.rb @@ -76,6 +76,15 @@ class RubyVM::InstructionSequence # return value i2_bin if CHECK_TO_BINARY end if CHECK_TO_A || CHECK_TO_BINARY + + if opt == "prism" + # If RUBY_ISEQ_DUMP_DEBUG is "prism", we'll set up + # InstructionSequence.load_iseq to intercept loading filepaths to compile + # using prism. + def self.load_iseq(filepath) + RubyVM::InstructionSequence.compile_file_prism(filepath) + end + end end #require_relative 'x'; exit(1) diff --git a/tool/lib/leakchecker.rb b/tool/lib/leakchecker.rb index 26d75b92fa..4cd28b9dd5 100644 --- a/tool/lib/leakchecker.rb +++ b/tool/lib/leakchecker.rb @@ -233,7 +233,13 @@ class LeakChecker old_env = @env_info new_env = find_env return false if old_env == new_env + if defined?(Bundler::EnvironmentPreserver) + bundler_prefix = Bundler::EnvironmentPreserver::BUNDLER_PREFIX + end (old_env.keys | new_env.keys).sort.each {|k| + # Don't report changed environment variables caused by Bundler's backups + next if bundler_prefix and k.start_with?(bundler_prefix) + if old_env.has_key?(k) if new_env.has_key?(k) if old_env[k] != new_env[k] diff --git a/tool/lib/memory_status.rb b/tool/lib/memory_status.rb index 5e9e80a68a..60632523a8 100644 --- a/tool/lib/memory_status.rb +++ b/tool/lib/memory_status.rb @@ -12,7 +12,7 @@ module Memory PROC_FILE = procfile VM_PAT = pat def self.read_status - IO.foreach(PROC_FILE, encoding: Encoding::ASCII_8BIT) do |l| + File.foreach(PROC_FILE, encoding: Encoding::ASCII_8BIT) do |l| yield($1.downcase.intern, $2.to_i * 1024) if VM_PAT =~ l end end diff --git a/tool/lib/output.rb b/tool/lib/output.rb new file mode 100644 index 0000000000..8cb426ae4a --- /dev/null +++ b/tool/lib/output.rb @@ -0,0 +1,70 @@ +require_relative 'vpath' +require_relative 'colorize' + +class Output + attr_reader :path, :vpath + + def initialize(path: nil, timestamp: nil, ifchange: nil, color: nil, + overwrite: false, create_only: false, vpath: VPath.new) + @path = path + @timestamp = timestamp + @ifchange = ifchange + @color = color + @overwrite = overwrite + @create_only = create_only + @vpath = vpath + end + + COLOR_WHEN = { + 'always' => true, 'auto' => nil, 'never' => false, + nil => true, false => false, + } + + def def_options(opt) + opt.separator(" Output common options:") + opt.on('-o', '--output=PATH') {|v| @path = v} + opt.on('-t', '--timestamp[=PATH]') {|v| @timestamp = v || true} + opt.on('-c', '--[no-]if-change') {|v| @ifchange = v} + opt.on('--[no-]color=[WHEN]', COLOR_WHEN.keys) {|v| @color = COLOR_WHEN[v]} + opt.on('--[no-]create-only') {|v| @create_only = v} + opt.on('--[no-]overwrite') {|v| @overwrite = v} + @vpath.def_options(opt) + end + + def write(data, overwrite: @overwrite, create_only: @create_only) + unless @path + $stdout.print data + return true + end + color = Colorize.new(@color) + unchanged = color.pass("unchanged") + updated = color.fail("updated") + outpath = nil + + if (@ifchange or overwrite or create_only) and (@vpath.open(@path, "rb") {|f| + outpath = f.path + if @ifchange or create_only + original = f.read + (@ifchange and original == data) or (create_only and !original.empty?) + end + } rescue false) + puts "#{outpath} #{unchanged}" + written = false + else + unless overwrite and outpath and (File.binwrite(outpath, data) rescue nil) + File.binwrite(outpath = @path, data) + end + puts "#{outpath} #{updated}" + written = true + end + if timestamp = @timestamp + if timestamp == true + dir, base = File.split(@path) + timestamp = File.join(dir, ".time." + base) + end + File.binwrite(timestamp, '') + File.utime(nil, nil, timestamp) + end + written + end +end diff --git a/tool/lib/path.rb b/tool/lib/path.rb new file mode 100644 index 0000000000..5582b2851e --- /dev/null +++ b/tool/lib/path.rb @@ -0,0 +1,101 @@ +module Path + module_function + + def clean(path) + path = "#{path}/".gsub(/(\A|\/)(?:\.\/)+/, '\1').tr_s('/', '/') + nil while path.sub!(/[^\/]+\/\.\.\//, '') + path + end + + def relative(path, base) + path = clean(path) + base = clean(base) + path, base = [path, base].map{|s|s.split("/")} + until path.empty? or base.empty? or path[0] != base[0] + path.shift + base.shift + end + path, base = [path, base].map{|s|s.join("/")} + if base.empty? + path + elsif base.start_with?("../") or File.absolute_path?(base) + File.expand_path(path) + else + base.gsub!(/[^\/]+/, '..') + File.join(base, path) + end + end + + def clean_link(src, dest) + begin + link = File.readlink(dest) + rescue + else + return if link == src + File.unlink(dest) + end + yield src, dest + end + + # Extensions to FileUtils + + module Mswin + def ln_safe(src, dest, *opt) + cmd = ["mklink", dest.tr("/", "\\"), src.tr("/", "\\")] + cmd[1, 0] = opt + return if system("cmd", "/c", *cmd) + # TODO: use RUNAS or something + puts cmd.join(" ") + end + + def ln_dir_safe(src, dest) + ln_safe(src, dest, "/d") + end + end + + module HardlinkExcutable + def ln_exe(src, dest) + ln(src, dest, force: true) + end + end + + def ln_safe(src, dest) + ln_sf(src, dest) + rescue Errno::ENOENT + # Windows disallows to create broken symboic links, probably because + # it is a kind of reparse points. + raise if File.exist?(src) + end + + alias ln_dir_safe ln_safe + alias ln_exe ln_safe + + def ln_relative(src, dest, executable = false) + return if File.identical?(src, dest) + parent = File.dirname(dest) + File.directory?(parent) or mkdir_p(parent) + if executable + return (ln_exe(src, dest) if File.exist?(src)) + end + clean_link(relative(src, parent), dest) {|s, d| ln_safe(s, d)} + end + + def ln_dir_relative(src, dest) + return if File.identical?(src, dest) + parent = File.dirname(dest) + File.directory?(parent) or mkdir_p(parent) + clean_link(relative(src, parent), dest) {|s, d| ln_dir_safe(s, d)} + end + + case (CROSS_COMPILING || RUBY_PLATFORM) + when /linux|darwin|solaris/ + prepend HardlinkExcutable + extend HardlinkExcutable + when /mingw|mswin/ + unless File.respond_to?(:symlink) + prepend Mswin + extend Mswin + end + else + end +end diff --git a/tool/lib/test/unit.rb b/tool/lib/test/unit.rb index 8cb6d8f651..d758b5fb02 100644 --- a/tool/lib/test/unit.rb +++ b/tool/lib/test/unit.rb @@ -1,5 +1,20 @@ # frozen_string_literal: true +# Enable deprecation warnings for test-all, so deprecated methods/constants/functions are dealt with early. +Warning[:deprecated] = true + +if ENV['BACKTRACE_FOR_DEPRECATION_WARNINGS'] + Warning.extend Module.new { + def warn(message, category: nil, **kwargs) + if category == :deprecated and $stderr.respond_to?(:puts) + $stderr.puts nil, message, caller, nil + else + super + end + end + } +end + require_relative '../envutil' require_relative '../colorize' require_relative '../leakchecker' @@ -9,42 +24,6 @@ require 'optparse' # See Test::Unit module Test - class << self - ## - # Filter object for backtraces. - - attr_accessor :backtrace_filter - end - - class BacktraceFilter # :nodoc: - def filter bt - return ["No backtrace"] unless bt - - new_bt = [] - pattern = %r[/(?:lib\/test/|core_assertions\.rb:)] - - unless $DEBUG then - bt.each do |line| - break if pattern.match?(line) - new_bt << line - end - - new_bt = bt.reject { |line| pattern.match?(line) } if new_bt.empty? - new_bt = bt.dup if new_bt.empty? - else - new_bt = bt.dup - end - - new_bt - end - end - - self.backtrace_filter = BacktraceFilter.new - - def self.filter_backtrace bt # :nodoc: - backtrace_filter.filter bt - end - ## # Test::Unit is an implementation of the xUnit testing framework for Ruby. module Unit @@ -74,17 +53,7 @@ module Test end end - module MJITFirst - def group(list) - # MJIT first - mjit, others = list.partition {|e| /test_mjit/ =~ e} - mjit + others - end - end - class Alpha < NoSort - include MJITFirst - def sort_by_name(list) list.sort_by(&:name) end @@ -97,8 +66,6 @@ module Test # shuffle test suites based on CRC32 of their names Shuffle = Struct.new(:seed, :salt) do - include MJITFirst - def initialize(seed) self.class::CRC_TBL ||= (0..255).map {|i| (0..7).inject(i) {|c,| (c & 1 == 1) ? (0xEDB88320 ^ (c >> 1)) : (c >> 1) } @@ -116,6 +83,10 @@ module Test list.sort_by {|e| randomize_key(e)} end + def group(list) + list + end + private def crc32(str, crc32 = 0xffffffff) @@ -271,10 +242,16 @@ module Test @jobserver = nil makeflags = ENV.delete("MAKEFLAGS") if !options[:parallel] and - /(?:\A|\s)--jobserver-(?:auth|fds)=(\d+),(\d+)/ =~ makeflags + /(?:\A|\s)--jobserver-(?:auth|fds)=(?:(\d+),(\d+)|fifo:((?:\\.|\S)+))/ =~ makeflags begin - r = IO.for_fd($1.to_i(10), "rb", autoclose: false) - w = IO.for_fd($2.to_i(10), "wb", autoclose: false) + if fifo = $3 + fifo.gsub!(/\\(?=.)/, '') + r = File.open(fifo, IO::RDONLY|IO::NONBLOCK|IO::BINARY) + w = File.open(fifo, IO::WRONLY|IO::NONBLOCK|IO::BINARY) + else + r = IO.for_fd($1.to_i(10), "rb", autoclose: false) + w = IO.for_fd($2.to_i(10), "wb", autoclose: false) + end rescue r.close if r nil @@ -282,7 +259,7 @@ module Test r.close_on_exec = true w.close_on_exec = true @jobserver = [r, w] - options[:parallel] ||= 1 + options[:parallel] ||= 256 # number of tokens to acquire first end end @worker_timeout = EnvUtil.apply_timeout_scale(options[:worker_timeout] || 180) @@ -325,7 +302,8 @@ module Test options[:retry] = false end - opts.on '--ruby VAL', "Path to ruby which is used at -j option" do |a| + opts.on '--ruby VAL', "Path to ruby which is used at -j option", + "Also used as EnvUtil.rubybin by some assertion methods" do |a| options[:ruby] = a.split(/ /).reject(&:empty?) end @@ -675,10 +653,18 @@ module Test @ios = [] # Array of worker IOs @job_tokens = String.new(encoding: Encoding::ASCII_8BIT) if @jobserver begin - [@tasks.size, @options[:parallel]].min.times {launch_worker} - while true - timeout = [(@workers.filter_map {|w| w.response_at}.min&.-(Time.now) || 0) + @worker_timeout, 1].max + newjobs = [@tasks.size, @options[:parallel]].min - @workers.size + if newjobs > 0 + if @jobserver + t = @jobserver[0].read_nonblock(newjobs, exception: false) + @job_tokens << t if String === t + newjobs = @job_tokens.size + 1 - @workers.size + end + newjobs.times {launch_worker} + end + + timeout = [(@workers.filter_map {|w| w.response_at}.min&.-(Time.now) || 0), 0].max + @worker_timeout if !(_io = IO.select(@ios, nil, nil, timeout)) timeout = Time.now - @worker_timeout @@ -692,15 +678,9 @@ module Test } break end - break if @tasks.empty? and @workers.empty? - if @jobserver and @job_tokens and !@tasks.empty? and - ((newjobs = [@tasks.size, @options[:parallel]].min) > @workers.size or - !@workers.any? {|x| x.status == :ready}) - t = @jobserver[0].read_nonblock(newjobs, exception: false) - if String === t - @job_tokens << t - t.size.times {launch_worker} - end + if @tasks.empty? + break if @workers.empty? + next # wait for all workers to finish end end rescue Interrupt => ex @@ -708,7 +688,7 @@ module Test return result ensure if file = @options[:timetable_data] - open(file, 'w'){|f| + File.open(file, 'w'){|f| @records.each{|(worker, suite), (st, ed)| f.puts '[' + [worker.dump, suite.dump, st.to_f * 1_000, ed.to_f * 1_000].join(", ") + '],' } @@ -736,7 +716,15 @@ module Test del_status_line or puts error, suites = suites.partition {|r| r[:error]} unless suites.empty? - puts "\n""Retrying..." + puts "\n" + @failed_output.puts "Failed tests:" + suites.each {|r| + r[:report].each {|c, m, e| + @failed_output.puts "#{c}##{m}: #{e&.class}: #{e&.message&.slice(/\A.*/)}" + } + } + @failed_output.puts "\n" + puts "Retrying..." @verbose = options[:verbose] suites.map! {|r| ::Object.const_get(r[:testcase])} _run_suites(suites, type) @@ -772,7 +760,7 @@ module Test unless rep.empty? rep.each do |r| if r[:error] - puke(*r[:error], Timeout::Error) + puke(*r[:error], Timeout::Error.new) next end r[:report]&.each do |f| @@ -792,6 +780,7 @@ module Test warn "" @warnings.uniq! {|w| w[1].message} @warnings.each do |w| + @errors += 1 warn "#{w[0]}: #{w[1].message} (#{w[1].class})" end warn "" @@ -861,7 +850,7 @@ module Test end end - def record(suite, method, assertions, time, error) + def record(suite, method, assertions, time, error, source_location = nil) if @options.values_at(:longest, :most_asserted).any? @tops ||= {} rec = [suite.name, method, assertions, time, error] @@ -959,7 +948,7 @@ module Test end def _prepare_run(suites, type) - options[:job_status] ||= :replace if @tty && !@verbose + options[:job_status] ||= @tty ? :replace : :normal unless @verbose case options[:color] when :always color = true @@ -975,11 +964,14 @@ module Test @output = Output.new(self) unless @options[:testing] filter = options[:filter] type = "#{type}_methods" - total = if filter - suites.inject(0) {|n, suite| n + suite.send(type).grep(filter).size} - else - suites.inject(0) {|n, suite| n + suite.send(type).size} - end + total = suites.sum {|suite| + methods = suite.send(type) + if filter + methods.count {|method| filter === "#{suite}##{method}"} + else + methods.size + end + } @test_count = 0 @total_tests = total.to_s(10) end @@ -1073,7 +1065,7 @@ module Test runner.add_status(" = #$1") when /\A\.+\z/ runner.succeed - when /\A\.*[EFS][EFS.]*\z/ + when /\A\.*[EFST][EFST.]*\z/ runner.failed(s) else $stdout.print(s) @@ -1261,8 +1253,13 @@ module Test puts "#{f}: #{$!}" end } + @load_failed = errors.size.nonzero? result end + + def run(*) + super or @load_failed + end end module RepeatOption # :nodoc: all @@ -1364,6 +1361,182 @@ module Test end end + module LaunchableOption + module Nothing + private + def setup_options(opts, options) + super + opts.define_tail 'Launchable options:' + # This is expected to be called by Test::Unit::Worker. + opts.on_tail '--launchable-test-reports=PATH', String, 'Do nothing' + end + end + + def record(suite, method, assertions, time, error, source_location = nil) + if writer = @options[:launchable_test_reports] + if loc = (source_location || suite.instance_method(method).source_location) + path, lineno = loc + # Launchable JSON schema is defined at + # https://github.com/search?q=repo%3Alaunchableinc%2Fcli+https%3A%2F%2Flaunchableinc.com%2Fschema%2FRecordTestInput&type=code. + e = case error + when nil + status = 'TEST_PASSED' + nil + when Test::Unit::PendedError + status = 'TEST_SKIPPED' + "Skipped:\n#{suite.name}##{method} [#{location error}]:\n#{error.message}\n" + when Test::Unit::AssertionFailedError + status = 'TEST_FAILED' + "Failure:\n#{suite.name}##{method} [#{location error}]:\n#{error.message}\n" + when Timeout::Error + status = 'TEST_FAILED' + "Timeout:\n#{suite.name}##{method}\n" + else + status = 'TEST_FAILED' + bt = Test::filter_backtrace(error.backtrace).join "\n " + "Error:\n#{suite.name}##{method}:\n#{error.class}: #{error.message.b}\n #{bt}\n" + end + repo_path = File.expand_path("#{__dir__}/../../../") + relative_path = path.delete_prefix("#{repo_path}/") + # The test path is a URL-encoded representation. + # https://github.com/launchableinc/cli/blob/v1.81.0/launchable/testpath.py#L18 + test_path = {file: relative_path, class: suite.name, testcase: method}.map{|key, val| + "#{encode_test_path_component(key)}=#{encode_test_path_component(val)}" + }.join('#') + end + end + super + ensure + if writer && test_path && status + # Occasionally, the file writing operation may be paused, especially when `--repeat-count` is specified. + # In such cases, we proceed to execute the operation here. + writer.write_object( + { + testPath: test_path, + status: status, + duration: time, + createdAt: Time.now.to_s, + stderr: e, + stdout: nil, + data: { + lineNumber: lineno + } + } + ) + end + end + + private + def setup_options(opts, options) + super + opts.on_tail '--launchable-test-reports=PATH', String, 'Report test results in Launchable JSON format' do |path| + require 'json' + require 'uri' + options[:launchable_test_reports] = writer = JsonStreamWriter.new(path) + writer.write_array('testCases') + main_pid = Process.pid + at_exit { + # This block is executed when the fork block in a test is completed. + # Therefore, we need to verify whether all tests have been completed. + stack = caller + if stack.size == 0 && main_pid == Process.pid && $!.is_a?(SystemExit) + writer.close + end + } + end + + def encode_test_path_component component + component.to_s.gsub('%', '%25').gsub('=', '%3D').gsub('#', '%23').gsub('&', '%26') + end + end + + ## + # JsonStreamWriter writes a JSON file using a stream. + # By utilizing a stream, we can minimize memory usage, especially for large files. + class JsonStreamWriter + def initialize(path) + @file = File.open(path, "w") + @file.write("{") + @indent_level = 0 + @is_first_key_val = true + @is_first_obj = true + write_new_line + end + + def write_object obj + if @is_first_obj + @is_first_obj = false + else + write_comma + write_new_line + end + @indent_level += 1 + @file.write(to_json_str(obj)) + @indent_level -= 1 + @is_first_key_val = true + # Occasionally, invalid JSON will be created as shown below, especially when `--repeat-count` is specified. + # { + # "testPath": "file=test%2Ftest_timeout.rb&class=TestTimeout&testcase=test_allows_zero_seconds", + # "status": "TEST_PASSED", + # "duration": 2.7e-05, + # "createdAt": "2024-02-09 12:21:07 +0000", + # "stderr": null, + # "stdout": null + # }: null <- here + # }, + # To prevent this, IO#flush is called here. + @file.flush + end + + def write_array(key) + @indent_level += 1 + @file.write(to_json_str(key)) + write_colon + @file.write(" ", "[") + write_new_line + end + + def close + return if @file.closed? + close_array + @indent_level -= 1 + write_new_line + @file.write("}", "\n") + @file.flush + @file.close + end + + private + def to_json_str(obj) + json = JSON.pretty_generate(obj) + json.gsub(/^/, ' ' * (2 * @indent_level)) + end + + def write_indent + @file.write(" " * 2 * @indent_level) + end + + def write_new_line + @file.write("\n") + end + + def write_comma + @file.write(',') + end + + def write_colon + @file.write(":") + end + + def close_array + write_new_line + write_indent + @file.write("]") + @indent_level -= 1 + end + end + end + class Runner # :nodoc: all attr_accessor :report, :failures, :errors, :skips # :nodoc: @@ -1553,7 +1726,7 @@ module Test _start_method(inst) inst._assertions = 0 - print "#{suite}##{method} = " if @verbose + print "#{suite}##{method.inspect.sub(/\A:/, '')} = " if @verbose start_time = Time.now if @verbose result = @@ -1568,7 +1741,7 @@ module Test puts if @verbose $stdout.flush - unless defined?(RubyVM::MJIT) && RubyVM::MJIT.enabled? # compiler process is wrongly considered as leak + unless defined?(RubyVM::RJIT) && RubyVM::RJIT.enabled? # compiler process is wrongly considered as leak leakchecker.check("#{inst.class}\##{inst.__name__}") end @@ -1601,16 +1774,16 @@ module Test # failure or error in teardown, it will be sent again with the # error or failure. - def record suite, method, assertions, time, error + def record suite, method, assertions, time, error, source_location = nil end def location e # :nodoc: last_before_assertion = "" - return '<empty>' unless e.backtrace # SystemStackError can return nil. + return '<empty>' unless e&.backtrace # SystemStackError can return nil. e.backtrace.reverse_each do |s| - break if s =~ /in .(assert|refute|flunk|pass|fail|raise|must|wont)/ + break if s =~ /in .(?:Test::Unit::(?:Core)?Assertions#)?(assert|refute|flunk|pass|fail|raise|must|wont)/ last_before_assertion = s end last_before_assertion.sub(/:in .*$/, '') @@ -1659,7 +1832,7 @@ module Test break unless report.empty? end - return failures + errors if self.test_count > 0 # or return nil... + return (failures + errors).nonzero? # or return nil... rescue Interrupt abort 'Interrupted' end @@ -1692,6 +1865,7 @@ module Test prepend Test::Unit::ExcludesOption prepend Test::Unit::TimeoutOption prepend Test::Unit::RunCount + prepend Test::Unit::LaunchableOption::Nothing ## # Begins the full test run. Delegates to +runner+'s #_run method. @@ -1727,6 +1901,9 @@ module Test when Test::Unit::AssertionFailedError then @failures += 1 "Failure:\n#{klass}##{meth} [#{location e}]:\n#{e.message}\n" + when Timeout::Error + @errors += 1 + "Timeout:\n#{klass}##{meth}\n" else @errors += 1 bt = Test::filter_backtrace(e.backtrace).join "\n " @@ -1745,6 +1922,7 @@ module Test class AutoRunner # :nodoc: all class Runner < Test::Unit::Runner include Test::Unit::RequireFiles + include Test::Unit::LaunchableOption end attr_accessor :to_run, :options diff --git a/tool/lib/test/unit/parallel.rb b/tool/lib/test/unit/parallel.rb index b3a8957f26..ac297d4a0e 100644 --- a/tool/lib/test/unit/parallel.rb +++ b/tool/lib/test/unit/parallel.rb @@ -1,12 +1,6 @@ # frozen_string_literal: true -$LOAD_PATH.unshift "#{__dir__}/../.." -require_relative '../../test/unit' -require_relative '../../profile_test_all' if ENV.key?('RUBY_TEST_ALL_PROFILE') -require_relative '../../tracepointchecker' -require_relative '../../zombie_hunter' -require_relative '../../iseq_loader_checker' -require_relative '../../gc_checker' +require_relative "../../../test/init" module Test module Unit @@ -186,7 +180,7 @@ module Test else error = ProxyError.new(error) end - _report "record", Marshal.dump([suite.name, method, assertions, time, error]) + _report "record", Marshal.dump([suite.name, method, assertions, time, error, suite.instance_method(method).source_location]) super end end @@ -208,5 +202,9 @@ if $0 == __FILE__ end end require 'rubygems' + begin + require 'rake' + rescue LoadError + end Test::Unit::Worker.new.run(ARGV) end diff --git a/tool/lib/test/unit/testcase.rb b/tool/lib/test/unit/testcase.rb index 44d9ba7fdb..51ffff37eb 100644 --- a/tool/lib/test/unit/testcase.rb +++ b/tool/lib/test/unit/testcase.rb @@ -137,6 +137,9 @@ module Test attr_reader :__name__ # :nodoc: + # Method name of this test. + alias method_name __name__ + PASSTHROUGH_EXCEPTIONS = [NoMemoryError, SignalException, Interrupt, SystemExit] # :nodoc: @@ -144,8 +147,7 @@ module Test # Runs the tests reporting the status to +runner+ def run runner - @options = runner.options - + @__runner_options__ = runner.options trap "INFO" do runner.report.each_with_index do |msg, i| warn "\n%3d) %s" % [i + 1, msg] @@ -161,7 +163,7 @@ module Test result = "" begin - @passed = nil + @__passed__ = nil self.before_setup self.setup self.after_setup @@ -169,11 +171,11 @@ module Test result = "." unless io? time = Time.now - start_time runner.record self.class, self.__name__, self._assertions, time, nil - @passed = true + @__passed__ = true rescue *PASSTHROUGH_EXCEPTIONS raise rescue Exception => e - @passed = Test::Unit::PendedError === e + @__passed__ = Test::Unit::PendedError === e time = Time.now - start_time runner.record self.class, self.__name__, self._assertions, time, e result = runner.puke self.class, self.__name__, e @@ -184,7 +186,7 @@ module Test rescue *PASSTHROUGH_EXCEPTIONS raise rescue Exception => e - @passed = false + @__passed__ = false runner.record self.class, self.__name__, self._assertions, time, e result = runner.puke self.class, self.__name__, e end @@ -206,12 +208,12 @@ module Test def initialize name # :nodoc: @__name__ = name @__io__ = nil - @passed = nil - @@current = self # FIX: make thread local + @__passed__ = nil + @@__current__ = self # FIX: make thread local end def self.current # :nodoc: - @@current # FIX: make thread local + @@__current__ # FIX: make thread local end ## @@ -263,7 +265,7 @@ module Test # Returns true if the test passed. def passed? - @passed + @__passed__ end ## diff --git a/tool/lib/vcs.rb b/tool/lib/vcs.rb index 29be8db3b4..3894f9c8e8 100644 --- a/tool/lib/vcs.rb +++ b/tool/lib/vcs.rb @@ -1,6 +1,8 @@ # vcs require 'fileutils' require 'optparse' +require 'pp' +require 'tempfile' # This library is used by several other tools/ scripts to detect the current # VCS in use (e.g. SVN, Git) or to interact with that VCS. @@ -9,6 +11,22 @@ ENV.delete('PWD') class VCS DEBUG_OUT = STDERR.dup + + def self.dump(obj, pre = nil) + out = DEBUG_OUT + @pp ||= PP.new(out) + @pp.guard_inspect_key do + if pre + @pp.group(pre.size, pre) { + obj.pretty_print(@pp) + } + else + obj.pretty_print(@pp) + end + @pp.flush + out << "\n" + end + end end unless File.respond_to? :realpath @@ -19,14 +37,14 @@ unless File.respond_to? :realpath end def IO.pread(*args) - VCS::DEBUG_OUT.puts(args.inspect) if $DEBUG + VCS.dump(args, "args: ") if $DEBUG popen(*args) {|f|f.read} end module DebugPOpen refine IO.singleton_class do def popen(*args) - VCS::DEBUG_OUT.puts args.inspect if $DEBUG + VCS.dump(args, "args: ") if $DEBUG super end end @@ -34,7 +52,7 @@ end using DebugPOpen module DebugSystem def system(*args) - VCS::DEBUG_OUT.puts args.inspect if $DEBUG + VCS.dump(args, "args: ") if $DEBUG exception = false opts = Hash.try_convert(args[-1]) if RUBY_VERSION >= "2.6" @@ -69,6 +87,9 @@ class VCS begin @@dirs.each do |dir, klass, pred| if pred ? pred[curr, dir] : File.directory?(File.join(curr, dir)) + if klass.const_defined?(:COMMAND) + IO.pread([{'LANG' => 'C', 'LC_ALL' => 'C'}, klass::COMMAND, "--version"]) rescue next + end vcs = klass.new(curr) vcs.define_options(parser) if parser vcs.set_options(options) @@ -92,9 +113,23 @@ class VCS parser.separator(" VCS common options:") parser.define("--[no-]dryrun") {|v| opts[:dryrun] = v} parser.define("--[no-]debug") {|v| opts[:debug] = v} + parser.define("-z", "--zone=OFFSET", /\A[-+]\d\d:\d\d\z/) {|v| opts[:zone] = v} opts end + def release_date(time) + t = time.getlocal(@zone) + [ + t.strftime('#define RUBY_RELEASE_YEAR %Y'), + t.strftime('#define RUBY_RELEASE_MONTH %-m'), + t.strftime('#define RUBY_RELEASE_DAY %-d'), + ] + end + + def self.short_revision(rev) + rev + end + attr_reader :srcdir def initialize(path) @@ -112,14 +147,14 @@ class VCS def set_options(opts) @debug = opts.fetch(:debug) {$DEBUG} @dryrun = opts.fetch(:dryrun) {@debug} + @zone = opts.fetch(:zone) {'+09:00'} end attr_reader :dryrun, :debug alias dryrun? dryrun alias debug? debug - NullDevice = defined?(IO::NULL) ? IO::NULL : - %w[/dev/null NUL NIL: NL:].find {|dev| File.exist?(dev)} + NullDevice = IO::NULL # returns # * the last revision of the current branch @@ -159,6 +194,7 @@ class VCS rescue ArgumentError modified = Time.utc(*$~[1..6]) + $7.to_i * 3600 + $8.to_i * 60 end + modified = modified.getlocal(@zone) end return last, changed, modified, *rest end @@ -190,6 +226,7 @@ class VCS def after_export(dir) FileUtils.rm_rf(Dir.glob("#{dir}/.git*")) + FileUtils.rm_rf(Dir.glob("#{dir}/.mailmap")) end def revision_handler(rev) @@ -204,6 +241,36 @@ class VCS revision_handler(rev).short_revision(rev) end + # make-snapshot generates only release_date whereas file2lastrev generates both release_date and release_datetime + def revision_header(last, release_date, release_datetime = nil, branch = nil, title = nil, limit: 20) + short = short_revision(last) + if /[^\x00-\x7f]/ =~ title and title.respond_to?(:force_encoding) + title = title.dup.force_encoding("US-ASCII") + end + code = [ + "#define RUBY_REVISION #{short.inspect}", + ] + unless short == last + code << "#define RUBY_FULL_REVISION #{last.inspect}" + end + if branch + e = '..' + name = branch.sub(/\A(.{#{limit-e.size}}).{#{e.size+1},}/o) {$1+e} + name = name.dump.sub(/\\#/, '#') + code << "#define RUBY_BRANCH_NAME #{name}" + end + if title + title = title.dump.sub(/\\#/, '#') + code << "#define RUBY_LAST_COMMIT_TITLE #{title}" + end + if release_datetime + t = release_datetime.utc + code << t.strftime('#define RUBY_RELEASE_DATETIME "%FT%TZ"') + end + code += self.release_date(release_date) + code + end + class SVN < self register(".svn") COMMAND = ENV['SVN'] || 'svn' @@ -212,10 +279,6 @@ class VCS "r#{rev}" end - def self.short_revision(rev) - rev - end - def _get_revisions(path, srcdir = nil) if srcdir and self.class.local_path?(path) path = File.join(srcdir, path) @@ -350,7 +413,7 @@ class VCS def commit args = %W"#{COMMAND} commit" if dryrun? - VCS::DEBUG_OUT.puts(args.inspect) + VCS.dump(args, "commit: ") return true end system(*args) @@ -358,8 +421,21 @@ class VCS end class GIT < self - register(".git") {|path, dir| File.exist?(File.join(path, dir))} - COMMAND = ENV["GIT"] || 'git' + register(".git") do |path, dir| + SAFE_DIRECTORIES ||= + begin + command = ENV["GIT"] || 'git' + dirs = IO.popen(%W"#{command} config --global --get-all safe.directory", &:read).split("\n") + rescue + command = nil + dirs = [] + ensure + VCS.dump(dirs, "safe.directory: ") if $DEBUG + COMMAND = command + end + + COMMAND and File.exist?(File.join(path, dir)) + end def cmd_args(cmds, srcdir = nil) (opts = cmds.last).kind_of?(Hash) or cmds << (opts = {}) @@ -367,7 +443,7 @@ class VCS if srcdir opts[:chdir] ||= srcdir end - VCS::DEBUG_OUT.puts cmds.inspect if debug? + VCS.dump(cmds, "cmds: ") if debug? and !$DEBUG cmds end @@ -377,7 +453,7 @@ class VCS def cmd_read_at(srcdir, cmds) result = without_gitconfig { IO.pread(*cmd_args(cmds, srcdir)) } - VCS::DEBUG_OUT.puts result.inspect if debug? + VCS.dump(result, "result: ") if debug? result end @@ -398,7 +474,14 @@ class VCS def _get_revisions(path, srcdir = nil) ref = Branch === path ? path.to_str : 'HEAD' gitcmd = [COMMAND] - last = cmd_read_at(srcdir, [[*gitcmd, 'rev-parse', ref]]).rstrip + last = nil + IO.pipe do |r, w| + last = cmd_read_at(srcdir, [[*gitcmd, 'rev-parse', ref, err: w]]).rstrip + w.close + unless r.eof? + raise "#{COMMAND} rev-parse failed\n#{r.read.gsub(/^(?=\s*\S)/, ' ')}" + end + end log = cmd_read_at(srcdir, [[*gitcmd, 'log', '-n1', '--date=iso', '--pretty=fuller', *path]]) changed = log[/\Acommit (\h+)/, 1] modified = log[/^CommitDate:\s+(.*)/, 1] @@ -460,19 +543,35 @@ class VCS end def without_gitconfig - envs = %w'HOME XDG_CONFIG_HOME GIT_SYSTEM_CONFIG GIT_CONFIG_SYSTEM'.each_with_object({}) do |v, h| + envs = (%w'HOME XDG_CONFIG_HOME' + ENV.keys.grep(/\AGIT_/)).each_with_object({}) do |v, h| h[v] = ENV.delete(v) - ENV[v] = NullDevice if v.start_with?('GIT_') end + ENV['GIT_CONFIG_SYSTEM'] = NullDevice + ENV['GIT_CONFIG_GLOBAL'] = global_config yield ensure ENV.update(envs) end + def global_config + return NullDevice if SAFE_DIRECTORIES.empty? + unless @gitconfig + @gitconfig = Tempfile.new(%w"vcs_ .gitconfig") + @gitconfig.close + ENV['GIT_CONFIG_GLOBAL'] = @gitconfig.path + SAFE_DIRECTORIES.each do |dir| + system(*%W[#{COMMAND} config --global --add safe.directory #{dir}]) + end + VCS.dump(`#{COMMAND} config --global --get-all safe.directory`, "safe.directory: ") if debug? + end + @gitconfig.path + end + def initialize(*) super @srcdir = File.realpath(@srcdir) - VCS::DEBUG_OUT.puts @srcdir.inspect if debug? + @gitconfig = nil + VCS.dump(@srcdir, "srcdir: ") if debug? self end @@ -582,7 +681,10 @@ class VCS def format_changelog(path, arg, base_url = nil) env = {'TZ' => 'JST-9', 'LANG' => 'C', 'LC_ALL' => 'C'} - cmd = %W"#{COMMAND} log --format=fuller --notes=commits --notes=log-fix --topo-order --no-merges" + cmd = %W[#{COMMAND} log + --format=fuller --notes=commits --notes=log-fix --topo-order --no-merges + --fixed-strings --invert-grep --grep=[ci\ skip] --grep=[skip\ ci] + ] date = "--date=iso-local" unless system(env, *cmd, date, "-1", chdir: @srcdir, out: NullDevice, exception: false) date = "--date=iso" @@ -595,20 +697,49 @@ class VCS cmd_pipe(env, cmd, chdir: @srcdir) do |r| while s = r.gets("\ncommit ") h, s = s.split(/^$/, 2) + + next if /^Author: *dependabot\[bot\]/ =~ h + h.gsub!(/^(?:(?:Author|Commit)(?:Date)?|Date): /, ' \&') if s.sub!(/\nNotes \(log-fix\):\n((?: +.*\n)+)/, '') fix = $1 s = s.lines fix.each_line do |x| + next unless x.sub!(/^(\s+)(?:(\d+)|\$(?:-\d+)?)/, '') + b = ($2&.to_i || (s.size - 1 + $3.to_i)) + sp = $1 + if x.sub!(/^,(?:(\d+)|\$(?:-\d+)?)/, '') + range = b..($1&.to_i || (s.size - 1 + $2.to_i)) + else + range = b..b + end case x - when %r[^ +(\d+)s([#{LOG_FIX_REGEXP_SEPARATORS}])(.+)\2(.*)\2]o - n = $1.to_i - wrong = $3 - correct = $4 - begin - s[n][wrong] = correct - rescue IndexError - message = ["format_changelog failed to replace #{wrong.dump} with #{correct.dump} at #$1\n"] + when %r[^s([#{LOG_FIX_REGEXP_SEPARATORS}])(.+)\1(.*)\1([gr]+)?]o + wrong = $2 + correct = $3 + if opt = $4 and opt.include?("r") # regexp + wrong = Regexp.new(wrong) + correct.gsub!(/(?<!\\)(?:\\\\)*\K(?:\\n)+/) {"\n" * ($&.size / 2)} + sub = opt.include?("g") ? :gsub! : :sub! + else + sub = false + end + range.each do |n| + if sub + ss = s[n].sub(/^#{sp}/, "") # un-indent for /^/ + if ss.__send__(sub, wrong, correct) + s[n, 1] = ss.lines.map {|l| "#{sp}#{l}"} + next + end + else + begin + s[n][wrong] = correct + rescue IndexError + else + next + end + end + message = ["format_changelog failed to replace #{wrong.dump} with #{correct.dump} at #{n}\n"] from = [1, n-2].max to = [s.size-1, n+2].min s.each_with_index do |e, i| @@ -618,12 +749,13 @@ class VCS end raise message.join('') end - when %r[^( +)(\d+)i([#{LOG_FIX_REGEXP_SEPARATORS}])(.*)\3]o - s[$2.to_i, 0] = "#{$1}#{$4}\n" - when %r[^ +(\d+)(?:,(\d+))?d] - n = $1.to_i - e = $2 - s[n..(e ? e.to_i : n)] = [] + when %r[^i([#{LOG_FIX_REGEXP_SEPARATORS}])(.*)\1]o + insert = "#{sp}#{$2}\n" + range.reverse_each do |n| + s[n, 0] = insert + end + when %r[^d] + s[range] = [] end end s = s.join('') @@ -631,7 +763,7 @@ class VCS if %r[^ +(https://github\.com/[^/]+/[^/]+/)commit/\h+\n(?=(?: +\n(?i: +Co-authored-by: .*\n)+)?(?:\n|\Z))] =~ s issue = "#{$1}pull/" - s.gsub!(/\b[Ff]ix(?:e[sd])? \K#(?=\d+)/) {issue} + s.gsub!(/\b(?:(?i:fix(?:e[sd])?) +|GH-)\K#(?=\d+\b)|\(\K#(?=\d+\))/) {issue} end s.gsub!(/ +\n/, "\n") @@ -677,13 +809,13 @@ class VCS def commit(opts = {}) args = [COMMAND, "push"] - args << "-n" if dryrun + args << "-n" if dryrun? remote, branch = upstream args << remote branches = %W[refs/notes/commits:refs/notes/commits HEAD:#{branch}] if dryrun? branches.each do |b| - VCS::DEBUG_OUT.puts((args + [b]).inspect) + VCS.dump(args + [b], "commit: ") end return true end @@ -713,7 +845,7 @@ class VCS commits.each_with_index do |l, i| r, a, c = l.split(' ') dcommit = [COMMAND, "svn", "dcommit"] - dcommit.insert(-2, "-n") if dryrun + dcommit.insert(-2, "-n") if dryrun? dcommit << "--add-author-from" unless a == c dcommit << r system(*dcommit) or return false @@ -733,4 +865,15 @@ class VCS true end end + + class Null < self + def get_revisions(path, srcdir = nil) + @modified ||= Time.now - 10 + return nil, nil, @modified + end + + def revision_header(last, release_date, release_datetime = nil, branch = nil, title = nil, limit: 20) + self.release_date(release_date) + end + end end diff --git a/tool/lib/vpath.rb b/tool/lib/vpath.rb index 48ab148405..fa819f3242 100644 --- a/tool/lib/vpath.rb +++ b/tool/lib/vpath.rb @@ -53,10 +53,11 @@ class VPath end def def_options(opt) + opt.separator(" VPath common options:") opt.on("-I", "--srcdir=DIR", "add a directory to search path") {|dir| @additional << dir } - opt.on("-L", "--vpath=PATH LIST", "add directories to search path") {|dirs| + opt.on("-L", "--vpath=PATH-LIST", "add directories to search path") {|dirs| @additional << [dirs] } opt.on("--path-separator=SEP", /\A(?:\W\z|\.(\W).+)/, "separator for vpath") {|sep, vsep| @@ -80,6 +81,10 @@ class VPath @list end + def add(path) + @additional << path + end + def strip(path) prefix = list.map {|dir| Regexp.quote(dir)} path.sub(/\A#{prefix.join('|')}(?:\/|\z)/, '') diff --git a/tool/lib/webrick/httprequest.rb b/tool/lib/webrick/httprequest.rb index d34eac7ecf..258ee37a38 100644 --- a/tool/lib/webrick/httprequest.rb +++ b/tool/lib/webrick/httprequest.rb @@ -402,7 +402,7 @@ module WEBrick # This method provides the metavariables defined by the revision 3 # of "The WWW Common Gateway Interface Version 1.1" # To browse the current document of CGI Version 1.1, see below: - # http://tools.ietf.org/html/rfc3875 + # https://www.rfc-editor.org/rfc/rfc3875 def meta_vars meta = Hash.new diff --git a/tool/lib/webrick/httpserver.rb b/tool/lib/webrick/httpserver.rb index e85d059319..f3f948da3b 100644 --- a/tool/lib/webrick/httpserver.rb +++ b/tool/lib/webrick/httpserver.rb @@ -9,7 +9,6 @@ # # $IPR: httpserver.rb,v 1.63 2002/10/01 17:16:32 gotoyuzo Exp $ -require 'io/wait' require_relative 'server' require_relative 'httputils' require_relative 'httpstatus' diff --git a/tool/lib/webrick/httputils.rb b/tool/lib/webrick/httputils.rb index f1b9ddf9f0..e21284ee7f 100644 --- a/tool/lib/webrick/httputils.rb +++ b/tool/lib/webrick/httputils.rb @@ -112,7 +112,7 @@ module WEBrick def load_mime_types(file) # note: +file+ may be a "| command" for now; some people may # rely on this, but currently we do not use this method by default. - open(file){ |io| + File.open(file){ |io| hash = Hash.new io.each{ |line| next if /^#/ =~ line diff --git a/tool/ln_sr.rb b/tool/ln_sr.rb index 6ab412edde..e1b5b6f76b 100755 --- a/tool/ln_sr.rb +++ b/tool/ln_sr.rb @@ -3,6 +3,7 @@ target_directory = true noop = false force = false +quiet = false until ARGV.empty? case ARGV[0] @@ -12,6 +13,8 @@ until ARGV.empty? force = true when '-T' target_directory = false + when '-q' + quiet = true else break end @@ -93,7 +96,7 @@ unless respond_to?(:ln_sr) while c = comp.shift if c == ".." and clean.last != ".." and !(fu_have_symlink? && File.symlink?(path)) clean.pop - path.chomp!(%r((?<=\A|/)[^/]+/\z), "") + path.sub!(%r((?<=\A|/)[^/]+/\z), "") else clean << c path << c << "/" @@ -114,9 +117,12 @@ unless respond_to?(:ln_sr) end if File.respond_to?(:symlink) + if quiet and File.identical?(src, dest) + exit + end begin ln_sr(src, dest, verbose: true, target_directory: target_directory, force: force, noop: noop) - rescue NotImplementedError, Errno::EPERM + rescue NotImplementedError, Errno::EPERM, Errno::EACCES else exit end diff --git a/tool/lrama/LEGAL.md b/tool/lrama/LEGAL.md new file mode 100644 index 0000000000..a3ef848514 --- /dev/null +++ b/tool/lrama/LEGAL.md @@ -0,0 +1,12 @@ +# LEGAL NOTICE INFORMATION + +All the files in this distribution are covered under the MIT License except some files +mentioned below. + +## GNU General Public License version 3 + +These files are licensed under the GNU General Public License version 3 or later. See these files for more information. + +* template/bison/_yacc.h +* template/bison/yacc.c +* template/bison/yacc.h diff --git a/tool/lrama/MIT b/tool/lrama/MIT new file mode 100644 index 0000000000..b23d5210d5 --- /dev/null +++ b/tool/lrama/MIT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2023 Yuichiro Kaneko + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/tool/lrama/NEWS.md b/tool/lrama/NEWS.md new file mode 100644 index 0000000000..96aaaf94f5 --- /dev/null +++ b/tool/lrama/NEWS.md @@ -0,0 +1,382 @@ +# NEWS for Lrama + +## Lrama 0.6.5 (2024-03-25) + +### Typed Midrule Actions + +User can specify the type of mid rule action by tag (`<bar>`) instead of specifying it with in an action. + +``` +primary: k_case expr_value terms? + { + $<val>$ = p->case_labels; + p->case_labels = Qnil; + } + case_body + k_end + { + ... + } +``` + +can be written as + +``` +primary: k_case expr_value terms? + { + $$ = p->case_labels; + p->case_labels = Qnil; + }<val> + case_body + k_end + { + ... + } +``` + +`%destructor` for midrule action is invoked only when tag is specified by Typed Midrule Actions. + +Difference from Bison's Typed Midrule Actions is that tag is postposed in Lrama however it's preposed in Bison. + +Bison supports this feature from 3.1. + +## Lrama 0.6.4 (2024-03-22) + +### Parameterizing rules (preceded, terminated, delimited) + +Support `preceded`, `terminated` and `delimited` rules. + +``` +program: preceded(opening, X) + +// Expanded to + +program: preceded_opening_X +preceded_opening_X: opening X +``` + +``` +program: terminated(X, closing) + +// Expanded to + +program: terminated_X_closing +terminated_X_closing: X closing +``` + +``` +program: delimited(opening, X, closing) + +// Expanded to + +program: delimited_opening_X_closing +delimited_opening_X_closing: opening X closing +``` + +https://github.com/ruby/lrama/pull/382 + +### Support `%destructor` declaration + +User can set codes for freeing semantic value resources by using `%destructor`. +In general, these resources are freed by actions or after parsing. +However if syntax error happens in parsing, these codes may not be executed. +Codes associated to `%destructor` are executed when semantic value is popped from the stack by an error. + +``` +%token <val1> NUM +%type <val2> expr2 +%type <val3> expr + +%destructor { + printf("destructor for val1: %d\n", $$); +} <val1> // printer for TAG + +%destructor { + printf("destructor for val2: %d\n", $$); +} <val2> + +%destructor { + printf("destructor for expr: %d\n", $$); +} expr // printer for symbol +``` + +Bison supports this feature from 1.75b. + +https://github.com/ruby/lrama/pull/385 + +## Lrama 0.6.3 (2024-02-15) + +### Bring Your Own Stack + +Provide functionalities for Bring Your Own Stack. + +Ruby’s Ripper library requires their own semantic value stack to manage Ruby Objects returned by user defined callback method. Currently Ripper uses semantic value stack (`yyvsa`) which is used by parser to manage Node. This hack introduces some limitation on Ripper. For example, Ripper can not execute semantic analysis depending on Node structure. + +Lrama introduces two features to support another semantic value stack by parser generator users. + +1. Callback entry points + +User can emulate semantic value stack by these callbacks. +Lrama provides these five callbacks. Registered functions are called when each event happen. For example %after-shift function is called when shift happens on original semantic value stack. + +* `%after-shift` function_name +* `%before-reduce` function_name +* `%after-reduce` function_name +* `%after-shift-error-token` function_name +* `%after-pop-stack` function_name + +2. `$:n` variable to access index of each grammar symbols + +User also needs to access semantic value of their stack in grammar action. `$:n` provides the way to access to it. `$:n` is translated to the minus index from the top of the stack. +For example + +``` +primary: k_if expr_value then compstmt if_tail k_end + { + /*% ripper: if!($:2, $:4, $:5) %*/ + /* $:2 = -5, $:4 = -3, $:5 = -2. */ + } +``` + +https://github.com/ruby/lrama/pull/367 + +## Lrama 0.6.2 (2024-01-27) + +### %no-stdlib directive + +If `%no-stdlib` directive is set, Lrama doesn't load Lrama standard library for +parameterizing rules, stdlib.y. + +https://github.com/ruby/lrama/pull/344 + +## Lrama 0.6.1 (2024-01-13) + +### Nested parameterizing rules + +Allow to pass an instantiated rule to other parameterizing rules. + +``` +%rule constant(X) : X + ; + +%rule option(Y) : /* empty */ + | Y + ; + +%% + +program : option(constant(number)) // Nested rule + ; +%% +``` + +Allow to use nested parameterizing rules when define parameterizing rules. + +``` +%rule option(x) : /* empty */ + | X + ; + +%rule double(Y) : Y Y + ; + +%rule double_opt(A) : option(double(A)) // Nested rule + ; + +%% + +program : double_opt(number) + ; + +%% +``` + +https://github.com/ruby/lrama/pull/337 + +## Lrama 0.6.0 (2023-12-25) + +### User defined parameterizing rules + +Allow to define parameterizing rule by `%rule` directive. + +``` +%rule pair(X, Y): X Y { $$ = $1 + $2; } + ; + +%% + +program: stmt + ; + +stmt: pair(ODD, EVEN) <num> + | pair(EVEN, ODD) <num> + ; +``` + +https://github.com/ruby/lrama/pull/285 + +## Lrama 0.5.11 (2023-12-02) + +### Type specification of parameterizing rules + +Allow to specify type of rules by specifying tag, `<i>` in below example. +Tag is post-modification style. + +``` +%union { + int i; +} + +%% + +program : option(number) <i> + | number_alias? <i> + ; +``` + +https://github.com/ruby/lrama/pull/272 + + +## Lrama 0.5.10 (2023-11-18) + +### Parameterizing rules (option, nonempty_list, list) + +Support function call style parameterizing rules for `option`, `nonempty_list` and `list`. + +https://github.com/ruby/lrama/pull/197 + +### Parameterizing rules (separated_list) + +Support `separated_list` and `separated_nonempty_list` parameterizing rules. + +``` +program: separated_list(',', number) + +// Expanded to + +program: separated_list_number +separated_list_number: ε +separated_list_number: separated_nonempty_list_number +separated_nonempty_list_number: number +separated_nonempty_list_number: separated_nonempty_list_number ',' number +``` + +``` +program: separated_nonempty_list(',', number) + +// Expanded to + +program: separated_nonempty_list_number +separated_nonempty_list_number: number +separated_nonempty_list_number: separated_nonempty_list_number ',' number +``` + +https://github.com/ruby/lrama/pull/204 + +## Lrama 0.5.9 (2023-11-05) + +### Parameterizing rules (suffix) + +Parameterizing rules are template of rules. +It's very common pattern to write "list" grammar rule like: + +``` +opt_args: /* none */ + | args + ; + +args: arg + | args arg +``` + +Lrama supports these suffixes: + +* `?`: option +* `+`: nonempty list +* `*`: list + +Idea of Parameterizing rules comes from Menhir LR(1) parser generator (https://gallium.inria.fr/~fpottier/menhir/manual.html#sec32). + +https://github.com/ruby/lrama/pull/181 + +## Lrama 0.5.7 (2023-10-23) + +### Racc parser + +Replace Lrama's parser from hand written parser to LR parser generated by Racc. +Lrama uses `--embedded` option to generate LR parser because Racc is changed from default gem to bundled gem by Ruby 3.3 (https://github.com/ruby/lrama/pull/132). + +https://github.com/ruby/lrama/pull/62 + +## Lrama 0.5.4 (2023-08-17) + +### Runtime configuration for error recovery + +Meke error recovery function configurable on runtime by two new macros. + +* `YYMAXREPAIR`: Expected to return max length of repair operations. `%parse-param` is passed to this function. +* `YYERROR_RECOVERY_ENABLED`: Expected to return bool value to determine error recovery is enabled or not. `%parse-param` is passed to this function. + +https://github.com/ruby/lrama/pull/74 + +## Lrama 0.5.3 (2023-08-05) + +### Error Recovery + +Support token insert base Error Recovery. +`-e` option is needed to generate parser with error recovery functions. + +https://github.com/ruby/lrama/pull/44 + +## Lrama 0.5.2 (2023-06-14) + +### Named References + +Instead of positional references like `$1` or `$$`, +named references allow to access to symbol by name. + +``` +primary: k_class cpath superclass bodystmt k_end + { + $primary = new_class($cpath, $bodystmt, $superclass); + } +``` + +Alias name can be declared. + +``` +expr[result]: expr[ex-left] '+' expr[ex.right] + { + $result = $[ex-left] + $[ex.right]; + } +``` + +Bison supports this feature from 2.5. + +### Add parse params to some macros and functions + +`%parse-param` are added to these macros and functions to remove ytab.sed hack from Ruby. + +* `YY_LOCATION_PRINT` +* `YY_SYMBOL_PRINT` +* `yy_stack_print` +* `YY_STACK_PRINT` +* `YY_REDUCE_PRINT` +* `yysyntax_error` + +https://github.com/ruby/lrama/pull/40 + +See also: https://github.com/ruby/ruby/pull/7807 + +## Lrama 0.5.0 (2023-05-17) + +### stdin mode + +When `-` is given as grammar file name, reads the grammar source from STDIN, and takes the next argument as the input file name. This mode helps pre-process a grammar source. + +https://github.com/ruby/lrama/pull/8 + +## Lrama 0.4.0 (2023-05-13) + +This is the first version migrated to Ruby. +This version generates "parse.c" compatible with Bison 3.8.2. diff --git a/tool/lrama/exe/lrama b/tool/lrama/exe/lrama new file mode 100755 index 0000000000..ba5fb06c82 --- /dev/null +++ b/tool/lrama/exe/lrama @@ -0,0 +1,6 @@ +#!/usr/bin/env ruby + +$LOAD_PATH << File.join(__dir__, "../lib") +require "lrama" + +Lrama::Command.new.run(ARGV.dup) diff --git a/tool/lrama/lib/lrama.rb b/tool/lrama/lib/lrama.rb new file mode 100644 index 0000000000..9e517b0d71 --- /dev/null +++ b/tool/lrama/lib/lrama.rb @@ -0,0 +1,17 @@ +require "lrama/bitmap" +require "lrama/command" +require "lrama/context" +require "lrama/counterexamples" +require "lrama/digraph" +require "lrama/grammar" +require "lrama/lexer" +require "lrama/option_parser" +require "lrama/options" +require "lrama/output" +require "lrama/parser" +require "lrama/report" +require "lrama/state" +require "lrama/states" +require "lrama/states_reporter" +require "lrama/version" +require "lrama/warning" diff --git a/tool/lrama/lib/lrama/bitmap.rb b/tool/lrama/lib/lrama/bitmap.rb new file mode 100644 index 0000000000..8349a23c34 --- /dev/null +++ b/tool/lrama/lib/lrama/bitmap.rb @@ -0,0 +1,29 @@ +module Lrama + module Bitmap + def self.from_array(ary) + bit = 0 + + ary.each do |int| + bit |= (1 << int) + end + + bit + end + + def self.to_array(int) + a = [] + i = 0 + + while int > 0 do + if int & 1 == 1 + a << i + end + + i += 1 + int >>= 1 + end + + a + end + end +end diff --git a/tool/lrama/lib/lrama/command.rb b/tool/lrama/lib/lrama/command.rb new file mode 100644 index 0000000000..94e86c6c94 --- /dev/null +++ b/tool/lrama/lib/lrama/command.rb @@ -0,0 +1,68 @@ +module Lrama + class Command + LRAMA_LIB = File.realpath(File.join(File.dirname(__FILE__))) + STDLIB_FILE_PATH = File.join(LRAMA_LIB, 'grammar', 'stdlib.y') + + def run(argv) + begin + options = OptionParser.new.parse(argv) + rescue => e + message = e.message + message = message.gsub(/.+/, "\e[1m\\&\e[m") if Exception.to_tty? + abort message + end + + Report::Duration.enable if options.trace_opts[:time] + + warning = Lrama::Warning.new + text = options.y.read + options.y.close if options.y != STDIN + begin + grammar = Lrama::Parser.new(text, options.grammar_file, options.debug).parse + unless grammar.no_stdlib + stdlib_grammar = Lrama::Parser.new(File.read(STDLIB_FILE_PATH), STDLIB_FILE_PATH, options.debug).parse + grammar.insert_before_parameterizing_rules(stdlib_grammar.parameterizing_rules) + end + grammar.prepare + grammar.validate! + rescue => e + raise e if options.debug + message = e.message + message = message.gsub(/.+/, "\e[1m\\&\e[m") if Exception.to_tty? + abort message + end + states = Lrama::States.new(grammar, warning, trace_state: (options.trace_opts[:automaton] || options.trace_opts[:closure])) + states.compute + context = Lrama::Context.new(states) + + if options.report_file + reporter = Lrama::StatesReporter.new(states) + File.open(options.report_file, "w+") do |f| + reporter.report(f, **options.report_opts) + end + end + + if options.trace_opts && options.trace_opts[:rules] + puts "Grammar rules:" + puts grammar.rules + end + + File.open(options.outfile, "w+") do |f| + Lrama::Output.new( + out: f, + output_file_path: options.outfile, + template_name: options.skeleton, + grammar_file_path: options.grammar_file, + header_file_path: options.header_file, + context: context, + grammar: grammar, + error_recovery: options.error_recovery, + ).render + end + + if warning.has_error? + exit false + end + end + end +end diff --git a/tool/lrama/lib/lrama/context.rb b/tool/lrama/lib/lrama/context.rb new file mode 100644 index 0000000000..32017e65fc --- /dev/null +++ b/tool/lrama/lib/lrama/context.rb @@ -0,0 +1,497 @@ +require "lrama/report/duration" + +module Lrama + # This is passed to a template + class Context + include Report::Duration + + ErrorActionNumber = -Float::INFINITY + BaseMin = -Float::INFINITY + + # TODO: It might be better to pass `states` to Output directly? + attr_reader :states, :yylast, :yypact_ninf, :yytable_ninf, :yydefact, :yydefgoto + + def initialize(states) + @states = states + @yydefact = nil + @yydefgoto = nil + # Array of array + @_actions = [] + + compute_tables + end + + # enum yytokentype + def yytokentype + @states.terms.reject do |term| + 0 < term.token_id && term.token_id < 128 + end.map do |term| + [term.id.s_value, term.token_id, term.display_name] + end.unshift(["YYEMPTY", -2, nil]) + end + + # enum yysymbol_kind_t + def yysymbol_kind_t + @states.symbols.map do |sym| + [sym.enum_name, sym.number, sym.comment] + end.unshift(["YYSYMBOL_YYEMPTY", -2, nil]) + end + + # State number of final (accepted) state + def yyfinal + @states.states.find do |state| + state.items.find do |item| + item.lhs.accept_symbol? && item.end_of_rule? + end + end.id + end + + # Number of terms + def yyntokens + @states.terms.count + end + + # Number of nterms + def yynnts + @states.nterms.count + end + + # Number of rules + def yynrules + @states.rules.count + end + + # Number of states + def yynstates + @states.states.count + end + + # Last token number + def yymaxutok + @states.terms.map(&:token_id).max + end + + # YYTRANSLATE + # + # yytranslate is a mapping from token id to symbol number + def yytranslate + # 2 is YYSYMBOL_YYUNDEF + a = Array.new(yymaxutok, 2) + + @states.terms.each do |term| + a[term.token_id] = term.number + end + + return a + end + + def yytranslate_inverted + a = Array.new(@states.symbols.count, @states.undef_symbol.token_id) + + @states.terms.each do |term| + a[term.number] = term.token_id + end + + return a + end + + # Mapping from rule number to line number of the rule is defined. + # Dummy rule is appended as the first element whose value is 0 + # because 0 means error in yydefact. + def yyrline + a = [0] + + @states.rules.each do |rule| + a << rule.lineno + end + + return a + end + + # Mapping from symbol number to its name + def yytname + @states.symbols.sort_by(&:number).map do |sym| + sym.display_name + end + end + + def yypact + @base[0...yynstates] + end + + def yypgoto + @base[yynstates..-1] + end + + def yytable + @table + end + + def yycheck + @check + end + + def yystos + @states.states.map do |state| + state.accessing_symbol.number + end + end + + # Mapping from rule number to symbol number of LHS. + # Dummy rule is appended as the first element whose value is 0 + # because 0 means error in yydefact. + def yyr1 + a = [0] + + @states.rules.each do |rule| + a << rule.lhs.number + end + + return a + end + + # Mapping from rule number to length of RHS. + # Dummy rule is appended as the first element whose value is 0 + # because 0 means error in yydefact. + def yyr2 + a = [0] + + @states.rules.each do |rule| + a << rule.rhs.count + end + + return a + end + + private + + # Compute these + # + # See also: "src/tables.c" of Bison. + # + # * yydefact + # * yydefgoto + # * yypact and yypgoto + # * yytable + # * yycheck + # * yypact_ninf + # * yytable_ninf + def compute_tables + report_duration(:compute_yydefact) { compute_yydefact } + report_duration(:compute_yydefgoto) { compute_yydefgoto } + report_duration(:sort_actions) { sort_actions } + # debug_sorted_actions + report_duration(:compute_packed_table) { compute_packed_table } + end + + def vectors_count + @states.states.count + @states.nterms.count + end + + # In compressed table, rule 0 is appended as an error case + # and reduce is represented as minus number. + def rule_id_to_action_number(rule_id) + (rule_id + 1) * -1 + end + + # Symbol number is assigned to term first then nterm. + # This method calculates sequence_number for nterm. + def nterm_number_to_sequence_number(nterm_number) + nterm_number - @states.terms.count + end + + # Vector is states + nterms + def nterm_number_to_vector_number(nterm_number) + @states.states.count + (nterm_number - @states.terms.count) + end + + def compute_yydefact + # Default action (shift/reduce/error) for each state. + # Index is state id, value is `rule id + 1` of a default reduction. + @yydefact = Array.new(@states.states.count, 0) + + @states.states.each do |state| + # Action number means + # + # * number = 0, default action + # * number = -Float::INFINITY, error by %nonassoc + # * number > 0, shift then move to state "number" + # * number < 0, reduce by "-number" rule. Rule "number" is already added by 1. + actions = Array.new(@states.terms.count, 0) + + if state.reduces.map(&:selected_look_ahead).any? {|la| !la.empty? } + # Iterate reduces with reverse order so that first rule is used. + state.reduces.reverse_each do |reduce| + reduce.look_ahead.each do |term| + actions[term.number] = rule_id_to_action_number(reduce.rule.id) + end + end + end + + # Shift is selected when S/R conflict exists. + state.selected_term_transitions.each do |shift, next_state| + actions[shift.next_sym.number] = next_state.id + end + + state.resolved_conflicts.select do |conflict| + conflict.which == :error + end.each do |conflict| + actions[conflict.symbol.number] = ErrorActionNumber + end + + # If default_reduction_rule, replace default_reduction_rule in + # actions with zero. + if state.default_reduction_rule + actions.map! do |e| + if e == rule_id_to_action_number(state.default_reduction_rule.id) + 0 + else + e + end + end + end + + # If no default_reduction_rule, default behavior is an + # error then replace ErrorActionNumber with zero. + if !state.default_reduction_rule + actions.map! do |e| + if e == ErrorActionNumber + 0 + else + e + end + end + end + + s = actions.each_with_index.map do |n, i| + [i, n] + end.reject do |i, n| + # Remove default_reduction_rule entries + n == 0 + end + + if s.count != 0 + # Entry of @_actions is an array of + # + # * State id + # * Array of tuple, [from, to] where from is term number and to is action. + # * The number of "Array of tuple" used by sort_actions + # * "width" used by sort_actions + @_actions << [state.id, s, s.count, s.last[0] - s.first[0] + 1] + end + + @yydefact[state.id] = state.default_reduction_rule ? state.default_reduction_rule.id + 1 : 0 + end + end + + def compute_yydefgoto + # Default GOTO (nterm transition) for each nterm. + # Index is sequence number of nterm, value is state id + # of a default nterm transition destination. + @yydefgoto = Array.new(@states.nterms.count, 0) + # Mapping from nterm to next_states + nterm_to_next_states = {} + + @states.states.each do |state| + state.nterm_transitions.each do |shift, next_state| + key = shift.next_sym + nterm_to_next_states[key] ||= [] + nterm_to_next_states[key] << [state, next_state] # [from_state, to_state] + end + end + + @states.nterms.each do |nterm| + if !(states = nterm_to_next_states[nterm]) + default_goto = 0 + not_default_gotos = [] + else + default_state = states.map(&:last).group_by {|s| s }.max_by {|_, v| v.count }.first + default_goto = default_state.id + not_default_gotos = [] + states.each do |from_state, to_state| + next if to_state.id == default_goto + not_default_gotos << [from_state.id, to_state.id] + end + end + + k = nterm_number_to_sequence_number(nterm.number) + @yydefgoto[k] = default_goto + + if not_default_gotos.count != 0 + v = nterm_number_to_vector_number(nterm.number) + + # Entry of @_actions is an array of + # + # * Nterm number as vector number + # * Array of tuple, [from, to] where from is state number and to is state number. + # * The number of "Array of tuple" used by sort_actions + # * "width" used by sort_actions + @_actions << [v, not_default_gotos, not_default_gotos.count, not_default_gotos.last[0] - not_default_gotos.first[0] + 1] + end + end + end + + def sort_actions + # This is not same with #sort_actions + # + # @sorted_actions = @_actions.sort_by do |_, _, count, width| + # [-width, -count] + # end + + @sorted_actions = [] + + @_actions.each do |action| + if @sorted_actions.empty? + @sorted_actions << action + next + end + + j = @sorted_actions.count - 1 + _state_id, _froms_and_tos, count, width = action + + while (j >= 0) do + case + when @sorted_actions[j][3] < width + j -= 1 + when @sorted_actions[j][3] == width && @sorted_actions[j][2] < count + j -= 1 + else + break + end + end + + @sorted_actions.insert(j + 1, action) + end + end + + def debug_sorted_actions + ary = Array.new + @sorted_actions.each do |state_id, froms_and_tos, count, width| + ary[state_id] = [state_id, froms_and_tos, count, width] + end + + print sprintf("table_print:\n\n") + + print sprintf("order [\n") + vectors_count.times do |i| + print sprintf("%d, ", @sorted_actions[i] ? @sorted_actions[i][0] : 0) + print "\n" if i % 10 == 9 + end + print sprintf("]\n\n") + + print sprintf("width [\n") + vectors_count.times do |i| + print sprintf("%d, ", ary[i] ? ary[i][3] : 0) + print "\n" if i % 10 == 9 + end + print sprintf("]\n\n") + + print sprintf("tally [\n") + vectors_count.times do |i| + print sprintf("%d, ", ary[i] ? ary[i][2] : 0) + print "\n" if i % 10 == 9 + end + print sprintf("]\n\n") + end + + def compute_packed_table + # yypact and yypgoto + @base = Array.new(vectors_count, BaseMin) + # yytable + @table = [] + # yycheck + @check = [] + # Key is froms_and_tos, value is index position + pushed = {} + userd_res = {} + lowzero = 0 + high = 0 + + @sorted_actions.each do |state_id, froms_and_tos, _, _| + if (res = pushed[froms_and_tos]) + @base[state_id] = res + next + end + + res = lowzero - froms_and_tos.first[0] + + while true do + ok = true + + froms_and_tos.each do |from, to| + loc = res + from + + if @table[loc] + # If the cell of table is set, can not use the cell. + ok = false + break + end + end + + if ok && userd_res[res] + ok = false + end + + if ok + break + else + res += 1 + end + end + + loc = 0 + + froms_and_tos.each do |from, to| + loc = res + from + + @table[loc] = to + @check[loc] = from + end + + while (@table[lowzero]) do + lowzero += 1 + end + + high = loc if high < loc + + @base[state_id] = res + pushed[froms_and_tos] = res + userd_res[res] = true + end + + @yylast = high + + # replace_ninf + @yypact_ninf = (@base.reject {|i| i == BaseMin } + [0]).min - 1 + @base.map! do |i| + case i + when BaseMin + @yypact_ninf + else + i + end + end + + @yytable_ninf = (@table.compact.reject {|i| i == ErrorActionNumber } + [0]).min - 1 + @table.map! do |i| + case i + when nil + 0 + when ErrorActionNumber + @yytable_ninf + else + i + end + end + + @check.map! do |i| + case i + when nil + -1 + else + i + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/counterexamples.rb b/tool/lrama/lib/lrama/counterexamples.rb new file mode 100644 index 0000000000..046265da59 --- /dev/null +++ b/tool/lrama/lib/lrama/counterexamples.rb @@ -0,0 +1,286 @@ +require "set" + +require "lrama/counterexamples/derivation" +require "lrama/counterexamples/example" +require "lrama/counterexamples/path" +require "lrama/counterexamples/production_path" +require "lrama/counterexamples/start_path" +require "lrama/counterexamples/state_item" +require "lrama/counterexamples/transition_path" +require "lrama/counterexamples/triple" + +module Lrama + # See: https://www.cs.cornell.edu/andru/papers/cupex/cupex.pdf + # 4. Constructing Nonunifying Counterexamples + class Counterexamples + attr_reader :transitions, :productions + + def initialize(states) + @states = states + setup_transitions + setup_productions + end + + def to_s + "#<Counterexamples>" + end + alias :inspect :to_s + + def compute(conflict_state) + conflict_state.conflicts.flat_map do |conflict| + case conflict.type + when :shift_reduce + shift_reduce_example(conflict_state, conflict) + when :reduce_reduce + reduce_reduce_examples(conflict_state, conflict) + end + end.compact + end + + private + + def setup_transitions + # Hash [StateItem, Symbol] => StateItem + @transitions = {} + # Hash [StateItem, Symbol] => Set(StateItem) + @reverse_transitions = {} + + @states.states.each do |src_state| + trans = {} + + src_state.transitions.each do |shift, next_state| + trans[shift.next_sym] = next_state + end + + src_state.items.each do |src_item| + next if src_item.end_of_rule? + sym = src_item.next_sym + dest_state = trans[sym] + + dest_state.kernels.each do |dest_item| + next unless (src_item.rule == dest_item.rule) && (src_item.position + 1 == dest_item.position) + src_state_item = StateItem.new(src_state, src_item) + dest_state_item = StateItem.new(dest_state, dest_item) + + @transitions[[src_state_item, sym]] = dest_state_item + + key = [dest_state_item, sym] + @reverse_transitions[key] ||= Set.new + @reverse_transitions[key] << src_state_item + end + end + end + end + + def setup_productions + # Hash [StateItem] => Set(Item) + @productions = {} + # Hash [State, Symbol] => Set(Item). Symbol is nterm + @reverse_productions = {} + + @states.states.each do |state| + # LHS => Set(Item) + h = {} + + state.closure.each do |item| + sym = item.lhs + + h[sym] ||= Set.new + h[sym] << item + end + + state.items.each do |item| + next if item.end_of_rule? + next if item.next_sym.term? + + sym = item.next_sym + state_item = StateItem.new(state, item) + key = [state, sym] + + @productions[state_item] = h[sym] + + @reverse_productions[key] ||= Set.new + @reverse_productions[key] << item + end + end + end + + def shift_reduce_example(conflict_state, conflict) + conflict_symbol = conflict.symbols.first + shift_conflict_item = conflict_state.items.find { |item| item.next_sym == conflict_symbol } + path2 = shortest_path(conflict_state, conflict.reduce.item, conflict_symbol) + path1 = find_shift_conflict_shortest_path(path2, conflict_state, shift_conflict_item) + + Example.new(path1, path2, conflict, conflict_symbol, self) + end + + def reduce_reduce_examples(conflict_state, conflict) + conflict_symbol = conflict.symbols.first + path1 = shortest_path(conflict_state, conflict.reduce1.item, conflict_symbol) + path2 = shortest_path(conflict_state, conflict.reduce2.item, conflict_symbol) + + Example.new(path1, path2, conflict, conflict_symbol, self) + end + + def find_shift_conflict_shortest_path(reduce_path, conflict_state, conflict_item) + state_items = find_shift_conflict_shortest_state_items(reduce_path, conflict_state, conflict_item) + build_paths_from_state_items(state_items) + end + + def find_shift_conflict_shortest_state_items(reduce_path, conflict_state, conflict_item) + target_state_item = StateItem.new(conflict_state, conflict_item) + result = [target_state_item] + reversed_reduce_path = reduce_path.to_a.reverse + # Index for state_item + i = 0 + + while (path = reversed_reduce_path[i]) + # Index for prev_state_item + j = i + 1 + _j = j + + while (prev_path = reversed_reduce_path[j]) + if prev_path.production? + j += 1 + else + break + end + end + + state_item = path.to + prev_state_item = prev_path&.to + + if target_state_item == state_item || target_state_item.item.start_item? + result.concat(reversed_reduce_path[_j..-1].map(&:to)) + break + end + + if target_state_item.item.beginning_of_rule? + queue = [] + queue << [target_state_item] + + # Find reverse production + while (sis = queue.shift) + si = sis.last + + # Reach to start state + if si.item.start_item? + sis.shift + result.concat(sis) + target_state_item = si + break + end + + if !si.item.beginning_of_rule? + key = [si, si.item.previous_sym] + @reverse_transitions[key].each do |prev_target_state_item| + next if prev_target_state_item.state != prev_state_item.state + sis.shift + result.concat(sis) + result << prev_target_state_item + target_state_item = prev_target_state_item + i = j + queue.clear + break + end + else + key = [si.state, si.item.lhs] + @reverse_productions[key].each do |item| + state_item = StateItem.new(si.state, item) + queue << (sis + [state_item]) + end + end + end + else + # Find reverse transition + key = [target_state_item, target_state_item.item.previous_sym] + @reverse_transitions[key].each do |prev_target_state_item| + next if prev_target_state_item.state != prev_state_item.state + result << prev_target_state_item + target_state_item = prev_target_state_item + i = j + break + end + end + end + + result.reverse + end + + def build_paths_from_state_items(state_items) + state_items.zip([nil] + state_items).map do |si, prev_si| + case + when prev_si.nil? + StartPath.new(si) + when si.item.beginning_of_rule? + ProductionPath.new(prev_si, si) + else + TransitionPath.new(prev_si, si) + end + end + end + + def shortest_path(conflict_state, conflict_reduce_item, conflict_term) + # queue: is an array of [Triple, [Path]] + queue = [] + visited = {} + start_state = @states.states.first + raise "BUG: Start state should be just one kernel." if start_state.kernels.count != 1 + + start = Triple.new(start_state, start_state.kernels.first, Set.new([@states.eof_symbol])) + + queue << [start, [StartPath.new(start.state_item)]] + + while true + triple, paths = queue.shift + + next if visited[triple] + visited[triple] = true + + # Found + if triple.state == conflict_state && triple.item == conflict_reduce_item && triple.l.include?(conflict_term) + return paths + end + + # transition + triple.state.transitions.each do |shift, next_state| + next unless triple.item.next_sym && triple.item.next_sym == shift.next_sym + next_state.kernels.each do |kernel| + next if kernel.rule != triple.item.rule + t = Triple.new(next_state, kernel, triple.l) + queue << [t, paths + [TransitionPath.new(triple.state_item, t.state_item)]] + end + end + + # production step + triple.state.closure.each do |item| + next unless triple.item.next_sym && triple.item.next_sym == item.lhs + l = follow_l(triple.item, triple.l) + t = Triple.new(triple.state, item, l) + queue << [t, paths + [ProductionPath.new(triple.state_item, t.state_item)]] + end + + break if queue.empty? + end + + return nil + end + + def follow_l(item, current_l) + # 1. follow_L (A -> X1 ... Xn-1 • Xn) = L + # 2. follow_L (A -> X1 ... Xk • Xk+1 Xk+2 ... Xn) = {Xk+2} if Xk+2 is a terminal + # 3. follow_L (A -> X1 ... Xk • Xk+1 Xk+2 ... Xn) = FIRST(Xk+2) if Xk+2 is a nonnullable nonterminal + # 4. follow_L (A -> X1 ... Xk • Xk+1 Xk+2 ... Xn) = FIRST(Xk+2) + follow_L (A -> X1 ... Xk+1 • Xk+2 ... Xn) if Xk+2 is a nullable nonterminal + case + when item.number_of_rest_symbols == 1 + current_l + when item.next_next_sym.term? + Set.new([item.next_next_sym]) + when !item.next_next_sym.nullable + item.next_next_sym.first_set + else + item.next_next_sym.first_set + follow_l(item.new_by_next_position, current_l) + end + end + end +end diff --git a/tool/lrama/lib/lrama/counterexamples/derivation.rb b/tool/lrama/lib/lrama/counterexamples/derivation.rb new file mode 100644 index 0000000000..691e935356 --- /dev/null +++ b/tool/lrama/lib/lrama/counterexamples/derivation.rb @@ -0,0 +1,63 @@ +module Lrama + class Counterexamples + class Derivation + attr_reader :item, :left, :right + attr_writer :right + + def initialize(item, left, right = nil) + @item = item + @left = left + @right = right + end + + def to_s + "#<Derivation(#{item.display_name})>" + end + alias :inspect :to_s + + def render_strings_for_report + result = [] + _render_for_report(self, 0, result, 0) + result.map(&:rstrip) + end + + def render_for_report + render_strings_for_report.join("\n") + end + + private + + def _render_for_report(derivation, offset, strings, index) + item = derivation.item + if strings[index] + strings[index] << " " * (offset - strings[index].length) + else + strings[index] = " " * offset + end + str = strings[index] + str << "#{item.rule_id}: #{item.symbols_before_dot.map(&:display_name).join(" ")} " + + if derivation.left + len = str.length + str << "#{item.next_sym.display_name}" + length = _render_for_report(derivation.left, len, strings, index + 1) + # I want String#ljust! + str << " " * (length - str.length) + else + str << " • #{item.symbols_after_dot.map(&:display_name).join(" ")} " + return str.length + end + + if derivation.right&.left + length = _render_for_report(derivation.right.left, str.length, strings, index + 1) + str << "#{item.symbols_after_dot[1..-1].map(&:display_name).join(" ")} " + str << " " * (length - str.length) if length > str.length + elsif item.next_next_sym + str << "#{item.symbols_after_dot[1..-1].map(&:display_name).join(" ")} " + end + + return str.length + end + end + end +end diff --git a/tool/lrama/lib/lrama/counterexamples/example.rb b/tool/lrama/lib/lrama/counterexamples/example.rb new file mode 100644 index 0000000000..62244a77e0 --- /dev/null +++ b/tool/lrama/lib/lrama/counterexamples/example.rb @@ -0,0 +1,124 @@ +module Lrama + class Counterexamples + class Example + attr_reader :path1, :path2, :conflict, :conflict_symbol + + # path1 is shift conflict when S/R conflict + # path2 is always reduce conflict + def initialize(path1, path2, conflict, conflict_symbol, counterexamples) + @path1 = path1 + @path2 = path2 + @conflict = conflict + @conflict_symbol = conflict_symbol + @counterexamples = counterexamples + end + + def type + @conflict.type + end + + def path1_item + @path1.last.to.item + end + + def path2_item + @path2.last.to.item + end + + def derivations1 + @derivations1 ||= _derivations(path1) + end + + def derivations2 + @derivations2 ||= _derivations(path2) + end + + private + + def _derivations(paths) + derivation = nil + current = :production + lookahead_sym = paths.last.to.item.end_of_rule? ? @conflict_symbol : nil + + paths.reverse_each do |path| + item = path.to.item + + case current + when :production + case path + when StartPath + derivation = Derivation.new(item, derivation) + current = :start + when TransitionPath + derivation = Derivation.new(item, derivation) + current = :transition + when ProductionPath + derivation = Derivation.new(item, derivation) + current = :production + end + + if lookahead_sym && item.next_next_sym && item.next_next_sym.first_set.include?(lookahead_sym) + state_item = @counterexamples.transitions[[path.to, item.next_sym]] + derivation2 = find_derivation_for_symbol(state_item, lookahead_sym) + derivation.right = derivation2 + lookahead_sym = nil + end + + when :transition + case path + when StartPath + derivation = Derivation.new(item, derivation) + current = :start + when TransitionPath + # ignore + current = :transition + when ProductionPath + # ignore + current = :production + end + else + raise "BUG: Unknown #{current}" + end + + break if current == :start + end + + derivation + end + + def find_derivation_for_symbol(state_item, sym) + queue = [] + queue << [state_item] + + while (sis = queue.shift) + si = sis.last + next_sym = si.item.next_sym + + if next_sym == sym + derivation = nil + + sis.reverse_each do |si| + derivation = Derivation.new(si.item, derivation) + end + + return derivation + end + + if next_sym.nterm? && next_sym.first_set.include?(sym) + @counterexamples.productions[si].each do |next_item| + next if next_item.empty_rule? + next_si = StateItem.new(si.state, next_item) + next if sis.include?(next_si) + queue << (sis + [next_si]) + end + + if next_sym.nullable + next_si = @counterexamples.transitions[[si, next_sym]] + queue << (sis + [next_si]) + end + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/counterexamples/path.rb b/tool/lrama/lib/lrama/counterexamples/path.rb new file mode 100644 index 0000000000..edba67a3b6 --- /dev/null +++ b/tool/lrama/lib/lrama/counterexamples/path.rb @@ -0,0 +1,23 @@ +module Lrama + class Counterexamples + class Path + def initialize(from_state_item, to_state_item) + @from_state_item = from_state_item + @to_state_item = to_state_item + end + + def from + @from_state_item + end + + def to + @to_state_item + end + + def to_s + "#<Path(#{type})>" + end + alias :inspect :to_s + end + end +end diff --git a/tool/lrama/lib/lrama/counterexamples/production_path.rb b/tool/lrama/lib/lrama/counterexamples/production_path.rb new file mode 100644 index 0000000000..d7db688518 --- /dev/null +++ b/tool/lrama/lib/lrama/counterexamples/production_path.rb @@ -0,0 +1,17 @@ +module Lrama + class Counterexamples + class ProductionPath < Path + def type + :production + end + + def transition? + false + end + + def production? + true + end + end + end +end diff --git a/tool/lrama/lib/lrama/counterexamples/start_path.rb b/tool/lrama/lib/lrama/counterexamples/start_path.rb new file mode 100644 index 0000000000..4a6821cd0f --- /dev/null +++ b/tool/lrama/lib/lrama/counterexamples/start_path.rb @@ -0,0 +1,21 @@ +module Lrama + class Counterexamples + class StartPath < Path + def initialize(to_state_item) + super nil, to_state_item + end + + def type + :start + end + + def transition? + false + end + + def production? + false + end + end + end +end diff --git a/tool/lrama/lib/lrama/counterexamples/state_item.rb b/tool/lrama/lib/lrama/counterexamples/state_item.rb new file mode 100644 index 0000000000..930ff4a5f8 --- /dev/null +++ b/tool/lrama/lib/lrama/counterexamples/state_item.rb @@ -0,0 +1,6 @@ +module Lrama + class Counterexamples + class StateItem < Struct.new(:state, :item) + end + end +end diff --git a/tool/lrama/lib/lrama/counterexamples/transition_path.rb b/tool/lrama/lib/lrama/counterexamples/transition_path.rb new file mode 100644 index 0000000000..96e611612a --- /dev/null +++ b/tool/lrama/lib/lrama/counterexamples/transition_path.rb @@ -0,0 +1,17 @@ +module Lrama + class Counterexamples + class TransitionPath < Path + def type + :transition + end + + def transition? + true + end + + def production? + false + end + end + end +end diff --git a/tool/lrama/lib/lrama/counterexamples/triple.rb b/tool/lrama/lib/lrama/counterexamples/triple.rb new file mode 100644 index 0000000000..e802beccf4 --- /dev/null +++ b/tool/lrama/lib/lrama/counterexamples/triple.rb @@ -0,0 +1,21 @@ +module Lrama + class Counterexamples + # s: state + # itm: item within s + # l: precise lookahead set + class Triple < Struct.new(:s, :itm, :l) + alias :state :s + alias :item :itm + alias :precise_lookahead_set :l + + def state_item + StateItem.new(state, item) + end + + def inspect + "#{state.inspect}. #{item.display_name}. #{l.map(&:id).map(&:s_value)}" + end + alias :to_s :inspect + end + end +end diff --git a/tool/lrama/lib/lrama/digraph.rb b/tool/lrama/lib/lrama/digraph.rb new file mode 100644 index 0000000000..bbaa86019f --- /dev/null +++ b/tool/lrama/lib/lrama/digraph.rb @@ -0,0 +1,51 @@ +module Lrama + # Algorithm Digraph of https://dl.acm.org/doi/pdf/10.1145/69622.357187 (P. 625) + class Digraph + def initialize(sets, relation, base_function) + # X in the paper + @sets = sets + # R in the paper + @relation = relation + # F' in the paper + @base_function = base_function + # S in the paper + @stack = [] + # N in the paper + @h = Hash.new(0) + # F in the paper + @result = {} + end + + def compute + @sets.each do |x| + next if @h[x] != 0 + traverse(x) + end + + return @result + end + + private + + def traverse(x) + @stack.push(x) + d = @stack.count + @h[x] = d + @result[x] = @base_function[x] # F x = F' x + + @relation[x]&.each do |y| + traverse(y) if @h[y] == 0 + @h[x] = [@h[x], @h[y]].min + @result[x] |= @result[y] # F x = F x + F y + end + + if @h[x] == d + while (z = @stack.pop) do + @h[z] = Float::INFINITY + break if z == x + @result[z] = @result[x] # F (Top of S) = F x + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar.rb b/tool/lrama/lib/lrama/grammar.rb new file mode 100644 index 0000000000..a816b8261b --- /dev/null +++ b/tool/lrama/lib/lrama/grammar.rb @@ -0,0 +1,381 @@ +require "forwardable" +require "lrama/grammar/auxiliary" +require "lrama/grammar/binding" +require "lrama/grammar/code" +require "lrama/grammar/counter" +require "lrama/grammar/destructor" +require "lrama/grammar/error_token" +require "lrama/grammar/parameterizing_rule" +require "lrama/grammar/percent_code" +require "lrama/grammar/precedence" +require "lrama/grammar/printer" +require "lrama/grammar/reference" +require "lrama/grammar/rule" +require "lrama/grammar/rule_builder" +require "lrama/grammar/symbol" +require "lrama/grammar/symbols" +require "lrama/grammar/type" +require "lrama/grammar/union" +require "lrama/lexer" + +module Lrama + # Grammar is the result of parsing an input grammar file + class Grammar + extend Forwardable + + attr_reader :percent_codes, :eof_symbol, :error_symbol, :undef_symbol, :accept_symbol, :aux + attr_accessor :union, :expect, + :printers, :error_tokens, + :lex_param, :parse_param, :initial_action, + :after_shift, :before_reduce, :after_reduce, :after_shift_error_token, :after_pop_stack, + :symbols_resolver, :types, + :rules, :rule_builders, + :sym_to_rules, :no_stdlib + + def_delegators "@symbols_resolver", :symbols, :nterms, :terms, :add_nterm, :add_term, + :find_symbol_by_number!, :find_symbol_by_id!, :token_to_symbol, + :find_symbol_by_s_value!, :fill_symbol_number, :fill_nterm_type, + :fill_printer, :fill_destructor, :fill_error_token, :sort_by_number! + + + def initialize(rule_counter) + @rule_counter = rule_counter + + # Code defined by "%code" + @percent_codes = [] + @printers = [] + @destructors = [] + @error_tokens = [] + @symbols_resolver = Grammar::Symbols::Resolver.new + @types = [] + @rule_builders = [] + @rules = [] + @sym_to_rules = {} + @parameterizing_rule_resolver = ParameterizingRule::Resolver.new + @empty_symbol = nil + @eof_symbol = nil + @error_symbol = nil + @undef_symbol = nil + @accept_symbol = nil + @aux = Auxiliary.new + @no_stdlib = false + + append_special_symbols + end + + def add_percent_code(id:, code:) + @percent_codes << PercentCode.new(id.s_value, code.s_value) + end + + def add_destructor(ident_or_tags:, token_code:, lineno:) + @destructors << Destructor.new(ident_or_tags: ident_or_tags, token_code: token_code, lineno: lineno) + end + + def add_printer(ident_or_tags:, token_code:, lineno:) + @printers << Printer.new(ident_or_tags: ident_or_tags, token_code: token_code, lineno: lineno) + end + + def add_error_token(ident_or_tags:, token_code:, lineno:) + @error_tokens << ErrorToken.new(ident_or_tags: ident_or_tags, token_code: token_code, lineno: lineno) + end + + def add_type(id:, tag:) + @types << Type.new(id: id, tag: tag) + end + + def add_nonassoc(sym, precedence) + set_precedence(sym, Precedence.new(type: :nonassoc, precedence: precedence)) + end + + def add_left(sym, precedence) + set_precedence(sym, Precedence.new(type: :left, precedence: precedence)) + end + + def add_right(sym, precedence) + set_precedence(sym, Precedence.new(type: :right, precedence: precedence)) + end + + def add_precedence(sym, precedence) + set_precedence(sym, Precedence.new(type: :precedence, precedence: precedence)) + end + + def set_precedence(sym, precedence) + raise "" if sym.nterm? + sym.precedence = precedence + end + + def set_union(code, lineno) + @union = Union.new(code: code, lineno: lineno) + end + + def add_rule_builder(builder) + @rule_builders << builder + end + + def add_parameterizing_rule(rule) + @parameterizing_rule_resolver.add_parameterizing_rule(rule) + end + + def parameterizing_rules + @parameterizing_rule_resolver.rules + end + + def insert_before_parameterizing_rules(rules) + @parameterizing_rule_resolver.rules = rules + @parameterizing_rule_resolver.rules + end + + def prologue_first_lineno=(prologue_first_lineno) + @aux.prologue_first_lineno = prologue_first_lineno + end + + def prologue=(prologue) + @aux.prologue = prologue + end + + def epilogue_first_lineno=(epilogue_first_lineno) + @aux.epilogue_first_lineno = epilogue_first_lineno + end + + def epilogue=(epilogue) + @aux.epilogue = epilogue + end + + def prepare + normalize_rules + collect_symbols + set_lhs_and_rhs + fill_default_precedence + fill_symbols + fill_sym_to_rules + compute_nullable + compute_first_set + end + + # TODO: More validation methods + # + # * Validation for no_declared_type_reference + def validate! + @symbols_resolver.validate! + validate_rule_lhs_is_nterm! + end + + def find_rules_by_symbol!(sym) + find_rules_by_symbol(sym) || (raise "Rules for #{sym} not found") + end + + def find_rules_by_symbol(sym) + @sym_to_rules[sym.number] + end + + private + + def compute_nullable + @rules.each do |rule| + case + when rule.empty_rule? + rule.nullable = true + when rule.rhs.any?(&:term) + rule.nullable = false + else + # noop + end + end + + while true do + rs = @rules.select {|e| e.nullable.nil? } + nts = nterms.select {|e| e.nullable.nil? } + rule_count_1 = rs.count + nterm_count_1 = nts.count + + rs.each do |rule| + if rule.rhs.all?(&:nullable) + rule.nullable = true + end + end + + nts.each do |nterm| + find_rules_by_symbol!(nterm).each do |rule| + if rule.nullable + nterm.nullable = true + end + end + end + + rule_count_2 = @rules.count {|e| e.nullable.nil? } + nterm_count_2 = nterms.count {|e| e.nullable.nil? } + + if (rule_count_1 == rule_count_2) && (nterm_count_1 == nterm_count_2) + break + end + end + + rules.select {|r| r.nullable.nil? }.each do |rule| + rule.nullable = false + end + + nterms.select {|e| e.nullable.nil? }.each do |nterm| + nterm.nullable = false + end + end + + def compute_first_set + terms.each do |term| + term.first_set = Set.new([term]).freeze + term.first_set_bitmap = Lrama::Bitmap.from_array([term.number]) + end + + nterms.each do |nterm| + nterm.first_set = Set.new([]).freeze + nterm.first_set_bitmap = Lrama::Bitmap.from_array([]) + end + + while true do + changed = false + + @rules.each do |rule| + rule.rhs.each do |r| + if rule.lhs.first_set_bitmap | r.first_set_bitmap != rule.lhs.first_set_bitmap + changed = true + rule.lhs.first_set_bitmap = rule.lhs.first_set_bitmap | r.first_set_bitmap + end + + break unless r.nullable + end + end + + break unless changed + end + + nterms.each do |nterm| + nterm.first_set = Lrama::Bitmap.to_array(nterm.first_set_bitmap).map do |number| + find_symbol_by_number!(number) + end.to_set + end + end + + def setup_rules + @rule_builders.each do |builder| + builder.setup_rules(@parameterizing_rule_resolver) + end + end + + def append_special_symbols + # YYEMPTY (token_id: -2, number: -2) is added when a template is evaluated + # term = add_term(id: Token.new(Token::Ident, "YYEMPTY"), token_id: -2) + # term.number = -2 + # @empty_symbol = term + + # YYEOF + term = add_term(id: Lrama::Lexer::Token::Ident.new(s_value: "YYEOF"), alias_name: "\"end of file\"", token_id: 0) + term.number = 0 + term.eof_symbol = true + @eof_symbol = term + + # YYerror + term = add_term(id: Lrama::Lexer::Token::Ident.new(s_value: "YYerror"), alias_name: "error") + term.number = 1 + term.error_symbol = true + @error_symbol = term + + # YYUNDEF + term = add_term(id: Lrama::Lexer::Token::Ident.new(s_value: "YYUNDEF"), alias_name: "\"invalid token\"") + term.number = 2 + term.undef_symbol = true + @undef_symbol = term + + # $accept + term = add_nterm(id: Lrama::Lexer::Token::Ident.new(s_value: "$accept")) + term.accept_symbol = true + @accept_symbol = term + end + + def normalize_rules + # Add $accept rule to the top of rules + lineno = @rule_builders.first ? @rule_builders.first.line : 0 + @rules << Rule.new(id: @rule_counter.increment, _lhs: @accept_symbol.id, _rhs: [@rule_builders.first.lhs, @eof_symbol.id], token_code: nil, lineno: lineno) + + setup_rules + + @rule_builders.each do |builder| + builder.rules.each do |rule| + add_nterm(id: rule._lhs, tag: rule.lhs_tag) + @rules << rule + end + end + + @rules.sort_by!(&:id) + end + + # Collect symbols from rules + def collect_symbols + @rules.flat_map(&:_rhs).each do |s| + case s + when Lrama::Lexer::Token::Char + add_term(id: s) + when Lrama::Lexer::Token + # skip + else + raise "Unknown class: #{s}" + end + end + end + + def set_lhs_and_rhs + @rules.each do |rule| + rule.lhs = token_to_symbol(rule._lhs) if rule._lhs + + rule.rhs = rule._rhs.map do |t| + token_to_symbol(t) + end + end + end + + # Rule inherits precedence from the last term in RHS. + # + # https://www.gnu.org/software/bison/manual/html_node/How-Precedence.html + def fill_default_precedence + @rules.each do |rule| + # Explicitly specified precedence has the highest priority + next if rule.precedence_sym + + precedence_sym = nil + rule.rhs.each do |sym| + precedence_sym = sym if sym.term? + end + + rule.precedence_sym = precedence_sym + end + end + + def fill_symbols + fill_symbol_number + fill_nterm_type(@types) + fill_printer(@printers) + fill_destructor(@destructors) + fill_error_token(@error_tokens) + sort_by_number! + end + + def fill_sym_to_rules + @rules.each do |rule| + key = rule.lhs.number + @sym_to_rules[key] ||= [] + @sym_to_rules[key] << rule + end + end + + def validate_rule_lhs_is_nterm! + errors = [] + + rules.each do |rule| + next if rule.lhs.nterm? + + errors << "[BUG] LHS of #{rule} (line: #{rule.lineno}) is term. It should be nterm." + end + + return if errors.empty? + + raise errors.join("\n") + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/auxiliary.rb b/tool/lrama/lib/lrama/grammar/auxiliary.rb new file mode 100644 index 0000000000..933574b0f6 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/auxiliary.rb @@ -0,0 +1,7 @@ +module Lrama + class Grammar + # Grammar file information not used by States but by Output + class Auxiliary < Struct.new(:prologue_first_lineno, :prologue, :epilogue_first_lineno, :epilogue, keyword_init: true) + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/binding.rb b/tool/lrama/lib/lrama/grammar/binding.rb new file mode 100644 index 0000000000..e5ea3fb037 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/binding.rb @@ -0,0 +1,24 @@ +module Lrama + class Grammar + class Binding + attr_reader :actual_args, :count + + def initialize(parameterizing_rule, actual_args) + @parameters = parameterizing_rule.parameters + @actual_args = actual_args + @parameter_to_arg = @parameters.zip(actual_args).map do |param, arg| + [param.s_value, arg] + end.to_h + end + + def resolve_symbol(symbol) + if symbol.is_a?(Lexer::Token::InstantiateRule) + resolved_args = symbol.args.map { |arg| resolve_symbol(arg) } + Lrama::Lexer::Token::InstantiateRule.new(s_value: symbol.s_value, location: symbol.location, args: resolved_args, lhs_tag: symbol.lhs_tag) + else + @parameter_to_arg[symbol.s_value] || symbol + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/code.rb b/tool/lrama/lib/lrama/grammar/code.rb new file mode 100644 index 0000000000..3bad599dae --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/code.rb @@ -0,0 +1,51 @@ +require "forwardable" +require "lrama/grammar/code/destructor_code" +require "lrama/grammar/code/initial_action_code" +require "lrama/grammar/code/no_reference_code" +require "lrama/grammar/code/printer_code" +require "lrama/grammar/code/rule_action" + +module Lrama + class Grammar + class Code + extend Forwardable + + def_delegators "token_code", :s_value, :line, :column, :references + + attr_reader :type, :token_code + + def initialize(type:, token_code:) + @type = type + @token_code = token_code + end + + def ==(other) + self.class == other.class && + self.type == other.type && + self.token_code == other.token_code + end + + # $$, $n, @$, @n are translated to C code + def translated_code + t_code = s_value.dup + + references.reverse_each do |ref| + first_column = ref.first_column + last_column = ref.last_column + + str = reference_to_c(ref) + + t_code[first_column...last_column] = str + end + + return t_code + end + + private + + def reference_to_c(ref) + raise NotImplementedError.new("#reference_to_c is not implemented") + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/code/destructor_code.rb b/tool/lrama/lib/lrama/grammar/code/destructor_code.rb new file mode 100644 index 0000000000..70360eb90f --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/code/destructor_code.rb @@ -0,0 +1,40 @@ +module Lrama + class Grammar + class Code + class DestructorCode < Code + def initialize(type:, token_code:, tag:) + super(type: type, token_code: token_code) + @tag = tag + end + + private + + # * ($$) *yyvaluep + # * (@$) *yylocationp + # * ($:$) error + # * ($1) error + # * (@1) error + # * ($:1) error + def reference_to_c(ref) + case + when ref.type == :dollar && ref.name == "$" # $$ + member = @tag.member + "((*yyvaluep).#{member})" + when ref.type == :at && ref.name == "$" # @$ + "(*yylocationp)" + when ref.type == :index && ref.name == "$" # $:$ + raise "$:#{ref.value} can not be used in #{type}." + when ref.type == :dollar # $n + raise "$#{ref.value} can not be used in #{type}." + when ref.type == :at # @n + raise "@#{ref.value} can not be used in #{type}." + when ref.type == :index # $:n + raise "$:#{ref.value} can not be used in #{type}." + else + raise "Unexpected. #{self}, #{ref}" + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/code/initial_action_code.rb b/tool/lrama/lib/lrama/grammar/code/initial_action_code.rb new file mode 100644 index 0000000000..a694f193cb --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/code/initial_action_code.rb @@ -0,0 +1,34 @@ +module Lrama + class Grammar + class Code + class InitialActionCode < Code + private + + # * ($$) yylval + # * (@$) yylloc + # * ($:$) error + # * ($1) error + # * (@1) error + # * ($:1) error + def reference_to_c(ref) + case + when ref.type == :dollar && ref.name == "$" # $$ + "yylval" + when ref.type == :at && ref.name == "$" # @$ + "yylloc" + when ref.type == :index && ref.name == "$" # $:$ + raise "$:#{ref.value} can not be used in initial_action." + when ref.type == :dollar # $n + raise "$#{ref.value} can not be used in initial_action." + when ref.type == :at # @n + raise "@#{ref.value} can not be used in initial_action." + when ref.type == :index # $:n + raise "$:#{ref.value} can not be used in initial_action." + else + raise "Unexpected. #{self}, #{ref}" + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/code/no_reference_code.rb b/tool/lrama/lib/lrama/grammar/code/no_reference_code.rb new file mode 100644 index 0000000000..6e614cc64a --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/code/no_reference_code.rb @@ -0,0 +1,28 @@ +module Lrama + class Grammar + class Code + class NoReferenceCode < Code + private + + # * ($$) error + # * (@$) error + # * ($:$) error + # * ($1) error + # * (@1) error + # * ($:1) error + def reference_to_c(ref) + case + when ref.type == :dollar # $$, $n + raise "$#{ref.value} can not be used in #{type}." + when ref.type == :at # @$, @n + raise "@#{ref.value} can not be used in #{type}." + when ref.type == :index # $:$, $:n + raise "$:#{ref.value} can not be used in #{type}." + else + raise "Unexpected. #{self}, #{ref}" + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/code/printer_code.rb b/tool/lrama/lib/lrama/grammar/code/printer_code.rb new file mode 100644 index 0000000000..ffccd89395 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/code/printer_code.rb @@ -0,0 +1,40 @@ +module Lrama + class Grammar + class Code + class PrinterCode < Code + def initialize(type:, token_code:, tag:) + super(type: type, token_code: token_code) + @tag = tag + end + + private + + # * ($$) *yyvaluep + # * (@$) *yylocationp + # * ($:$) error + # * ($1) error + # * (@1) error + # * ($:1) error + def reference_to_c(ref) + case + when ref.type == :dollar && ref.name == "$" # $$ + member = @tag.member + "((*yyvaluep).#{member})" + when ref.type == :at && ref.name == "$" # @$ + "(*yylocationp)" + when ref.type == :index && ref.name == "$" # $:$ + raise "$:#{ref.value} can not be used in #{type}." + when ref.type == :dollar # $n + raise "$#{ref.value} can not be used in #{type}." + when ref.type == :at # @n + raise "@#{ref.value} can not be used in #{type}." + when ref.type == :index # $:n + raise "$:#{ref.value} can not be used in #{type}." + else + raise "Unexpected. #{self}, #{ref}" + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/code/rule_action.rb b/tool/lrama/lib/lrama/grammar/code/rule_action.rb new file mode 100644 index 0000000000..d3c0eab64a --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/code/rule_action.rb @@ -0,0 +1,88 @@ +module Lrama + class Grammar + class Code + class RuleAction < Code + def initialize(type:, token_code:, rule:) + super(type: type, token_code: token_code) + @rule = rule + end + + private + + # * ($$) yyval + # * (@$) yyloc + # * ($:$) error + # * ($1) yyvsp[i] + # * (@1) yylsp[i] + # * ($:1) i - 1 + # + # + # Consider a rule like + # + # class: keyword_class { $1 } tSTRING { $2 + $3 } keyword_end { $class = $1 + $keyword_end } + # + # For the semantic action of original rule: + # + # "Rule" class: keyword_class { $1 } tSTRING { $2 + $3 } keyword_end { $class = $1 + $keyword_end } + # "Position in grammar" $1 $2 $3 $4 $5 + # "Index for yyvsp" -4 -3 -2 -1 0 + # "$:n" $:1 $:2 $:3 $:4 $:5 + # "index of $:n" -5 -4 -3 -2 -1 + # + # + # For the first midrule action: + # + # "Rule" class: keyword_class { $1 } tSTRING { $2 + $3 } keyword_end { $class = $1 + $keyword_end } + # "Position in grammar" $1 + # "Index for yyvsp" 0 + # "$:n" $:1 + def reference_to_c(ref) + case + when ref.type == :dollar && ref.name == "$" # $$ + tag = ref.ex_tag || lhs.tag + raise_tag_not_found_error(ref) unless tag + "(yyval.#{tag.member})" + when ref.type == :at && ref.name == "$" # @$ + "(yyloc)" + when ref.type == :index && ref.name == "$" # $:$ + raise "$:$ is not supported" + when ref.type == :dollar # $n + i = -position_in_rhs + ref.index + tag = ref.ex_tag || rhs[ref.index - 1].tag + raise_tag_not_found_error(ref) unless tag + "(yyvsp[#{i}].#{tag.member})" + when ref.type == :at # @n + i = -position_in_rhs + ref.index + "(yylsp[#{i}])" + when ref.type == :index # $:n + i = -position_in_rhs + ref.index + "(#{i} - 1)" + else + raise "Unexpected. #{self}, #{ref}" + end + end + + def position_in_rhs + # If rule is not derived rule, User Code is only action at + # the end of rule RHS. In such case, the action is located on + # `@rule.rhs.count`. + @rule.position_in_original_rule_rhs || @rule.rhs.count + end + + # If this is midrule action, RHS is a RHS of the original rule. + def rhs + (@rule.original_rule || @rule).rhs + end + + # Unlike `rhs`, LHS is always a LHS of the rule. + def lhs + @rule.lhs + end + + def raise_tag_not_found_error(ref) + raise "Tag is not specified for '$#{ref.value}' in '#{@rule}'" + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/counter.rb b/tool/lrama/lib/lrama/grammar/counter.rb new file mode 100644 index 0000000000..c13f4ec3e3 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/counter.rb @@ -0,0 +1,15 @@ +module Lrama + class Grammar + class Counter + def initialize(number) + @number = number + end + + def increment + n = @number + @number += 1 + n + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/destructor.rb b/tool/lrama/lib/lrama/grammar/destructor.rb new file mode 100644 index 0000000000..4b7059e923 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/destructor.rb @@ -0,0 +1,9 @@ +module Lrama + class Grammar + class Destructor < Struct.new(:ident_or_tags, :token_code, :lineno, keyword_init: true) + def translated_code(tag) + Code::DestructorCode.new(type: :destructor, token_code: token_code, tag: tag).translated_code + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/error_token.rb b/tool/lrama/lib/lrama/grammar/error_token.rb new file mode 100644 index 0000000000..8efde7df33 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/error_token.rb @@ -0,0 +1,9 @@ +module Lrama + class Grammar + class ErrorToken < Struct.new(:ident_or_tags, :token_code, :lineno, keyword_init: true) + def translated_code(tag) + Code::PrinterCode.new(type: :error_token, token_code: token_code, tag: tag).translated_code + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/parameterizing_rule.rb b/tool/lrama/lib/lrama/grammar/parameterizing_rule.rb new file mode 100644 index 0000000000..d371805f4b --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/parameterizing_rule.rb @@ -0,0 +1,3 @@ +require_relative 'parameterizing_rule/resolver' +require_relative 'parameterizing_rule/rhs' +require_relative 'parameterizing_rule/rule' diff --git a/tool/lrama/lib/lrama/grammar/parameterizing_rule/resolver.rb b/tool/lrama/lib/lrama/grammar/parameterizing_rule/resolver.rb new file mode 100644 index 0000000000..1923e7819c --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/parameterizing_rule/resolver.rb @@ -0,0 +1,47 @@ +module Lrama + class Grammar + class ParameterizingRule + class Resolver + attr_accessor :rules, :created_lhs_list + + def initialize + @rules = [] + @created_lhs_list = [] + end + + def add_parameterizing_rule(rule) + @rules << rule + end + + def find(token) + select_rules(token).last + end + + def created_lhs(lhs_s_value) + @created_lhs_list.reverse.find { |created_lhs| created_lhs.s_value == lhs_s_value } + end + + private + + def select_rules(token) + rules = select_rules_by_name(token.rule_name) + rules = rules.select { |rule| rule.required_parameters_count == token.args_count } + if rules.empty? + raise "Invalid number of arguments. `#{token.rule_name}`" + else + rules + end + end + + def select_rules_by_name(rule_name) + rules = @rules.select { |rule| rule.name == rule_name } + if rules.empty? + raise "Parameterizing rule does not exist. `#{rule_name}`" + else + rules + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/parameterizing_rule/rhs.rb b/tool/lrama/lib/lrama/grammar/parameterizing_rule/rhs.rb new file mode 100644 index 0000000000..7f50be873c --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/parameterizing_rule/rhs.rb @@ -0,0 +1,15 @@ +module Lrama + class Grammar + class ParameterizingRule + class Rhs + attr_accessor :symbols, :user_code, :precedence_sym + + def initialize + @symbols = [] + @user_code = nil + @precedence_sym = nil + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/parameterizing_rule/rule.rb b/tool/lrama/lib/lrama/grammar/parameterizing_rule/rule.rb new file mode 100644 index 0000000000..9c1d46e4f5 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/parameterizing_rule/rule.rb @@ -0,0 +1,16 @@ +module Lrama + class Grammar + class ParameterizingRule + class Rule + attr_reader :name, :parameters, :rhs_list, :required_parameters_count + + def initialize(name, parameters, rhs_list) + @name = name + @parameters = parameters + @rhs_list = rhs_list + @required_parameters_count = parameters.count + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/percent_code.rb b/tool/lrama/lib/lrama/grammar/percent_code.rb new file mode 100644 index 0000000000..8cbc5aef2c --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/percent_code.rb @@ -0,0 +1,12 @@ +module Lrama + class Grammar + class PercentCode + attr_reader :name, :code + + def initialize(name, code) + @name = name + @code = code + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/precedence.rb b/tool/lrama/lib/lrama/grammar/precedence.rb new file mode 100644 index 0000000000..fed739b3c0 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/precedence.rb @@ -0,0 +1,11 @@ +module Lrama + class Grammar + class Precedence < Struct.new(:type, :precedence, keyword_init: true) + include Comparable + + def <=>(other) + self.precedence <=> other.precedence + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/printer.rb b/tool/lrama/lib/lrama/grammar/printer.rb new file mode 100644 index 0000000000..8984a96e1a --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/printer.rb @@ -0,0 +1,9 @@ +module Lrama + class Grammar + class Printer < Struct.new(:ident_or_tags, :token_code, :lineno, keyword_init: true) + def translated_code(tag) + Code::PrinterCode.new(type: :printer, token_code: token_code, tag: tag).translated_code + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/reference.rb b/tool/lrama/lib/lrama/grammar/reference.rb new file mode 100644 index 0000000000..c56e7673a6 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/reference.rb @@ -0,0 +1,14 @@ +module Lrama + class Grammar + # type: :dollar or :at + # name: String (e.g. $$, $foo, $expr.right) + # number: Integer (e.g. $1) + # index: Integer + # ex_tag: "$<tag>1" (Optional) + class Reference < Struct.new(:type, :name, :number, :index, :ex_tag, :first_column, :last_column, keyword_init: true) + def value + name || number + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/rule.rb b/tool/lrama/lib/lrama/grammar/rule.rb new file mode 100644 index 0000000000..9281e0574f --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/rule.rb @@ -0,0 +1,56 @@ +module Lrama + class Grammar + # _rhs holds original RHS element. Use rhs to refer to Symbol. + class Rule < Struct.new(:id, :_lhs, :lhs, :lhs_tag, :_rhs, :rhs, :token_code, :position_in_original_rule_rhs, :nullable, :precedence_sym, :lineno, keyword_init: true) + attr_accessor :original_rule + + def ==(other) + self.class == other.class && + self.lhs == other.lhs && + self.lhs_tag == other.lhs_tag && + self.rhs == other.rhs && + self.token_code == other.token_code && + self.position_in_original_rule_rhs == other.position_in_original_rule_rhs && + self.nullable == other.nullable && + self.precedence_sym == other.precedence_sym && + self.lineno == other.lineno + end + + # TODO: Change this to display_name + def to_s + l = lhs.id.s_value + r = empty_rule? ? "ε" : rhs.map {|r| r.id.s_value }.join(", ") + + "#{l} -> #{r}" + end + + # Used by #user_actions + def as_comment + l = lhs.id.s_value + r = empty_rule? ? "%empty" : rhs.map(&:display_name).join(" ") + + "#{l}: #{r}" + end + + # opt_nl: ε <-- empty_rule + # | '\n' <-- not empty_rule + def empty_rule? + rhs.empty? + end + + def precedence + precedence_sym&.precedence + end + + def initial_rule? + id == 0 + end + + def translated_code + return nil unless token_code + + Code::RuleAction.new(type: :rule_action, token_code: token_code, rule: self).translated_code + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/rule_builder.rb b/tool/lrama/lib/lrama/grammar/rule_builder.rb new file mode 100644 index 0000000000..b2ccc3e243 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/rule_builder.rb @@ -0,0 +1,218 @@ +module Lrama + class Grammar + class RuleBuilder + attr_accessor :lhs, :line + attr_reader :lhs_tag, :rhs, :user_code, :precedence_sym + + def initialize(rule_counter, midrule_action_counter, position_in_original_rule_rhs = nil, lhs_tag: nil, skip_preprocess_references: false) + @rule_counter = rule_counter + @midrule_action_counter = midrule_action_counter + @position_in_original_rule_rhs = position_in_original_rule_rhs + @skip_preprocess_references = skip_preprocess_references + + @lhs = nil + @lhs_tag = lhs_tag + @rhs = [] + @user_code = nil + @precedence_sym = nil + @line = nil + @rule_builders_for_parameterizing_rules = [] + @rule_builders_for_derived_rules = [] + end + + def add_rhs(rhs) + if !@line + @line = rhs.line + end + + flush_user_code + + @rhs << rhs + end + + def user_code=(user_code) + if !@line + @line = user_code&.line + end + + flush_user_code + + @user_code = user_code + end + + def precedence_sym=(precedence_sym) + flush_user_code + + @precedence_sym = precedence_sym + end + + def complete_input + freeze_rhs + end + + def setup_rules(parameterizing_rule_resolver) + preprocess_references unless @skip_preprocess_references + process_rhs(parameterizing_rule_resolver) + build_rules + end + + def rules + @parameterizing_rules + @midrule_action_rules + @rules + end + + private + + def freeze_rhs + @rhs.freeze + end + + def preprocess_references + numberize_references + end + + def build_rules + tokens = @replaced_rhs + + rule = Rule.new( + id: @rule_counter.increment, _lhs: lhs, _rhs: tokens, lhs_tag: lhs_tag, token_code: user_code, + position_in_original_rule_rhs: @position_in_original_rule_rhs, precedence_sym: precedence_sym, lineno: line + ) + @rules = [rule] + @parameterizing_rules = @rule_builders_for_parameterizing_rules.map do |rule_builder| + rule_builder.rules + end.flatten + @midrule_action_rules = @rule_builders_for_derived_rules.map do |rule_builder| + rule_builder.rules + end.flatten + @midrule_action_rules.each do |r| + r.original_rule = rule + end + end + + # rhs is a mixture of variety type of tokens like `Ident`, `InstantiateRule`, `UserCode` and so on. + # `#process_rhs` replaces some kind of tokens to `Ident` so that all `@replaced_rhs` are `Ident` or `Char`. + def process_rhs(parameterizing_rule_resolver) + return if @replaced_rhs + + @replaced_rhs = [] + + rhs.each_with_index do |token, i| + case token + when Lrama::Lexer::Token::Char + @replaced_rhs << token + when Lrama::Lexer::Token::Ident + @replaced_rhs << token + when Lrama::Lexer::Token::InstantiateRule + parameterizing_rule = parameterizing_rule_resolver.find(token) + raise "Unexpected token. #{token}" unless parameterizing_rule + + bindings = Binding.new(parameterizing_rule, token.args) + lhs_s_value = lhs_s_value(token, bindings) + if (created_lhs = parameterizing_rule_resolver.created_lhs(lhs_s_value)) + @replaced_rhs << created_lhs + else + lhs_token = Lrama::Lexer::Token::Ident.new(s_value: lhs_s_value, location: token.location) + @replaced_rhs << lhs_token + parameterizing_rule_resolver.created_lhs_list << lhs_token + parameterizing_rule.rhs_list.each do |r| + rule_builder = RuleBuilder.new(@rule_counter, @midrule_action_counter, lhs_tag: token.lhs_tag, skip_preprocess_references: true) + rule_builder.lhs = lhs_token + r.symbols.each { |sym| rule_builder.add_rhs(bindings.resolve_symbol(sym)) } + rule_builder.line = line + rule_builder.precedence_sym = r.precedence_sym + rule_builder.user_code = r.user_code + rule_builder.complete_input + rule_builder.setup_rules(parameterizing_rule_resolver) + @rule_builders_for_parameterizing_rules << rule_builder + end + end + when Lrama::Lexer::Token::UserCode + prefix = token.referred ? "@" : "$@" + tag = token.tag || lhs_tag + new_token = Lrama::Lexer::Token::Ident.new(s_value: prefix + @midrule_action_counter.increment.to_s) + @replaced_rhs << new_token + + rule_builder = RuleBuilder.new(@rule_counter, @midrule_action_counter, i, lhs_tag: tag, skip_preprocess_references: true) + rule_builder.lhs = new_token + rule_builder.user_code = token + rule_builder.complete_input + rule_builder.setup_rules(parameterizing_rule_resolver) + + @rule_builders_for_derived_rules << rule_builder + else + raise "Unexpected token. #{token}" + end + end + end + + def lhs_s_value(token, bindings) + s_values = token.args.map do |arg| + resolved = bindings.resolve_symbol(arg) + if resolved.is_a?(Lexer::Token::InstantiateRule) + [resolved.s_value, resolved.args.map(&:s_value)] + else + resolved.s_value + end + end + "#{token.rule_name}_#{s_values.join('_')}" + end + + def numberize_references + # Bison n'th component is 1-origin + (rhs + [user_code]).compact.each.with_index(1) do |token, i| + next unless token.is_a?(Lrama::Lexer::Token::UserCode) + + token.references.each do |ref| + ref_name = ref.name + + if ref_name + if ref_name == '$' + ref.name = '$' + else + candidates = ([lhs] + rhs).each_with_index.select {|token, _i| token.referred_by?(ref_name) } + + if candidates.size >= 2 + token.invalid_ref(ref, "Referring symbol `#{ref_name}` is duplicated.") + end + + unless (referring_symbol = candidates.first) + token.invalid_ref(ref, "Referring symbol `#{ref_name}` is not found.") + end + + if referring_symbol[1] == 0 # Refers to LHS + ref.name = '$' + else + ref.number = referring_symbol[1] + end + end + end + + if ref.number + # TODO: When Inlining is implemented, for example, if `$1` is expanded to multiple RHS tokens, + # `$2` needs to access `$2 + n` to actually access it. So, after the Inlining implementation, + # it needs resolves from number to index. + ref.index = ref.number + end + + # TODO: Need to check index of @ too? + next if ref.type == :at + + if ref.index + # TODO: Prohibit $0 even so Bison allows it? + # See: https://www.gnu.org/software/bison/manual/html_node/Actions.html + token.invalid_ref(ref, "Can not refer following component. #{ref.index} >= #{i}.") if ref.index >= i + rhs[ref.index - 1].referred = true + end + end + end + end + + def flush_user_code + if (c = @user_code) + @rhs << c + @user_code = nil + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/stdlib.y b/tool/lrama/lib/lrama/grammar/stdlib.y new file mode 100644 index 0000000000..d6e89c908c --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/stdlib.y @@ -0,0 +1,122 @@ +/********************************************************************** + + stdlib.y + + This is lrama's standard library. It provides a number of + parameterizing rule definitions, such as options and lists, + that should be useful in a number of situations. + +**********************************************************************/ + +// ------------------------------------------------------------------- +// Options + +/* + * program: option(number) + * + * => + * + * program: option_number + * option_number: %empty + * option_number: number + */ +%rule option(X): /* empty */ + | X + ; + +// ------------------------------------------------------------------- +// Sequences + +/* + * program: preceded(opening, X) + * + * => + * + * program: preceded_opening_X + * preceded_opening_X: opening X + */ +%rule preceded(opening, X): opening X { $$ = $2; } + ; + +/* + * program: terminated(X, closing) + * + * => + * + * program: terminated_X_closing + * terminated_X_closing: X closing + */ +%rule terminated(X, closing): X closing { $$ = $1; } + ; + +/* + * program: delimited(opening, X, closing) + * + * => + * + * program: delimited_opening_X_closing + * delimited_opening_X_closing: opening X closing + */ +%rule delimited(opening, X, closing): opening X closing { $$ = $2; } + ; + +// ------------------------------------------------------------------- +// Lists + +/* + * program: list(number) + * + * => + * + * program: list_number + * list_number: %empty + * list_number: list_number number + */ +%rule list(X): /* empty */ + | list(X) X + ; + +/* + * program: nonempty_list(number) + * + * => + * + * program: nonempty_list_number + * nonempty_list_number: number + * nonempty_list_number: nonempty_list_number number + */ +%rule nonempty_list(X): X + | nonempty_list(X) X + ; + +/* + * program: separated_nonempty_list(comma, number) + * + * => + * + * program: separated_nonempty_list_comma_number + * separated_nonempty_list_comma_number: number + * separated_nonempty_list_comma_number: separated_nonempty_list_comma_number comma number + */ +%rule separated_nonempty_list(separator, X): X + | separated_nonempty_list(separator, X) separator X + ; + +/* + * program: separated_list(comma, number) + * + * => + * + * program: separated_list_comma_number + * separated_list_comma_number: option_separated_nonempty_list_comma_number + * option_separated_nonempty_list_comma_number: %empty + * option_separated_nonempty_list_comma_number: separated_nonempty_list_comma_number + * separated_nonempty_list_comma_number: number + * separated_nonempty_list_comma_number: comma separated_nonempty_list_comma_number number + */ +%rule separated_list(separator, X): option(separated_nonempty_list(separator, X)) + ; + +%% + +%union{}; diff --git a/tool/lrama/lib/lrama/grammar/symbol.rb b/tool/lrama/lib/lrama/grammar/symbol.rb new file mode 100644 index 0000000000..deb67ad9a8 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/symbol.rb @@ -0,0 +1,103 @@ +# Symbol is both of nterm and term +# `number` is both for nterm and term +# `token_id` is tokentype for term, internal sequence number for nterm +# +# TODO: Add validation for ASCII code range for Token::Char + +module Lrama + class Grammar + class Symbol + attr_accessor :id, :alias_name, :tag, :number, :token_id, :nullable, :precedence, + :printer, :destructor, :error_token, :first_set, :first_set_bitmap + attr_reader :term + attr_writer :eof_symbol, :error_symbol, :undef_symbol, :accept_symbol + + def initialize(id:, term:, alias_name: nil, number: nil, tag: nil, token_id: nil, nullable: nil, precedence: nil, printer: nil, destructor: nil) + @id = id + @alias_name = alias_name + @number = number + @tag = tag + @term = term + @token_id = token_id + @nullable = nullable + @precedence = precedence + @printer = printer + @destructor = destructor + end + + def term? + term + end + + def nterm? + !term + end + + def eof_symbol? + !!@eof_symbol + end + + def error_symbol? + !!@error_symbol + end + + def undef_symbol? + !!@undef_symbol + end + + def accept_symbol? + !!@accept_symbol + end + + def display_name + alias_name || id.s_value + end + + # name for yysymbol_kind_t + # + # See: b4_symbol_kind_base + # @type var name: String + def enum_name + case + when accept_symbol? + name = "YYACCEPT" + when eof_symbol? + name = "YYEOF" + when term? && id.is_a?(Lrama::Lexer::Token::Char) + name = number.to_s + display_name + when term? && id.is_a?(Lrama::Lexer::Token::Ident) + name = id.s_value + when nterm? && (id.s_value.include?("$") || id.s_value.include?("@")) + name = number.to_s + id.s_value + when nterm? + name = id.s_value + else + raise "Unexpected #{self}" + end + + "YYSYMBOL_" + name.gsub(/\W+/, "_") + end + + # comment for yysymbol_kind_t + def comment + case + when accept_symbol? + # YYSYMBOL_YYACCEPT + id.s_value + when eof_symbol? + # YYEOF + alias_name + when (term? && 0 < token_id && token_id < 128) + # YYSYMBOL_3_backslash_, YYSYMBOL_14_ + alias_name || id.s_value + when id.s_value.include?("$") || id.s_value.include?("@") + # YYSYMBOL_21_1 + id.s_value + else + # YYSYMBOL_keyword_class, YYSYMBOL_strings_1 + alias_name || id.s_value + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/symbols.rb b/tool/lrama/lib/lrama/grammar/symbols.rb new file mode 100644 index 0000000000..cc9b4ec559 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/symbols.rb @@ -0,0 +1 @@ +require_relative "symbols/resolver" diff --git a/tool/lrama/lib/lrama/grammar/symbols/resolver.rb b/tool/lrama/lib/lrama/grammar/symbols/resolver.rb new file mode 100644 index 0000000000..1788ed63fa --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/symbols/resolver.rb @@ -0,0 +1,293 @@ +module Lrama + class Grammar + class Symbols + class Resolver + attr_reader :terms, :nterms + + def initialize + @terms = [] + @nterms = [] + end + + def symbols + @symbols ||= (@terms + @nterms) + end + + def sort_by_number! + symbols.sort_by!(&:number) + end + + def add_term(id:, alias_name: nil, tag: nil, token_id: nil, replace: false) + if token_id && (sym = find_symbol_by_token_id(token_id)) + if replace + sym.id = id + sym.alias_name = alias_name + sym.tag = tag + end + + return sym + end + + if (sym = find_symbol_by_id(id)) + return sym + end + + @symbols = nil + term = Symbol.new( + id: id, alias_name: alias_name, number: nil, tag: tag, + term: true, token_id: token_id, nullable: false + ) + @terms << term + term + end + + def add_nterm(id:, alias_name: nil, tag: nil) + return if find_symbol_by_id(id) + + @symbols = nil + nterm = Symbol.new( + id: id, alias_name: alias_name, number: nil, tag: tag, + term: false, token_id: nil, nullable: nil, + ) + @nterms << nterm + nterm + end + + def find_symbol_by_s_value(s_value) + symbols.find { |s| s.id.s_value == s_value } + end + + def find_symbol_by_s_value!(s_value) + find_symbol_by_s_value(s_value) || (raise "Symbol not found. value: `#{s_value}`") + end + + def find_symbol_by_id(id) + symbols.find do |s| + s.id == id || s.alias_name == id.s_value + end + end + + def find_symbol_by_id!(id) + find_symbol_by_id(id) || (raise "Symbol not found. #{id}") + end + + def find_symbol_by_token_id(token_id) + symbols.find {|s| s.token_id == token_id } + end + + def find_symbol_by_number!(number) + sym = symbols[number] + + raise "Symbol not found. number: `#{number}`" unless sym + raise "[BUG] Symbol number mismatch. #{number}, #{sym}" if sym.number != number + + sym + end + + def fill_symbol_number + # YYEMPTY = -2 + # YYEOF = 0 + # YYerror = 1 + # YYUNDEF = 2 + @number = 3 + fill_terms_number + fill_nterms_number + end + + def fill_nterm_type(types) + types.each do |type| + nterm = find_nterm_by_id!(type.id) + nterm.tag = type.tag + end + end + + def fill_printer(printers) + symbols.each do |sym| + printers.each do |printer| + printer.ident_or_tags.each do |ident_or_tag| + case ident_or_tag + when Lrama::Lexer::Token::Ident + sym.printer = printer if sym.id == ident_or_tag + when Lrama::Lexer::Token::Tag + sym.printer = printer if sym.tag == ident_or_tag + else + raise "Unknown token type. #{printer}" + end + end + end + end + end + + def fill_destructor(destructors) + symbols.each do |sym| + destructors.each do |destructor| + destructor.ident_or_tags.each do |ident_or_tag| + case ident_or_tag + when Lrama::Lexer::Token::Ident + sym.destructor = destructor if sym.id == ident_or_tag + when Lrama::Lexer::Token::Tag + sym.destructor = destructor if sym.tag == ident_or_tag + else + raise "Unknown token type. #{destructor}" + end + end + end + end + end + + def fill_error_token(error_tokens) + symbols.each do |sym| + error_tokens.each do |token| + token.ident_or_tags.each do |ident_or_tag| + case ident_or_tag + when Lrama::Lexer::Token::Ident + sym.error_token = token if sym.id == ident_or_tag + when Lrama::Lexer::Token::Tag + sym.error_token = token if sym.tag == ident_or_tag + else + raise "Unknown token type. #{token}" + end + end + end + end + end + + def token_to_symbol(token) + case token + when Lrama::Lexer::Token + find_symbol_by_id!(token) + else + raise "Unknown class: #{token}" + end + end + + def validate! + validate_number_uniqueness! + validate_alias_name_uniqueness! + end + + private + + def find_nterm_by_id!(id) + @nterms.find do |s| + s.id == id + end || (raise "Symbol not found. #{id}") + end + + def fill_terms_number + # Character literal in grammar file has + # token id corresponding to ASCII code by default, + # so start token_id from 256. + token_id = 256 + + @terms.each do |sym| + while used_numbers[@number] do + @number += 1 + end + + if sym.number.nil? + sym.number = @number + used_numbers[@number] = true + @number += 1 + end + + # If id is Token::Char, it uses ASCII code + if sym.token_id.nil? + if sym.id.is_a?(Lrama::Lexer::Token::Char) + # Ignore ' on the both sides + case sym.id.s_value[1..-2] + when "\\b" + sym.token_id = 8 + when "\\f" + sym.token_id = 12 + when "\\n" + sym.token_id = 10 + when "\\r" + sym.token_id = 13 + when "\\t" + sym.token_id = 9 + when "\\v" + sym.token_id = 11 + when "\"" + sym.token_id = 34 + when "'" + sym.token_id = 39 + when "\\\\" + sym.token_id = 92 + when /\A\\(\d+)\z/ + unless (id = Integer($1, 8)).nil? + sym.token_id = id + else + raise "Unknown Char s_value #{sym}" + end + when /\A(.)\z/ + unless (id = $1&.bytes&.first).nil? + sym.token_id = id + else + raise "Unknown Char s_value #{sym}" + end + else + raise "Unknown Char s_value #{sym}" + end + else + sym.token_id = token_id + token_id += 1 + end + end + end + end + + def fill_nterms_number + token_id = 0 + + @nterms.each do |sym| + while used_numbers[@number] do + @number += 1 + end + + if sym.number.nil? + sym.number = @number + used_numbers[@number] = true + @number += 1 + end + + if sym.token_id.nil? + sym.token_id = token_id + token_id += 1 + end + end + end + + def used_numbers + return @used_numbers if defined?(@used_numbers) + + @used_numbers = {} + symbols.map(&:number).each do |n| + @used_numbers[n] = true + end + @used_numbers + end + + def validate_number_uniqueness! + invalid = symbols.group_by(&:number).select do |number, syms| + syms.count > 1 + end + + return if invalid.empty? + + raise "Symbol number is duplicated. #{invalid}" + end + + def validate_alias_name_uniqueness! + invalid = symbols.select(&:alias_name).group_by(&:alias_name).select do |alias_name, syms| + syms.count > 1 + end + + return if invalid.empty? + + raise "Symbol alias name is duplicated. #{invalid}" + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/type.rb b/tool/lrama/lib/lrama/grammar/type.rb new file mode 100644 index 0000000000..6b4b0961a1 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/type.rb @@ -0,0 +1,18 @@ +module Lrama + class Grammar + class Type + attr_reader :id, :tag + + def initialize(id:, tag:) + @id = id + @tag = tag + end + + def ==(other) + self.class == other.class && + self.id == other.id && + self.tag == other.tag + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/union.rb b/tool/lrama/lib/lrama/grammar/union.rb new file mode 100644 index 0000000000..854bffb5c1 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/union.rb @@ -0,0 +1,10 @@ +module Lrama + class Grammar + class Union < Struct.new(:code, :lineno, keyword_init: true) + def braces_less_code + # Braces is already removed by lexer + code.s_value + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer.rb b/tool/lrama/lib/lrama/lexer.rb new file mode 100644 index 0000000000..db8f384fe6 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer.rb @@ -0,0 +1,187 @@ +require "strscan" + +require "lrama/lexer/grammar_file" +require "lrama/lexer/location" +require "lrama/lexer/token" + +module Lrama + class Lexer + attr_reader :head_line, :head_column, :line + attr_accessor :status, :end_symbol + + SYMBOLS = ['%{', '%}', '%%', '{', '}', '\[', '\]', '\(', '\)', '\,', ':', '\|', ';'] + PERCENT_TOKENS = %w( + %union + %token + %type + %left + %right + %nonassoc + %expect + %define + %require + %printer + %destructor + %lex-param + %parse-param + %initial-action + %precedence + %prec + %error-token + %before-reduce + %after-reduce + %after-shift-error-token + %after-shift + %after-pop-stack + %empty + %code + %rule + %no-stdlib + ) + + def initialize(grammar_file) + @grammar_file = grammar_file + @scanner = StringScanner.new(grammar_file.text) + @head_column = @head = @scanner.pos + @head_line = @line = 1 + @status = :initial + @end_symbol = nil + end + + def next_token + case @status + when :initial + lex_token + when :c_declaration + lex_c_code + end + end + + def column + @scanner.pos - @head + end + + def location + Location.new( + grammar_file: @grammar_file, + first_line: @head_line, first_column: @head_column, + last_line: line, last_column: column + ) + end + + def lex_token + while !@scanner.eos? do + case + when @scanner.scan(/\n/) + newline + when @scanner.scan(/\s+/) + # noop + when @scanner.scan(/\/\*/) + lex_comment + when @scanner.scan(/\/\/.*(?<newline>\n)?/) + newline if @scanner[:newline] + else + break + end + end + + reset_first_position + + case + when @scanner.eos? + return + when @scanner.scan(/#{SYMBOLS.join('|')}/) + return [@scanner.matched, @scanner.matched] + when @scanner.scan(/#{PERCENT_TOKENS.join('|')}/) + return [@scanner.matched, @scanner.matched] + when @scanner.scan(/[\?\+\*]/) + return [@scanner.matched, @scanner.matched] + when @scanner.scan(/<\w+>/) + return [:TAG, Lrama::Lexer::Token::Tag.new(s_value: @scanner.matched, location: location)] + when @scanner.scan(/'.'/) + return [:CHARACTER, Lrama::Lexer::Token::Char.new(s_value: @scanner.matched, location: location)] + when @scanner.scan(/'\\\\'|'\\b'|'\\t'|'\\f'|'\\r'|'\\n'|'\\v'|'\\13'/) + return [:CHARACTER, Lrama::Lexer::Token::Char.new(s_value: @scanner.matched, location: location)] + when @scanner.scan(/".*?"/) + return [:STRING, %Q(#{@scanner.matched})] + when @scanner.scan(/\d+/) + return [:INTEGER, Integer(@scanner.matched)] + when @scanner.scan(/([a-zA-Z_.][-a-zA-Z0-9_.]*)/) + token = Lrama::Lexer::Token::Ident.new(s_value: @scanner.matched, location: location) + type = + if @scanner.check(/\s*(\[\s*[a-zA-Z_.][-a-zA-Z0-9_.]*\s*\])?\s*:/) + :IDENT_COLON + else + :IDENTIFIER + end + return [type, token] + else + raise ParseError, "Unexpected token: #{@scanner.peek(10).chomp}." + end + end + + def lex_c_code + nested = 0 + code = '' + reset_first_position + + while !@scanner.eos? do + case + when @scanner.scan(/{/) + code += @scanner.matched + nested += 1 + when @scanner.scan(/}/) + if nested == 0 && @end_symbol == '}' + @scanner.unscan + return [:C_DECLARATION, Lrama::Lexer::Token::UserCode.new(s_value: code, location: location)] + else + code += @scanner.matched + nested -= 1 + end + when @scanner.check(/#{@end_symbol}/) + return [:C_DECLARATION, Lrama::Lexer::Token::UserCode.new(s_value: code, location: location)] + when @scanner.scan(/\n/) + code += @scanner.matched + newline + when @scanner.scan(/".*?"/) + code += %Q(#{@scanner.matched}) + @line += @scanner.matched.count("\n") + when @scanner.scan(/'.*?'/) + code += %Q(#{@scanner.matched}) + when @scanner.scan(/[^\"'\{\}\n]+/) + code += @scanner.matched + when @scanner.scan(/#{Regexp.escape(@end_symbol)}/) + code += @scanner.matched + else + code += @scanner.getch + end + end + raise ParseError, "Unexpected code: #{code}." + end + + private + + def lex_comment + while !@scanner.eos? do + case + when @scanner.scan(/\n/) + newline + when @scanner.scan(/\*\//) + return + else + @scanner.getch + end + end + end + + def reset_first_position + @head_line = line + @head_column = column + end + + def newline + @line += 1 + @head = @scanner.pos + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/grammar_file.rb b/tool/lrama/lib/lrama/lexer/grammar_file.rb new file mode 100644 index 0000000000..6be0767004 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/grammar_file.rb @@ -0,0 +1,21 @@ +module Lrama + class Lexer + class GrammarFile + attr_reader :path, :text + + def initialize(path, text) + @path = path + @text = text.freeze + end + + def ==(other) + self.class == other.class && + self.path == other.path + end + + def lines + @lines ||= text.split("\n") + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/location.rb b/tool/lrama/lib/lrama/lexer/location.rb new file mode 100644 index 0000000000..aefce3e16b --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/location.rb @@ -0,0 +1,97 @@ +module Lrama + class Lexer + class Location + attr_reader :grammar_file, :first_line, :first_column, :last_line, :last_column + + def initialize(grammar_file:, first_line:, first_column:, last_line:, last_column:) + @grammar_file = grammar_file + @first_line = first_line + @first_column = first_column + @last_line = last_line + @last_column = last_column + end + + def ==(other) + self.class == other.class && + self.grammar_file == other.grammar_file && + self.first_line == other.first_line && + self.first_column == other.first_column && + self.last_line == other.last_line && + self.last_column == other.last_column + end + + def partial_location(left, right) + offset = -first_column + new_first_line = -1 + new_first_column = -1 + new_last_line = -1 + new_last_column = -1 + + _text.each.with_index do |line, index| + new_offset = offset + line.length + 1 + + if offset <= left && left <= new_offset + new_first_line = first_line + index + new_first_column = left - offset + end + + if offset <= right && right <= new_offset + new_last_line = first_line + index + new_last_column = right - offset + end + + offset = new_offset + end + + Location.new( + grammar_file: grammar_file, + first_line: new_first_line, first_column: new_first_column, + last_line: new_last_line, last_column: new_last_column + ) + end + + def to_s + "#{path} (#{first_line},#{first_column})-(#{last_line},#{last_column})" + end + + def generate_error_message(error_message) + <<~ERROR.chomp + #{path}:#{first_line}:#{first_column}: #{error_message} + #{line_with_carets} + ERROR + end + + def line_with_carets + <<~TEXT + #{text} + #{carets} + TEXT + end + + private + + def path + grammar_file.path + end + + def blanks + (text[0...first_column] or raise "#{first_column} is invalid").gsub(/[^\t]/, ' ') + end + + def carets + blanks + '^' * (last_column - first_column) + end + + def text + @text ||= _text.join("\n") + end + + def _text + @_text ||=begin + range = (first_line - 1)...last_line + grammar_file.lines[range] or raise "#{range} is invalid" + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token.rb b/tool/lrama/lib/lrama/lexer/token.rb new file mode 100644 index 0000000000..59b49d5fba --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token.rb @@ -0,0 +1,56 @@ +require 'lrama/lexer/token/char' +require 'lrama/lexer/token/ident' +require 'lrama/lexer/token/instantiate_rule' +require 'lrama/lexer/token/tag' +require 'lrama/lexer/token/user_code' + +module Lrama + class Lexer + class Token + attr_reader :s_value, :location + attr_accessor :alias_name, :referred + + def initialize(s_value:, alias_name: nil, location: nil) + s_value.freeze + @s_value = s_value + @alias_name = alias_name + @location = location + end + + def to_s + "value: `#{s_value}`, location: #{location}" + end + + def referred_by?(string) + [self.s_value, self.alias_name].compact.include?(string) + end + + def ==(other) + self.class == other.class && self.s_value == other.s_value + end + + def first_line + location.first_line + end + alias :line :first_line + + def first_column + location.first_column + end + alias :column :first_column + + def last_line + location.last_line + end + + def last_column + location.last_column + end + + def invalid_ref(ref, message) + location = self.location.partial_location(ref.first_column, ref.last_column) + raise location.generate_error_message(message) + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/char.rb b/tool/lrama/lib/lrama/lexer/token/char.rb new file mode 100644 index 0000000000..ec3560ca09 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/char.rb @@ -0,0 +1,8 @@ +module Lrama + class Lexer + class Token + class Char < Token + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/ident.rb b/tool/lrama/lib/lrama/lexer/token/ident.rb new file mode 100644 index 0000000000..e576eaeccd --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/ident.rb @@ -0,0 +1,8 @@ +module Lrama + class Lexer + class Token + class Ident < Token + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/instantiate_rule.rb b/tool/lrama/lib/lrama/lexer/token/instantiate_rule.rb new file mode 100644 index 0000000000..1c4d1095c8 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/instantiate_rule.rb @@ -0,0 +1,23 @@ +module Lrama + class Lexer + class Token + class InstantiateRule < Token + attr_reader :args, :lhs_tag + + def initialize(s_value:, alias_name: nil, location: nil, args: [], lhs_tag: nil) + super s_value: s_value, alias_name: alias_name, location: location + @args = args + @lhs_tag = lhs_tag + end + + def rule_name + s_value + end + + def args_count + args.count + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/tag.rb b/tool/lrama/lib/lrama/lexer/token/tag.rb new file mode 100644 index 0000000000..e54d773915 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/tag.rb @@ -0,0 +1,12 @@ +module Lrama + class Lexer + class Token + class Tag < Token + # Omit "<>" + def member + s_value[1..-2] or raise "Unexpected Tag format (#{s_value})" + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/user_code.rb b/tool/lrama/lib/lrama/lexer/token/user_code.rb new file mode 100644 index 0000000000..4d487bf01c --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/user_code.rb @@ -0,0 +1,77 @@ +require "strscan" + +module Lrama + class Lexer + class Token + class UserCode < Token + attr_accessor :tag + + def references + @references ||= _references + end + + private + + def _references + scanner = StringScanner.new(s_value) + references = [] + + while !scanner.eos? do + case + when reference = scan_reference(scanner) + references << reference + when scanner.scan(/\/\*/) + scanner.scan_until(/\*\//) + else + scanner.getch + end + end + + references + end + + def scan_reference(scanner) + start = scanner.pos + case + # $ references + # It need to wrap an identifier with brackets to use ".-" for identifiers + when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?\$/) # $$, $<long>$ + tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil + return Lrama::Grammar::Reference.new(type: :dollar, name: "$", ex_tag: tag, first_column: start, last_column: scanner.pos) + when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?(\d+)/) # $1, $2, $<long>1 + tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil + return Lrama::Grammar::Reference.new(type: :dollar, number: Integer(scanner[2]), index: Integer(scanner[2]), ex_tag: tag, first_column: start, last_column: scanner.pos) + when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?([a-zA-Z_][a-zA-Z0-9_]*)/) # $foo, $expr, $<long>program (named reference without brackets) + tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil + return Lrama::Grammar::Reference.new(type: :dollar, name: scanner[2], ex_tag: tag, first_column: start, last_column: scanner.pos) + when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?\[([a-zA-Z_.][-a-zA-Z0-9_.]*)\]/) # $[expr.right], $[expr-right], $<long>[expr.right] (named reference with brackets) + tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil + return Lrama::Grammar::Reference.new(type: :dollar, name: scanner[2], ex_tag: tag, first_column: start, last_column: scanner.pos) + + # @ references + # It need to wrap an identifier with brackets to use ".-" for identifiers + when scanner.scan(/@\$/) # @$ + return Lrama::Grammar::Reference.new(type: :at, name: "$", first_column: start, last_column: scanner.pos) + when scanner.scan(/@(\d+)/) # @1 + return Lrama::Grammar::Reference.new(type: :at, number: Integer(scanner[1]), index: Integer(scanner[1]), first_column: start, last_column: scanner.pos) + when scanner.scan(/@([a-zA-Z][a-zA-Z0-9_]*)/) # @foo, @expr (named reference without brackets) + return Lrama::Grammar::Reference.new(type: :at, name: scanner[1], first_column: start, last_column: scanner.pos) + when scanner.scan(/@\[([a-zA-Z_.][-a-zA-Z0-9_.]*)\]/) # @[expr.right], @[expr-right] (named reference with brackets) + return Lrama::Grammar::Reference.new(type: :at, name: scanner[1], first_column: start, last_column: scanner.pos) + + # $: references + when scanner.scan(/\$:\$/) # $:$ + return Lrama::Grammar::Reference.new(type: :index, name: "$", first_column: start, last_column: scanner.pos) + when scanner.scan(/\$:(\d+)/) # $:1 + return Lrama::Grammar::Reference.new(type: :index, number: Integer(scanner[1]), first_column: start, last_column: scanner.pos) + when scanner.scan(/\$:([a-zA-Z_][a-zA-Z0-9_]*)/) # $:foo, $:expr (named reference without brackets) + return Lrama::Grammar::Reference.new(type: :index, name: scanner[1], first_column: start, last_column: scanner.pos) + when scanner.scan(/\$:\[([a-zA-Z_.][-a-zA-Z0-9_.]*)\]/) # $:[expr.right], $:[expr-right] (named reference with brackets) + return Lrama::Grammar::Reference.new(type: :index, name: scanner[1], first_column: start, last_column: scanner.pos) + + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/option_parser.rb b/tool/lrama/lib/lrama/option_parser.rb new file mode 100644 index 0000000000..3210b091ed --- /dev/null +++ b/tool/lrama/lib/lrama/option_parser.rb @@ -0,0 +1,141 @@ +require 'optparse' + +module Lrama + # Handle option parsing for the command line interface. + class OptionParser + def initialize + @options = Options.new + @trace = [] + @report = [] + end + + def parse(argv) + parse_by_option_parser(argv) + + @options.trace_opts = validate_trace(@trace) + @options.report_opts = validate_report(@report) + @options.grammar_file = argv.shift + + if !@options.grammar_file + abort "File should be specified\n" + end + + if @options.grammar_file == '-' + @options.grammar_file = argv.shift or abort "File name for STDIN should be specified\n" + else + @options.y = File.open(@options.grammar_file, 'r') + end + + if !@report.empty? && @options.report_file.nil? && @options.grammar_file + @options.report_file = File.dirname(@options.grammar_file) + "/" + File.basename(@options.grammar_file, ".*") + ".output" + end + + if !@options.header_file && @options.header + case + when @options.outfile + @options.header_file = File.dirname(@options.outfile) + "/" + File.basename(@options.outfile, ".*") + ".h" + when @options.grammar_file + @options.header_file = File.dirname(@options.grammar_file) + "/" + File.basename(@options.grammar_file, ".*") + ".h" + end + end + + @options + end + + private + + def parse_by_option_parser(argv) + ::OptionParser.new do |o| + o.banner = <<~BANNER + Lrama is LALR (1) parser generator written by Ruby. + + Usage: lrama [options] FILE + BANNER + o.separator '' + o.separator 'STDIN mode:' + o.separator 'lrama [options] - FILE read grammar from STDIN' + o.separator '' + o.separator 'Tuning the Parser:' + o.on('-S', '--skeleton=FILE', 'specify the skeleton to use') {|v| @options.skeleton = v } + o.on('-t', 'reserved, do nothing') { } + o.on('--debug', 'display debugging outputs of internal parser') {|v| @options.debug = true } + o.separator '' + o.separator 'Output:' + o.on('-H', '--header=[FILE]', 'also produce a header file named FILE') {|v| @options.header = true; @options.header_file = v } + o.on('-d', 'also produce a header file') { @options.header = true } + o.on('-r', '--report=THINGS', Array, 'also produce details on the automaton') {|v| @report = v } + o.on_tail '' + o.on_tail 'Valid Reports:' + o.on_tail " #{VALID_REPORTS.join(' ')}" + + o.on('--report-file=FILE', 'also produce details on the automaton output to a file named FILE') {|v| @options.report_file = v } + o.on('-o', '--output=FILE', 'leave output to FILE') {|v| @options.outfile = v } + + o.on('--trace=THINGS', Array, 'also output trace logs at runtime') {|v| @trace = v } + o.on_tail '' + o.on_tail 'Valid Traces:' + o.on_tail " #{VALID_TRACES.join(' ')}" + + o.on('-v', 'reserved, do nothing') { } + o.separator '' + o.separator 'Error Recovery:' + o.on('-e', 'enable error recovery') {|v| @options.error_recovery = true } + o.separator '' + o.separator 'Other options:' + o.on('-V', '--version', "output version information and exit") {|v| puts "lrama #{Lrama::VERSION}"; exit 0 } + o.on('-h', '--help', "display this help and exit") {|v| puts o; exit 0 } + o.on_tail + o.parse!(argv) + end + end + + BISON_REPORTS = %w[states itemsets lookaheads solved counterexamples cex all none] + OTHER_REPORTS = %w[verbose] + NOT_SUPPORTED_REPORTS = %w[cex none] + VALID_REPORTS = BISON_REPORTS + OTHER_REPORTS - NOT_SUPPORTED_REPORTS + + def validate_report(report) + list = VALID_REPORTS + h = { grammar: true } + + report.each do |r| + if list.include?(r) + h[r.to_sym] = true + else + raise "Invalid report option \"#{r}\"." + end + end + + if h[:all] + (BISON_REPORTS - NOT_SUPPORTED_REPORTS).each do |r| + h[r.to_sym] = true + end + + h.delete(:all) + end + + return h + end + + VALID_TRACES = %w[ + none locations scan parse automaton bitsets + closure grammar rules resource sets muscles tools + m4-early m4 skeleton time ielr cex all + ] + + def validate_trace(trace) + list = VALID_TRACES + h = {} + + trace.each do |t| + if list.include?(t) + h[t.to_sym] = true + else + raise "Invalid trace option \"#{t}\"." + end + end + + return h + end + end +end diff --git a/tool/lrama/lib/lrama/options.rb b/tool/lrama/lib/lrama/options.rb new file mode 100644 index 0000000000..739ca16f55 --- /dev/null +++ b/tool/lrama/lib/lrama/options.rb @@ -0,0 +1,24 @@ +module Lrama + # Command line options. + class Options + attr_accessor :skeleton, :header, :header_file, + :report_file, :outfile, + :error_recovery, :grammar_file, + :trace_opts, :report_opts, :y, + :debug + + def initialize + @skeleton = "bison/yacc.c" + @header = false + @header_file = nil + @report_file = nil + @outfile = "y.tab.c" + @error_recovery = false + @grammar_file = nil + @trace_opts = nil + @report_opts = nil + @y = STDIN + @debug = false + end + end +end diff --git a/tool/lrama/lib/lrama/output.rb b/tool/lrama/lib/lrama/output.rb new file mode 100644 index 0000000000..642c8b4708 --- /dev/null +++ b/tool/lrama/lib/lrama/output.rb @@ -0,0 +1,490 @@ +require "erb" +require "forwardable" +require "lrama/report/duration" + +module Lrama + class Output + extend Forwardable + include Report::Duration + + attr_reader :grammar_file_path, :context, :grammar, :error_recovery, :include_header + + def_delegators "@context", :yyfinal, :yylast, :yyntokens, :yynnts, :yynrules, :yynstates, + :yymaxutok, :yypact_ninf, :yytable_ninf + + def_delegators "@grammar", :eof_symbol, :error_symbol, :undef_symbol, :accept_symbol + + def initialize( + out:, output_file_path:, template_name:, grammar_file_path:, + context:, grammar:, header_out: nil, header_file_path: nil, error_recovery: false + ) + @out = out + @output_file_path = output_file_path + @template_name = template_name + @grammar_file_path = grammar_file_path + @header_out = header_out + @header_file_path = header_file_path + @context = context + @grammar = grammar + @error_recovery = error_recovery + @include_header = header_file_path ? header_file_path.sub("./", "") : nil + end + + if ERB.instance_method(:initialize).parameters.last.first == :key + def self.erb(input) + ERB.new(input, trim_mode: '-') + end + else + def self.erb(input) + ERB.new(input, nil, '-') + end + end + + def render_partial(file) + render_template(partial_file(file)) + end + + def render + report_duration(:render) do + tmp = eval_template(template_file, @output_file_path) + @out << tmp + + if @header_file_path + tmp = eval_template(header_template_file, @header_file_path) + + if @header_out + @header_out << tmp + else + File.write(@header_file_path, tmp) + end + end + end + end + + # A part of b4_token_enums + def token_enums + str = "" + + @context.yytokentype.each do |s_value, token_id, display_name| + s = sprintf("%s = %d%s", s_value, token_id, token_id == yymaxutok ? "" : ",") + + if display_name + str << sprintf(" %-30s /* %s */\n", s, display_name) + else + str << sprintf(" %s\n", s) + end + end + + str + end + + # b4_symbol_enum + def symbol_enum + str = "" + + last_sym_number = @context.yysymbol_kind_t.last[1] + @context.yysymbol_kind_t.each do |s_value, sym_number, display_name| + s = sprintf("%s = %d%s", s_value, sym_number, (sym_number == last_sym_number) ? "" : ",") + + if display_name + str << sprintf(" %-40s /* %s */\n", s, display_name) + else + str << sprintf(" %s\n", s) + end + end + + str + end + + def yytranslate + int_array_to_string(@context.yytranslate) + end + + def yytranslate_inverted + int_array_to_string(@context.yytranslate_inverted) + end + + def yyrline + int_array_to_string(@context.yyrline) + end + + def yytname + string_array_to_string(@context.yytname) + " YY_NULLPTR" + end + + # b4_int_type_for + def int_type_for(ary) + min = ary.min + max = ary.max + + case + when (-127 <= min && min <= 127) && (-127 <= max && max <= 127) + "yytype_int8" + when (0 <= min && min <= 255) && (0 <= max && max <= 255) + "yytype_uint8" + when (-32767 <= min && min <= 32767) && (-32767 <= max && max <= 32767) + "yytype_int16" + when (0 <= min && min <= 65535) && (0 <= max && max <= 65535) + "yytype_uint16" + else + "int" + end + end + + def symbol_actions_for_printer + str = "" + + @grammar.symbols.each do |sym| + next unless sym.printer + + str << <<-STR + case #{sym.enum_name}: /* #{sym.comment} */ +#line #{sym.printer.lineno} "#{@grammar_file_path}" + {#{sym.printer.translated_code(sym.tag)}} +#line [@oline@] [@ofile@] + break; + + STR + end + + str + end + + def symbol_actions_for_destructor + str = "" + + @grammar.symbols.each do |sym| + next unless sym.destructor + + str << <<-STR + case #{sym.enum_name}: /* #{sym.comment} */ +#line #{sym.destructor.lineno} "#{@grammar_file_path}" + {#{sym.destructor.translated_code(sym.tag)}} +#line [@oline@] [@ofile@] + break; + + STR + end + + str + end + + # b4_user_initial_action + def user_initial_action(comment = "") + return "" unless @grammar.initial_action + + <<-STR + #{comment} +#line #{@grammar.initial_action.line} "#{@grammar_file_path}" + {#{@grammar.initial_action.translated_code}} + STR + end + + def after_shift_function(comment = "") + return "" unless @grammar.after_shift + + <<-STR + #{comment} +#line #{@grammar.after_shift.line} "#{@grammar_file_path}" + {#{@grammar.after_shift.s_value}(#{parse_param_name});} +#line [@oline@] [@ofile@] + STR + end + + def before_reduce_function(comment = "") + return "" unless @grammar.before_reduce + + <<-STR + #{comment} +#line #{@grammar.before_reduce.line} "#{@grammar_file_path}" + {#{@grammar.before_reduce.s_value}(yylen#{user_args});} +#line [@oline@] [@ofile@] + STR + end + + def after_reduce_function(comment = "") + return "" unless @grammar.after_reduce + + <<-STR + #{comment} +#line #{@grammar.after_reduce.line} "#{@grammar_file_path}" + {#{@grammar.after_reduce.s_value}(yylen#{user_args});} +#line [@oline@] [@ofile@] + STR + end + + def after_shift_error_token_function(comment = "") + return "" unless @grammar.after_shift_error_token + + <<-STR + #{comment} +#line #{@grammar.after_shift_error_token.line} "#{@grammar_file_path}" + {#{@grammar.after_shift_error_token.s_value}(#{parse_param_name});} +#line [@oline@] [@ofile@] + STR + end + + def after_pop_stack_function(len, comment = "") + return "" unless @grammar.after_pop_stack + + <<-STR + #{comment} +#line #{@grammar.after_pop_stack.line} "#{@grammar_file_path}" + {#{@grammar.after_pop_stack.s_value}(#{len}#{user_args});} +#line [@oline@] [@ofile@] + STR + end + + def symbol_actions_for_error_token + str = "" + + @grammar.symbols.each do |sym| + next unless sym.error_token + + str << <<-STR + case #{sym.enum_name}: /* #{sym.comment} */ +#line #{sym.error_token.lineno} "#{@grammar_file_path}" + {#{sym.error_token.translated_code(sym.tag)}} +#line [@oline@] [@ofile@] + break; + + STR + end + + str + end + + # b4_user_actions + def user_actions + str = "" + + @context.states.rules.each do |rule| + next unless rule.token_code + + code = rule.token_code + spaces = " " * (code.column - 1) + + str << <<-STR + case #{rule.id + 1}: /* #{rule.as_comment} */ +#line #{code.line} "#{@grammar_file_path}" +#{spaces}{#{rule.translated_code}} +#line [@oline@] [@ofile@] + break; + + STR + end + + str << <<-STR + +#line [@oline@] [@ofile@] + STR + + str + end + + def omit_blanks(param) + param.strip + end + + # b4_parse_param + def parse_param + if @grammar.parse_param + omit_blanks(@grammar.parse_param) + else + "" + end + end + + def lex_param + if @grammar.lex_param + omit_blanks(@grammar.lex_param) + else + "" + end + end + + # b4_user_formals + def user_formals + if @grammar.parse_param + ", #{parse_param}" + else + "" + end + end + + # b4_user_args + def user_args + if @grammar.parse_param + ", #{parse_param_name}" + else + "" + end + end + + def extract_param_name(param) + param[/\b([a-zA-Z0-9_]+)(?=\s*\z)/] + end + + def parse_param_name + if @grammar.parse_param + extract_param_name(parse_param) + else + "" + end + end + + def lex_param_name + if @grammar.lex_param + extract_param_name(lex_param) + else + "" + end + end + + # b4_parse_param_use + def parse_param_use(val, loc) + str = <<-STR + YY_USE (#{val}); + YY_USE (#{loc}); + STR + + if @grammar.parse_param + str << " YY_USE (#{parse_param_name});" + end + + str + end + + # b4_yylex_formals + def yylex_formals + ary = ["&yylval", "&yylloc"] + + if @grammar.lex_param + ary << lex_param_name + end + + "(#{ary.join(', ')})" + end + + # b4_table_value_equals + def table_value_equals(table, value, literal, symbol) + if literal < table.min || table.max < literal + "0" + else + "((#{value}) == #{symbol})" + end + end + + # b4_yyerror_args + def yyerror_args + ary = ["&yylloc"] + + if @grammar.parse_param + ary << parse_param_name + end + + "#{ary.join(', ')}" + end + + def template_basename + File.basename(template_file) + end + + def aux + @grammar.aux + end + + def int_array_to_string(ary) + last = ary.count - 1 + + s = ary.each_with_index.each_slice(10).map do |slice| + str = " " + + slice.each do |e, i| + str << sprintf("%6d%s", e, (i == last) ? "" : ",") + end + + str + end + + s.join("\n") + end + + def spec_mapped_header_file + @header_file_path + end + + def b4_cpp_guard__b4_spec_mapped_header_file + if @header_file_path + "YY_YY_" + @header_file_path.gsub(/[^a-zA-Z_0-9]+/, "_").upcase + "_INCLUDED" + else + "" + end + end + + # b4_percent_code_get + def percent_code(name) + @grammar.percent_codes.select do |percent_code| + percent_code.name == name + end.map do |percent_code| + percent_code.code + end.join + end + + private + + def eval_template(file, path) + tmp = render_template(file) + replace_special_variables(tmp, path) + end + + def render_template(file) + erb = self.class.erb(File.read(file)) + erb.filename = file + erb.result_with_hash(context: @context, output: self) + end + + def template_file + File.join(template_dir, @template_name) + end + + def header_template_file + File.join(template_dir, "bison/yacc.h") + end + + def partial_file(file) + File.join(template_dir, file) + end + + def template_dir + File.expand_path("../../../template", __FILE__) + end + + def string_array_to_string(ary) + str = "" + tmp = " " + + ary.each do |s| + s = s.gsub('\\', '\\\\\\\\') + s = s.gsub('"', '\\"') + + if (tmp + s + " \"\",").length > 75 + str << tmp << "\n" + tmp = " \"#{s}\"," + else + tmp << " \"#{s}\"," + end + end + + str << tmp + end + + def replace_special_variables(str, ofile) + str.each_line.with_index(1).map do |line, i| + line.gsub!("[@oline@]", (i + 1).to_s) + line.gsub!("[@ofile@]", "\"#{ofile}\"") + line + end.join + end + end +end diff --git a/tool/lrama/lib/lrama/parser.rb b/tool/lrama/lib/lrama/parser.rb new file mode 100644 index 0000000000..0a46f759c0 --- /dev/null +++ b/tool/lrama/lib/lrama/parser.rb @@ -0,0 +1,2212 @@ +# +# DO NOT MODIFY!!!! +# This file is automatically generated by Racc 1.7.3 +# from Racc grammar file "parser.y". +# + +###### racc/parser.rb begin +unless $".find {|p| p.end_with?('/racc/parser.rb')} +$".push "#{__dir__}/racc/parser.rb" +self.class.module_eval(<<'...end racc/parser.rb/module_eval...', 'racc/parser.rb', 1) +#-- +# Copyright (c) 1999-2006 Minero Aoki +# +# This program is free software. +# You can distribute/modify this program under the same terms of ruby. +# +# As a special exception, when this code is copied by Racc +# into a Racc output file, you may use that output file +# without restriction. +#++ + +unless $".find {|p| p.end_with?('/racc/info.rb')} +$".push "#{__dir__}/racc/info.rb" + +module Racc + VERSION = '1.7.3' + Version = VERSION + Copyright = 'Copyright (c) 1999-2006 Minero Aoki' +end + +end + + +unless defined?(NotImplementedError) + NotImplementedError = NotImplementError # :nodoc: +end + +module Racc + class ParseError < StandardError; end +end +unless defined?(::ParseError) + ParseError = Racc::ParseError # :nodoc: +end + +# Racc is a LALR(1) parser generator. +# It is written in Ruby itself, and generates Ruby programs. +# +# == Command-line Reference +# +# racc [-o<var>filename</var>] [--output-file=<var>filename</var>] +# [-e<var>rubypath</var>] [--executable=<var>rubypath</var>] +# [-v] [--verbose] +# [-O<var>filename</var>] [--log-file=<var>filename</var>] +# [-g] [--debug] +# [-E] [--embedded] +# [-l] [--no-line-convert] +# [-c] [--line-convert-all] +# [-a] [--no-omit-actions] +# [-C] [--check-only] +# [-S] [--output-status] +# [--version] [--copyright] [--help] <var>grammarfile</var> +# +# [+grammarfile+] +# Racc grammar file. Any extension is permitted. +# [-o+outfile+, --output-file=+outfile+] +# A filename for output. default is <+filename+>.tab.rb +# [-O+filename+, --log-file=+filename+] +# Place logging output in file +filename+. +# Default log file name is <+filename+>.output. +# [-e+rubypath+, --executable=+rubypath+] +# output executable file(mode 755). where +path+ is the Ruby interpreter. +# [-v, --verbose] +# verbose mode. create +filename+.output file, like yacc's y.output file. +# [-g, --debug] +# add debug code to parser class. To display debugging information, +# use this '-g' option and set @yydebug true in parser class. +# [-E, --embedded] +# Output parser which doesn't need runtime files (racc/parser.rb). +# [-F, --frozen] +# Output parser which declares frozen_string_literals: true +# [-C, --check-only] +# Check syntax of racc grammar file and quit. +# [-S, --output-status] +# Print messages time to time while compiling. +# [-l, --no-line-convert] +# turns off line number converting. +# [-c, --line-convert-all] +# Convert line number of actions, inner, header and footer. +# [-a, --no-omit-actions] +# Call all actions, even if an action is empty. +# [--version] +# print Racc version and quit. +# [--copyright] +# Print copyright and quit. +# [--help] +# Print usage and quit. +# +# == Generating Parser Using Racc +# +# To compile Racc grammar file, simply type: +# +# $ racc parse.y +# +# This creates Ruby script file "parse.tab.y". The -o option can change the output filename. +# +# == Writing A Racc Grammar File +# +# If you want your own parser, you have to write a grammar file. +# A grammar file contains the name of your parser class, grammar for the parser, +# user code, and anything else. +# When writing a grammar file, yacc's knowledge is helpful. +# If you have not used yacc before, Racc is not too difficult. +# +# Here's an example Racc grammar file. +# +# class Calcparser +# rule +# target: exp { print val[0] } +# +# exp: exp '+' exp +# | exp '*' exp +# | '(' exp ')' +# | NUMBER +# end +# +# Racc grammar files resemble yacc files. +# But (of course), this is Ruby code. +# yacc's $$ is the 'result', $0, $1... is +# an array called 'val', and $-1, $-2... is an array called '_values'. +# +# See the {Grammar File Reference}[rdoc-ref:lib/racc/rdoc/grammar.en.rdoc] for +# more information on grammar files. +# +# == Parser +# +# Then you must prepare the parse entry method. There are two types of +# parse methods in Racc, Racc::Parser#do_parse and Racc::Parser#yyparse +# +# Racc::Parser#do_parse is simple. +# +# It's yyparse() of yacc, and Racc::Parser#next_token is yylex(). +# This method must returns an array like [TOKENSYMBOL, ITS_VALUE]. +# EOF is [false, false]. +# (TOKENSYMBOL is a Ruby symbol (taken from String#intern) by default. +# If you want to change this, see the grammar reference. +# +# Racc::Parser#yyparse is little complicated, but useful. +# It does not use Racc::Parser#next_token, instead it gets tokens from any iterator. +# +# For example, <code>yyparse(obj, :scan)</code> causes +# calling +obj#scan+, and you can return tokens by yielding them from +obj#scan+. +# +# == Debugging +# +# When debugging, "-v" or/and the "-g" option is helpful. +# +# "-v" creates verbose log file (.output). +# "-g" creates a "Verbose Parser". +# Verbose Parser prints the internal status when parsing. +# But it's _not_ automatic. +# You must use -g option and set +@yydebug+ to +true+ in order to get output. +# -g option only creates the verbose parser. +# +# === Racc reported syntax error. +# +# Isn't there too many "end"? +# grammar of racc file is changed in v0.10. +# +# Racc does not use '%' mark, while yacc uses huge number of '%' marks.. +# +# === Racc reported "XXXX conflicts". +# +# Try "racc -v xxxx.y". +# It causes producing racc's internal log file, xxxx.output. +# +# === Generated parsers does not work correctly +# +# Try "racc -g xxxx.y". +# This command let racc generate "debugging parser". +# Then set @yydebug=true in your parser. +# It produces a working log of your parser. +# +# == Re-distributing Racc runtime +# +# A parser, which is created by Racc, requires the Racc runtime module; +# racc/parser.rb. +# +# Ruby 1.8.x comes with Racc runtime module, +# you need NOT distribute Racc runtime files. +# +# If you want to include the Racc runtime module with your parser. +# This can be done by using '-E' option: +# +# $ racc -E -omyparser.rb myparser.y +# +# This command creates myparser.rb which `includes' Racc runtime. +# Only you must do is to distribute your parser file (myparser.rb). +# +# Note: parser.rb is ruby license, but your parser is not. +# Your own parser is completely yours. +module Racc + + unless defined?(Racc_No_Extensions) + Racc_No_Extensions = false # :nodoc: + end + + class Parser + + Racc_Runtime_Version = ::Racc::VERSION + Racc_Runtime_Core_Version_R = ::Racc::VERSION + + begin + if Object.const_defined?(:RUBY_ENGINE) and RUBY_ENGINE == 'jruby' + require 'jruby' + require 'racc/cparse-jruby.jar' + com.headius.racc.Cparse.new.load(JRuby.runtime, false) + else + require 'racc/cparse' + end + + unless new.respond_to?(:_racc_do_parse_c, true) + raise LoadError, 'old cparse.so' + end + if Racc_No_Extensions + raise LoadError, 'selecting ruby version of racc runtime core' + end + + Racc_Main_Parsing_Routine = :_racc_do_parse_c # :nodoc: + Racc_YY_Parse_Method = :_racc_yyparse_c # :nodoc: + Racc_Runtime_Core_Version = Racc_Runtime_Core_Version_C # :nodoc: + Racc_Runtime_Type = 'c' # :nodoc: + rescue LoadError + Racc_Main_Parsing_Routine = :_racc_do_parse_rb + Racc_YY_Parse_Method = :_racc_yyparse_rb + Racc_Runtime_Core_Version = Racc_Runtime_Core_Version_R + Racc_Runtime_Type = 'ruby' + end + + def Parser.racc_runtime_type # :nodoc: + Racc_Runtime_Type + end + + def _racc_setup + @yydebug = false unless self.class::Racc_debug_parser + @yydebug = false unless defined?(@yydebug) + if @yydebug + @racc_debug_out = $stderr unless defined?(@racc_debug_out) + @racc_debug_out ||= $stderr + end + arg = self.class::Racc_arg + arg[13] = true if arg.size < 14 + arg + end + + def _racc_init_sysvars + @racc_state = [0] + @racc_tstack = [] + @racc_vstack = [] + + @racc_t = nil + @racc_val = nil + + @racc_read_next = true + + @racc_user_yyerror = false + @racc_error_status = 0 + end + + # The entry point of the parser. This method is used with #next_token. + # If Racc wants to get token (and its value), calls next_token. + # + # Example: + # def parse + # @q = [[1,1], + # [2,2], + # [3,3], + # [false, '$']] + # do_parse + # end + # + # def next_token + # @q.shift + # end + class_eval <<~RUBY, __FILE__, __LINE__ + 1 + def do_parse + #{Racc_Main_Parsing_Routine}(_racc_setup(), false) + end + RUBY + + # The method to fetch next token. + # If you use #do_parse method, you must implement #next_token. + # + # The format of return value is [TOKEN_SYMBOL, VALUE]. + # +token-symbol+ is represented by Ruby's symbol by default, e.g. :IDENT + # for 'IDENT'. ";" (String) for ';'. + # + # The final symbol (End of file) must be false. + def next_token + raise NotImplementedError, "#{self.class}\#next_token is not defined" + end + + def _racc_do_parse_rb(arg, in_debug) + action_table, action_check, action_default, action_pointer, + _, _, _, _, + _, _, token_table, * = arg + + _racc_init_sysvars + tok = act = i = nil + + catch(:racc_end_parse) { + while true + if i = action_pointer[@racc_state[-1]] + if @racc_read_next + if @racc_t != 0 # not EOF + tok, @racc_val = next_token() + unless tok # EOF + @racc_t = 0 + else + @racc_t = (token_table[tok] or 1) # error token + end + racc_read_token(@racc_t, tok, @racc_val) if @yydebug + @racc_read_next = false + end + end + i += @racc_t + unless i >= 0 and + act = action_table[i] and + action_check[i] == @racc_state[-1] + act = action_default[@racc_state[-1]] + end + else + act = action_default[@racc_state[-1]] + end + while act = _racc_evalact(act, arg) + ; + end + end + } + end + + # Another entry point for the parser. + # If you use this method, you must implement RECEIVER#METHOD_ID method. + # + # RECEIVER#METHOD_ID is a method to get next token. + # It must 'yield' the token, which format is [TOKEN-SYMBOL, VALUE]. + class_eval <<~RUBY, __FILE__, __LINE__ + 1 + def yyparse(recv, mid) + #{Racc_YY_Parse_Method}(recv, mid, _racc_setup(), false) + end + RUBY + + def _racc_yyparse_rb(recv, mid, arg, c_debug) + action_table, action_check, action_default, action_pointer, + _, _, _, _, + _, _, token_table, * = arg + + _racc_init_sysvars + + catch(:racc_end_parse) { + until i = action_pointer[@racc_state[-1]] + while act = _racc_evalact(action_default[@racc_state[-1]], arg) + ; + end + end + recv.__send__(mid) do |tok, val| + unless tok + @racc_t = 0 + else + @racc_t = (token_table[tok] or 1) # error token + end + @racc_val = val + @racc_read_next = false + + i += @racc_t + unless i >= 0 and + act = action_table[i] and + action_check[i] == @racc_state[-1] + act = action_default[@racc_state[-1]] + end + while act = _racc_evalact(act, arg) + ; + end + + while !(i = action_pointer[@racc_state[-1]]) || + ! @racc_read_next || + @racc_t == 0 # $ + unless i and i += @racc_t and + i >= 0 and + act = action_table[i] and + action_check[i] == @racc_state[-1] + act = action_default[@racc_state[-1]] + end + while act = _racc_evalact(act, arg) + ; + end + end + end + } + end + + ### + ### common + ### + + def _racc_evalact(act, arg) + action_table, action_check, _, action_pointer, + _, _, _, _, + _, _, _, shift_n, + reduce_n, * = arg + nerr = 0 # tmp + + if act > 0 and act < shift_n + # + # shift + # + if @racc_error_status > 0 + @racc_error_status -= 1 unless @racc_t <= 1 # error token or EOF + end + @racc_vstack.push @racc_val + @racc_state.push act + @racc_read_next = true + if @yydebug + @racc_tstack.push @racc_t + racc_shift @racc_t, @racc_tstack, @racc_vstack + end + + elsif act < 0 and act > -reduce_n + # + # reduce + # + code = catch(:racc_jump) { + @racc_state.push _racc_do_reduce(arg, act) + false + } + if code + case code + when 1 # yyerror + @racc_user_yyerror = true # user_yyerror + return -reduce_n + when 2 # yyaccept + return shift_n + else + raise '[Racc Bug] unknown jump code' + end + end + + elsif act == shift_n + # + # accept + # + racc_accept if @yydebug + throw :racc_end_parse, @racc_vstack[0] + + elsif act == -reduce_n + # + # error + # + case @racc_error_status + when 0 + unless arg[21] # user_yyerror + nerr += 1 + on_error @racc_t, @racc_val, @racc_vstack + end + when 3 + if @racc_t == 0 # is $ + # We're at EOF, and another error occurred immediately after + # attempting auto-recovery + throw :racc_end_parse, nil + end + @racc_read_next = true + end + @racc_user_yyerror = false + @racc_error_status = 3 + while true + if i = action_pointer[@racc_state[-1]] + i += 1 # error token + if i >= 0 and + (act = action_table[i]) and + action_check[i] == @racc_state[-1] + break + end + end + throw :racc_end_parse, nil if @racc_state.size <= 1 + @racc_state.pop + @racc_vstack.pop + if @yydebug + @racc_tstack.pop + racc_e_pop @racc_state, @racc_tstack, @racc_vstack + end + end + return act + + else + raise "[Racc Bug] unknown action #{act.inspect}" + end + + racc_next_state(@racc_state[-1], @racc_state) if @yydebug + + nil + end + + def _racc_do_reduce(arg, act) + _, _, _, _, + goto_table, goto_check, goto_default, goto_pointer, + nt_base, reduce_table, _, _, + _, use_result, * = arg + + state = @racc_state + vstack = @racc_vstack + tstack = @racc_tstack + + i = act * -3 + len = reduce_table[i] + reduce_to = reduce_table[i+1] + method_id = reduce_table[i+2] + void_array = [] + + tmp_t = tstack[-len, len] if @yydebug + tmp_v = vstack[-len, len] + tstack[-len, len] = void_array if @yydebug + vstack[-len, len] = void_array + state[-len, len] = void_array + + # tstack must be updated AFTER method call + if use_result + vstack.push __send__(method_id, tmp_v, vstack, tmp_v[0]) + else + vstack.push __send__(method_id, tmp_v, vstack) + end + tstack.push reduce_to + + racc_reduce(tmp_t, reduce_to, tstack, vstack) if @yydebug + + k1 = reduce_to - nt_base + if i = goto_pointer[k1] + i += state[-1] + if i >= 0 and (curstate = goto_table[i]) and goto_check[i] == k1 + return curstate + end + end + goto_default[k1] + end + + # This method is called when a parse error is found. + # + # ERROR_TOKEN_ID is an internal ID of token which caused error. + # You can get string representation of this ID by calling + # #token_to_str. + # + # ERROR_VALUE is a value of error token. + # + # value_stack is a stack of symbol values. + # DO NOT MODIFY this object. + # + # This method raises ParseError by default. + # + # If this method returns, parsers enter "error recovering mode". + def on_error(t, val, vstack) + raise ParseError, sprintf("parse error on value %s (%s)", + val.inspect, token_to_str(t) || '?') + end + + # Enter error recovering mode. + # This method does not call #on_error. + def yyerror + throw :racc_jump, 1 + end + + # Exit parser. + # Return value is +Symbol_Value_Stack[0]+. + def yyaccept + throw :racc_jump, 2 + end + + # Leave error recovering mode. + def yyerrok + @racc_error_status = 0 + end + + # For debugging output + def racc_read_token(t, tok, val) + @racc_debug_out.print 'read ' + @racc_debug_out.print tok.inspect, '(', racc_token2str(t), ') ' + @racc_debug_out.puts val.inspect + @racc_debug_out.puts + end + + def racc_shift(tok, tstack, vstack) + @racc_debug_out.puts "shift #{racc_token2str tok}" + racc_print_stacks tstack, vstack + @racc_debug_out.puts + end + + def racc_reduce(toks, sim, tstack, vstack) + out = @racc_debug_out + out.print 'reduce ' + if toks.empty? + out.print ' <none>' + else + toks.each {|t| out.print ' ', racc_token2str(t) } + end + out.puts " --> #{racc_token2str(sim)}" + racc_print_stacks tstack, vstack + @racc_debug_out.puts + end + + def racc_accept + @racc_debug_out.puts 'accept' + @racc_debug_out.puts + end + + def racc_e_pop(state, tstack, vstack) + @racc_debug_out.puts 'error recovering mode: pop token' + racc_print_states state + racc_print_stacks tstack, vstack + @racc_debug_out.puts + end + + def racc_next_state(curstate, state) + @racc_debug_out.puts "goto #{curstate}" + racc_print_states state + @racc_debug_out.puts + end + + def racc_print_stacks(t, v) + out = @racc_debug_out + out.print ' [' + t.each_index do |i| + out.print ' (', racc_token2str(t[i]), ' ', v[i].inspect, ')' + end + out.puts ' ]' + end + + def racc_print_states(s) + out = @racc_debug_out + out.print ' [' + s.each {|st| out.print ' ', st } + out.puts ' ]' + end + + def racc_token2str(tok) + self.class::Racc_token_to_s_table[tok] or + raise "[Racc Bug] can't convert token #{tok} to string" + end + + # Convert internal ID of token symbol to the string. + def token_to_str(t) + self.class::Racc_token_to_s_table[t] + end + + end + +end + +...end racc/parser.rb/module_eval... +end +###### racc/parser.rb end +module Lrama + class Parser < Racc::Parser + +module_eval(<<'...end parser.y/module_eval...', 'parser.y', 529) + +include Lrama::Report::Duration + +def initialize(text, path, debug = false) + @grammar_file = Lrama::Lexer::GrammarFile.new(path, text) + @yydebug = debug + @rule_counter = Lrama::Grammar::Counter.new(0) + @midrule_action_counter = Lrama::Grammar::Counter.new(1) +end + +def parse + report_duration(:parse) do + @lexer = Lrama::Lexer.new(@grammar_file) + @grammar = Lrama::Grammar.new(@rule_counter) + @precedence_number = 0 + reset_precs + do_parse + @grammar + end +end + +def next_token + @lexer.next_token +end + +def on_error(error_token_id, error_value, value_stack) + if error_value.is_a?(Lrama::Lexer::Token) + location = error_value.location + value = "'#{error_value.s_value}'" + else + location = @lexer.location + value = error_value.inspect + end + + error_message = "parse error on value #{value} (#{token_to_str(error_token_id) || '?'})" + + raise_parse_error(error_message, location) +end + +def on_action_error(error_message, error_value) + if error_value.is_a?(Lrama::Lexer::Token) + location = error_value.location + else + location = @lexer.location + end + + raise_parse_error(error_message, location) +end + +private + +def reset_precs + @prec_seen = false + @code_after_prec = false +end + +def begin_c_declaration(end_symbol) + @lexer.status = :c_declaration + @lexer.end_symbol = end_symbol +end + +def end_c_declaration + @lexer.status = :initial + @lexer.end_symbol = nil +end + +def raise_parse_error(error_message, location) + raise ParseError, location.generate_error_message(error_message) +end +...end parser.y/module_eval... +##### State transition tables begin ### + +racc_action_table = [ + 96, 50, 97, 156, 155, 78, 50, 50, 156, 199, + 78, 78, 50, 50, 199, 49, 78, 158, 69, 6, + 3, 7, 158, 200, 210, 154, 8, 50, 200, 49, + 40, 174, 175, 176, 47, 50, 46, 49, 53, 78, + 74, 50, 53, 49, 159, 53, 81, 98, 56, 159, + 201, 174, 175, 176, 94, 201, 22, 24, 25, 26, + 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, + 37, 46, 50, 50, 49, 49, 91, 81, 81, 50, + 50, 49, 49, 50, 81, 49, 57, 78, 184, 58, + 59, 22, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 9, 50, 60, 49, + 13, 14, 15, 16, 17, 18, 61, 62, 19, 20, + 21, 22, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 50, 50, 49, + 49, 78, 184, 50, 50, 49, 49, 78, 184, 50, + 50, 49, 49, 78, 184, 50, 50, 49, 49, 78, + 184, 50, 50, 49, 49, 78, 184, 50, 50, 49, + 49, 78, 78, 50, 50, 49, 49, 78, 78, 50, + 50, 49, 49, 78, 78, 50, 50, 190, 49, 78, + 78, 50, 50, 190, 49, 78, 78, 50, 50, 190, + 49, 78, 50, 50, 49, 49, 152, 203, 153, 204, + 174, 175, 176, 219, 221, 204, 204, 63, 64, 65, + 66, 87, 88, 92, 94, 99, 99, 99, 101, 107, + 111, 112, 115, 115, 115, 115, 118, 121, 122, 124, + 126, 127, 128, 129, 130, 133, 137, 138, 139, 142, + 143, 144, 146, 161, 163, 164, 165, 166, 167, 168, + 169, 142, 171, 179, 180, 189, 194, 195, 197, 202, + 189, 94, 194, 216, 218, 94, 194, 224, 94 ] + +racc_action_check = [ + 48, 141, 48, 141, 140, 141, 170, 188, 170, 188, + 170, 188, 207, 32, 207, 32, 207, 141, 32, 2, + 1, 2, 170, 188, 199, 140, 3, 14, 207, 14, + 7, 199, 199, 199, 13, 33, 9, 33, 15, 33, + 33, 34, 16, 34, 141, 17, 34, 48, 18, 170, + 188, 157, 157, 157, 157, 207, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 41, 35, 36, 35, 36, 41, 35, 36, 37, + 68, 37, 68, 165, 37, 165, 19, 165, 165, 22, + 24, 41, 41, 41, 41, 41, 41, 41, 41, 41, + 41, 41, 41, 41, 41, 41, 4, 69, 25, 69, + 4, 4, 4, 4, 4, 4, 26, 27, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 166, 80, 166, + 80, 166, 166, 167, 81, 167, 81, 167, 167, 181, + 107, 181, 107, 181, 181, 185, 109, 185, 109, 185, + 185, 186, 115, 186, 115, 186, 186, 73, 74, 73, + 74, 73, 74, 112, 114, 112, 114, 112, 114, 134, + 159, 134, 159, 134, 159, 171, 201, 171, 201, 171, + 201, 202, 204, 202, 204, 202, 204, 210, 117, 210, + 117, 210, 131, 135, 131, 135, 136, 191, 136, 191, + 192, 192, 192, 213, 217, 213, 217, 28, 29, 30, + 31, 38, 39, 44, 45, 52, 54, 55, 56, 67, + 71, 72, 79, 84, 85, 86, 87, 93, 94, 100, + 102, 103, 104, 105, 106, 110, 118, 119, 120, 121, + 122, 123, 125, 145, 147, 148, 149, 150, 151, 152, + 153, 154, 156, 160, 162, 168, 173, 177, 187, 190, + 197, 198, 203, 206, 211, 216, 220, 222, 224 ] + +racc_action_pointer = [ + nil, 20, 9, 26, 97, nil, nil, 23, nil, 32, + nil, nil, nil, 28, 24, 19, 23, 26, 43, 67, + nil, nil, 70, nil, 71, 89, 97, 112, 212, 213, + 214, 215, 10, 32, 38, 69, 70, 76, 216, 220, + nil, 67, nil, nil, 200, 174, nil, nil, -5, nil, + nil, nil, 206, nil, 207, 208, 209, nil, nil, nil, + nil, nil, nil, nil, nil, nil, nil, 221, 77, 104, + nil, 224, 223, 164, 165, nil, nil, nil, nil, 224, + 135, 141, nil, nil, 225, 226, 227, 196, nil, nil, + nil, nil, nil, 195, 233, nil, nil, nil, nil, nil, + 237, nil, 238, 239, 240, 241, 242, 147, nil, 153, + 238, nil, 170, nil, 171, 159, nil, 195, 241, 236, + 246, 204, 199, 249, nil, 250, nil, nil, nil, nil, + nil, 199, nil, nil, 176, 200, 165, nil, nil, nil, + -19, -2, nil, nil, nil, 233, nil, 234, 235, 236, + 237, 238, 217, 255, 216, nil, 222, 4, nil, 177, + 243, nil, 244, nil, nil, 80, 134, 140, 220, nil, + 3, 182, nil, 258, nil, nil, nil, 265, nil, nil, + nil, 146, nil, nil, nil, 152, 158, 224, 4, nil, + 229, 166, 163, nil, nil, nil, nil, 225, 221, -16, + nil, 183, 188, 264, 189, nil, 253, 9, nil, nil, + 194, 272, nil, 172, nil, nil, 225, 173, nil, nil, + 268, nil, 257, nil, 228, nil ] + +racc_action_default = [ + -2, -136, -8, -136, -136, -3, -4, -136, 226, -136, + -9, -10, -11, -136, -136, -136, -136, -136, -136, -136, + -23, -24, -136, -28, -136, -136, -136, -136, -136, -136, + -136, -136, -136, -136, -136, -136, -136, -136, -136, -136, + -7, -121, -94, -96, -136, -118, -120, -12, -125, -92, + -93, -124, -14, -83, -15, -16, -136, -20, -25, -29, + -32, -35, -38, -39, -40, -41, -42, -43, -49, -136, + -52, -69, -44, -73, -136, -76, -78, -79, -133, -45, + -86, -136, -89, -91, -46, -47, -48, -136, -5, -1, + -95, -122, -97, -136, -136, -13, -126, -127, -128, -80, + -136, -17, -136, -136, -136, -136, -136, -136, -53, -50, + -71, -70, -136, -77, -74, -136, -90, -87, -136, -136, + -136, -102, -136, -136, -84, -136, -21, -26, -30, -33, + -36, -51, -54, -72, -75, -88, -136, -56, -6, -123, + -98, -99, -103, -119, -81, -136, -18, -136, -136, -136, + -136, -136, -136, -136, -102, -101, -92, -118, -107, -136, + -136, -85, -136, -22, -27, -136, -136, -136, -60, -57, + -100, -136, -104, -134, -111, -112, -113, -136, -110, -82, + -19, -31, -129, -131, -132, -34, -37, -55, -58, -61, + -92, -136, -114, -105, -135, -108, -130, -60, -118, -92, + -65, -136, -136, -134, -136, -116, -136, -59, -62, -63, + -136, -136, -68, -136, -106, -115, -118, -136, -66, -117, + -134, -64, -136, -109, -118, -67 ] + +racc_goto_table = [ + 93, 75, 51, 68, 73, 193, 116, 108, 191, 173, + 196, 1, 117, 2, 196, 196, 141, 4, 42, 41, + 71, 89, 83, 83, 83, 83, 188, 79, 84, 85, + 86, 52, 54, 55, 5, 214, 181, 185, 186, 213, + 109, 113, 75, 116, 205, 114, 135, 217, 108, 170, + 90, 209, 223, 39, 119, 207, 71, 71, 10, 11, + 12, 116, 48, 95, 125, 162, 102, 147, 83, 83, + 108, 103, 148, 104, 149, 105, 150, 106, 131, 151, + 75, 67, 113, 134, 72, 110, 132, 136, 187, 211, + 222, 123, 160, 100, 145, 71, 140, 71, 177, 206, + 120, nil, 113, 83, nil, 83, nil, nil, nil, 157, + nil, nil, 172, nil, nil, nil, nil, nil, nil, 71, + nil, nil, nil, 83, nil, nil, nil, 178, nil, nil, + nil, nil, nil, nil, nil, nil, nil, nil, 157, 192, + nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, 208, nil, nil, 198, nil, nil, nil, + nil, nil, nil, nil, nil, nil, nil, nil, nil, 212, + 192, 220, 215, nil, nil, 198, nil, nil, 192, 225 ] + +racc_goto_check = [ + 41, 40, 34, 32, 46, 59, 53, 33, 43, 42, + 63, 1, 52, 2, 63, 63, 58, 3, 54, 4, + 34, 5, 34, 34, 34, 34, 39, 31, 31, 31, + 31, 14, 14, 14, 6, 59, 20, 20, 20, 43, + 32, 40, 40, 53, 42, 46, 52, 43, 33, 58, + 54, 42, 59, 7, 8, 39, 34, 34, 9, 10, + 11, 53, 12, 13, 15, 16, 17, 18, 34, 34, + 33, 21, 22, 23, 24, 25, 26, 27, 32, 28, + 40, 29, 40, 46, 30, 35, 36, 37, 38, 44, + 45, 48, 49, 50, 51, 34, 57, 34, 60, 61, + 62, nil, 40, 34, nil, 34, nil, nil, nil, 40, + nil, nil, 41, nil, nil, nil, nil, nil, nil, 34, + nil, nil, nil, 34, nil, nil, nil, 40, nil, nil, + nil, nil, nil, nil, nil, nil, nil, nil, 40, 40, + nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, 41, nil, nil, 40, nil, nil, nil, + nil, nil, nil, nil, nil, nil, nil, nil, nil, 40, + 40, 41, 40, nil, nil, 40, nil, nil, 40, 41 ] + +racc_goto_pointer = [ + nil, 11, 13, 15, 10, -20, 32, 47, -34, 54, + 55, 56, 48, 15, 16, -37, -81, 9, -59, nil, + -129, 13, -55, 14, -54, 15, -53, 16, -51, 49, + 51, -7, -29, -61, -12, 14, -24, -31, -80, -142, + -32, -45, -148, -163, -111, -128, -29, nil, -8, -52, + 40, -30, -69, -74, 9, nil, nil, -25, -105, -168, + -60, -96, 9, -171 ] + +racc_goto_default = [ + nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + 44, nil, nil, nil, nil, nil, nil, nil, nil, 23, + nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, 70, 76, nil, nil, nil, nil, nil, + 183, nil, nil, nil, nil, nil, nil, 77, nil, nil, + nil, nil, 80, 82, nil, 43, 45, nil, nil, nil, + nil, nil, nil, 182 ] + +racc_reduce_table = [ + 0, 0, :racc_error, + 5, 54, :_reduce_none, + 0, 55, :_reduce_none, + 2, 55, :_reduce_none, + 0, 60, :_reduce_4, + 0, 61, :_reduce_5, + 5, 59, :_reduce_6, + 2, 59, :_reduce_none, + 0, 56, :_reduce_8, + 2, 56, :_reduce_none, + 1, 62, :_reduce_none, + 1, 62, :_reduce_none, + 2, 62, :_reduce_12, + 3, 62, :_reduce_none, + 2, 62, :_reduce_none, + 2, 62, :_reduce_15, + 2, 62, :_reduce_16, + 0, 68, :_reduce_17, + 0, 69, :_reduce_18, + 7, 62, :_reduce_19, + 0, 70, :_reduce_20, + 0, 71, :_reduce_21, + 6, 62, :_reduce_22, + 1, 62, :_reduce_23, + 1, 62, :_reduce_none, + 0, 74, :_reduce_25, + 0, 75, :_reduce_26, + 6, 63, :_reduce_27, + 1, 63, :_reduce_none, + 0, 76, :_reduce_29, + 0, 77, :_reduce_30, + 7, 63, :_reduce_31, + 0, 78, :_reduce_32, + 0, 79, :_reduce_33, + 7, 63, :_reduce_34, + 0, 80, :_reduce_35, + 0, 81, :_reduce_36, + 7, 63, :_reduce_37, + 2, 63, :_reduce_38, + 2, 63, :_reduce_39, + 2, 63, :_reduce_40, + 2, 63, :_reduce_41, + 2, 63, :_reduce_42, + 2, 72, :_reduce_none, + 2, 72, :_reduce_44, + 2, 72, :_reduce_45, + 2, 72, :_reduce_46, + 2, 72, :_reduce_47, + 2, 72, :_reduce_48, + 1, 82, :_reduce_49, + 2, 82, :_reduce_50, + 3, 82, :_reduce_51, + 1, 85, :_reduce_52, + 2, 85, :_reduce_53, + 3, 86, :_reduce_54, + 7, 64, :_reduce_55, + 1, 90, :_reduce_56, + 3, 90, :_reduce_57, + 1, 91, :_reduce_58, + 3, 91, :_reduce_59, + 0, 92, :_reduce_60, + 1, 92, :_reduce_61, + 3, 92, :_reduce_62, + 3, 92, :_reduce_63, + 5, 92, :_reduce_64, + 0, 97, :_reduce_65, + 0, 98, :_reduce_66, + 7, 92, :_reduce_67, + 3, 92, :_reduce_68, + 0, 88, :_reduce_none, + 1, 88, :_reduce_none, + 0, 89, :_reduce_none, + 1, 89, :_reduce_none, + 1, 83, :_reduce_73, + 2, 83, :_reduce_74, + 3, 83, :_reduce_75, + 1, 99, :_reduce_76, + 2, 99, :_reduce_77, + 1, 93, :_reduce_none, + 1, 93, :_reduce_none, + 0, 101, :_reduce_80, + 0, 102, :_reduce_81, + 6, 67, :_reduce_82, + 0, 103, :_reduce_83, + 0, 104, :_reduce_84, + 5, 67, :_reduce_85, + 1, 84, :_reduce_86, + 2, 84, :_reduce_87, + 3, 84, :_reduce_88, + 1, 105, :_reduce_89, + 2, 105, :_reduce_90, + 1, 106, :_reduce_none, + 1, 87, :_reduce_92, + 1, 87, :_reduce_93, + 1, 57, :_reduce_none, + 2, 57, :_reduce_none, + 1, 107, :_reduce_none, + 2, 107, :_reduce_none, + 4, 108, :_reduce_98, + 1, 110, :_reduce_99, + 3, 110, :_reduce_100, + 2, 110, :_reduce_none, + 0, 111, :_reduce_102, + 1, 111, :_reduce_103, + 3, 111, :_reduce_104, + 4, 111, :_reduce_105, + 6, 111, :_reduce_106, + 0, 113, :_reduce_107, + 0, 114, :_reduce_108, + 8, 111, :_reduce_109, + 3, 111, :_reduce_110, + 1, 95, :_reduce_111, + 1, 95, :_reduce_112, + 1, 95, :_reduce_113, + 1, 96, :_reduce_114, + 3, 96, :_reduce_115, + 2, 96, :_reduce_116, + 4, 96, :_reduce_117, + 0, 94, :_reduce_none, + 3, 94, :_reduce_119, + 1, 109, :_reduce_none, + 0, 58, :_reduce_none, + 0, 115, :_reduce_122, + 3, 58, :_reduce_123, + 1, 65, :_reduce_none, + 0, 66, :_reduce_none, + 1, 66, :_reduce_none, + 1, 66, :_reduce_none, + 1, 66, :_reduce_none, + 1, 73, :_reduce_129, + 2, 73, :_reduce_130, + 1, 116, :_reduce_none, + 1, 116, :_reduce_none, + 1, 100, :_reduce_133, + 0, 112, :_reduce_none, + 1, 112, :_reduce_none ] + +racc_reduce_n = 136 + +racc_shift_n = 226 + +racc_token_table = { + false => 0, + :error => 1, + :C_DECLARATION => 2, + :CHARACTER => 3, + :IDENT_COLON => 4, + :IDENTIFIER => 5, + :INTEGER => 6, + :STRING => 7, + :TAG => 8, + "%%" => 9, + "%{" => 10, + "%}" => 11, + "%require" => 12, + "%expect" => 13, + "%define" => 14, + "%param" => 15, + "%lex-param" => 16, + "%parse-param" => 17, + "%code" => 18, + "{" => 19, + "}" => 20, + "%initial-action" => 21, + "%no-stdlib" => 22, + ";" => 23, + "%union" => 24, + "%destructor" => 25, + "%printer" => 26, + "%error-token" => 27, + "%after-shift" => 28, + "%before-reduce" => 29, + "%after-reduce" => 30, + "%after-shift-error-token" => 31, + "%after-pop-stack" => 32, + "%token" => 33, + "%type" => 34, + "%left" => 35, + "%right" => 36, + "%precedence" => 37, + "%nonassoc" => 38, + "%rule" => 39, + "(" => 40, + ")" => 41, + ":" => 42, + "," => 43, + "|" => 44, + "%empty" => 45, + "%prec" => 46, + "?" => 47, + "+" => 48, + "*" => 49, + "[" => 50, + "]" => 51, + "{...}" => 52 } + +racc_nt_base = 53 + +racc_use_result_var = true + +Racc_arg = [ + racc_action_table, + racc_action_check, + racc_action_default, + racc_action_pointer, + racc_goto_table, + racc_goto_check, + racc_goto_default, + racc_goto_pointer, + racc_nt_base, + racc_reduce_table, + racc_token_table, + racc_shift_n, + racc_reduce_n, + racc_use_result_var ] +Ractor.make_shareable(Racc_arg) if defined?(Ractor) + +Racc_token_to_s_table = [ + "$end", + "error", + "C_DECLARATION", + "CHARACTER", + "IDENT_COLON", + "IDENTIFIER", + "INTEGER", + "STRING", + "TAG", + "\"%%\"", + "\"%{\"", + "\"%}\"", + "\"%require\"", + "\"%expect\"", + "\"%define\"", + "\"%param\"", + "\"%lex-param\"", + "\"%parse-param\"", + "\"%code\"", + "\"{\"", + "\"}\"", + "\"%initial-action\"", + "\"%no-stdlib\"", + "\";\"", + "\"%union\"", + "\"%destructor\"", + "\"%printer\"", + "\"%error-token\"", + "\"%after-shift\"", + "\"%before-reduce\"", + "\"%after-reduce\"", + "\"%after-shift-error-token\"", + "\"%after-pop-stack\"", + "\"%token\"", + "\"%type\"", + "\"%left\"", + "\"%right\"", + "\"%precedence\"", + "\"%nonassoc\"", + "\"%rule\"", + "\"(\"", + "\")\"", + "\":\"", + "\",\"", + "\"|\"", + "\"%empty\"", + "\"%prec\"", + "\"?\"", + "\"+\"", + "\"*\"", + "\"[\"", + "\"]\"", + "\"{...}\"", + "$start", + "input", + "prologue_declarations", + "bison_declarations", + "grammar", + "epilogue_opt", + "prologue_declaration", + "@1", + "@2", + "bison_declaration", + "grammar_declaration", + "rule_declaration", + "variable", + "value", + "params", + "@3", + "@4", + "@5", + "@6", + "symbol_declaration", + "generic_symlist", + "@7", + "@8", + "@9", + "@10", + "@11", + "@12", + "@13", + "@14", + "token_declarations", + "symbol_declarations", + "token_declarations_for_precedence", + "token_declaration_list", + "token_declaration", + "id", + "int_opt", + "alias", + "rule_args", + "rule_rhs_list", + "rule_rhs", + "symbol", + "named_ref_opt", + "parameterizing_suffix", + "parameterizing_args", + "@15", + "@16", + "symbol_declaration_list", + "string_as_id", + "@17", + "@18", + "@19", + "@20", + "token_declaration_list_for_precedence", + "token_declaration_for_precedence", + "rules_or_grammar_declaration", + "rules", + "id_colon", + "rhs_list", + "rhs", + "tag_opt", + "@21", + "@22", + "@23", + "generic_symlist_item" ] +Ractor.make_shareable(Racc_token_to_s_table) if defined?(Ractor) + +Racc_debug_parser = true + +##### State transition tables end ##### + +# reduce 0 omitted + +# reduce 1 omitted + +# reduce 2 omitted + +# reduce 3 omitted + +module_eval(<<'.,.,', 'parser.y', 14) + def _reduce_4(val, _values, result) + begin_c_declaration("%}") + @grammar.prologue_first_lineno = @lexer.line + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 19) + def _reduce_5(val, _values, result) + end_c_declaration + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 23) + def _reduce_6(val, _values, result) + @grammar.prologue = val[2].s_value + + result + end +.,., + +# reduce 7 omitted + +module_eval(<<'.,.,', 'parser.y', 27) + def _reduce_8(val, _values, result) + result = "" + result + end +.,., + +# reduce 9 omitted + +# reduce 10 omitted + +# reduce 11 omitted + +module_eval(<<'.,.,', 'parser.y', 32) + def _reduce_12(val, _values, result) + @grammar.expect = val[1] + result + end +.,., + +# reduce 13 omitted + +# reduce 14 omitted + +module_eval(<<'.,.,', 'parser.y', 37) + def _reduce_15(val, _values, result) + val[1].each {|token| + @grammar.lex_param = Grammar::Code::NoReferenceCode.new(type: :lex_param, token_code: token).token_code.s_value + } + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 43) + def _reduce_16(val, _values, result) + val[1].each {|token| + @grammar.parse_param = Grammar::Code::NoReferenceCode.new(type: :parse_param, token_code: token).token_code.s_value + } + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 49) + def _reduce_17(val, _values, result) + begin_c_declaration("}") + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 53) + def _reduce_18(val, _values, result) + end_c_declaration + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 57) + def _reduce_19(val, _values, result) + @grammar.add_percent_code(id: val[1], code: val[4]) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 61) + def _reduce_20(val, _values, result) + begin_c_declaration("}") + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 65) + def _reduce_21(val, _values, result) + end_c_declaration + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 69) + def _reduce_22(val, _values, result) + @grammar.initial_action = Grammar::Code::InitialActionCode.new(type: :initial_action, token_code: val[3]) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 71) + def _reduce_23(val, _values, result) + @grammar.no_stdlib = true + result + end +.,., + +# reduce 24 omitted + +module_eval(<<'.,.,', 'parser.y', 76) + def _reduce_25(val, _values, result) + begin_c_declaration("}") + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 80) + def _reduce_26(val, _values, result) + end_c_declaration + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 84) + def _reduce_27(val, _values, result) + @grammar.set_union( + Grammar::Code::NoReferenceCode.new(type: :union, token_code: val[3]), + val[3].line + ) + + result + end +.,., + +# reduce 28 omitted + +module_eval(<<'.,.,', 'parser.y', 92) + def _reduce_29(val, _values, result) + begin_c_declaration("}") + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 96) + def _reduce_30(val, _values, result) + end_c_declaration + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 100) + def _reduce_31(val, _values, result) + @grammar.add_destructor( + ident_or_tags: val[6], + token_code: val[3], + lineno: val[3].line + ) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 108) + def _reduce_32(val, _values, result) + begin_c_declaration("}") + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 112) + def _reduce_33(val, _values, result) + end_c_declaration + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 116) + def _reduce_34(val, _values, result) + @grammar.add_printer( + ident_or_tags: val[6], + token_code: val[3], + lineno: val[3].line + ) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 124) + def _reduce_35(val, _values, result) + begin_c_declaration("}") + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 128) + def _reduce_36(val, _values, result) + end_c_declaration + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 132) + def _reduce_37(val, _values, result) + @grammar.add_error_token( + ident_or_tags: val[6], + token_code: val[3], + lineno: val[3].line + ) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 140) + def _reduce_38(val, _values, result) + @grammar.after_shift = val[1] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 144) + def _reduce_39(val, _values, result) + @grammar.before_reduce = val[1] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 148) + def _reduce_40(val, _values, result) + @grammar.after_reduce = val[1] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 152) + def _reduce_41(val, _values, result) + @grammar.after_shift_error_token = val[1] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 156) + def _reduce_42(val, _values, result) + @grammar.after_pop_stack = val[1] + + result + end +.,., + +# reduce 43 omitted + +module_eval(<<'.,.,', 'parser.y', 162) + def _reduce_44(val, _values, result) + val[1].each {|hash| + hash[:tokens].each {|id| + @grammar.add_type(id: id, tag: hash[:tag]) + } + } + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 170) + def _reduce_45(val, _values, result) + val[1].each {|hash| + hash[:tokens].each {|id| + sym = @grammar.add_term(id: id) + @grammar.add_left(sym, @precedence_number) + } + } + @precedence_number += 1 + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 180) + def _reduce_46(val, _values, result) + val[1].each {|hash| + hash[:tokens].each {|id| + sym = @grammar.add_term(id: id) + @grammar.add_right(sym, @precedence_number) + } + } + @precedence_number += 1 + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 190) + def _reduce_47(val, _values, result) + val[1].each {|hash| + hash[:tokens].each {|id| + sym = @grammar.add_term(id: id) + @grammar.add_precedence(sym, @precedence_number) + } + } + @precedence_number += 1 + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 200) + def _reduce_48(val, _values, result) + val[1].each {|hash| + hash[:tokens].each {|id| + sym = @grammar.add_term(id: id) + @grammar.add_nonassoc(sym, @precedence_number) + } + } + @precedence_number += 1 + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 211) + def _reduce_49(val, _values, result) + val[0].each {|token_declaration| + @grammar.add_term(id: token_declaration[0], alias_name: token_declaration[2], token_id: token_declaration[1], tag: nil, replace: true) + } + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 217) + def _reduce_50(val, _values, result) + val[1].each {|token_declaration| + @grammar.add_term(id: token_declaration[0], alias_name: token_declaration[2], token_id: token_declaration[1], tag: val[0], replace: true) + } + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 223) + def _reduce_51(val, _values, result) + val[2].each {|token_declaration| + @grammar.add_term(id: token_declaration[0], alias_name: token_declaration[2], token_id: token_declaration[1], tag: val[1], replace: true) + } + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 228) + def _reduce_52(val, _values, result) + result = [val[0]] + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 229) + def _reduce_53(val, _values, result) + result = val[0].append(val[1]) + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 231) + def _reduce_54(val, _values, result) + result = val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 235) + def _reduce_55(val, _values, result) + rule = Grammar::ParameterizingRule::Rule.new(val[1].s_value, val[3], val[6]) + @grammar.add_parameterizing_rule(rule) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 239) + def _reduce_56(val, _values, result) + result = [val[0]] + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 240) + def _reduce_57(val, _values, result) + result = val[0].append(val[2]) + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 244) + def _reduce_58(val, _values, result) + builder = val[0] + result = [builder] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 249) + def _reduce_59(val, _values, result) + builder = val[2] + result = val[0].append(builder) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 255) + def _reduce_60(val, _values, result) + reset_precs + result = Grammar::ParameterizingRule::Rhs.new + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 260) + def _reduce_61(val, _values, result) + reset_precs + result = Grammar::ParameterizingRule::Rhs.new + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 265) + def _reduce_62(val, _values, result) + token = val[1] + token.alias_name = val[2] + builder = val[0] + builder.symbols << token + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 273) + def _reduce_63(val, _values, result) + builder = val[0] + builder.symbols << Lrama::Lexer::Token::InstantiateRule.new(s_value: val[2], location: @lexer.location, args: [val[1]]) + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 279) + def _reduce_64(val, _values, result) + builder = val[0] + builder.symbols << Lrama::Lexer::Token::InstantiateRule.new(s_value: val[1].s_value, location: @lexer.location, args: val[3]) + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 285) + def _reduce_65(val, _values, result) + if @prec_seen + on_action_error("multiple User_code after %prec", val[0]) if @code_after_prec + @code_after_prec = true + end + begin_c_declaration("}") + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 293) + def _reduce_66(val, _values, result) + end_c_declaration + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 297) + def _reduce_67(val, _values, result) + user_code = val[3] + user_code.alias_name = val[6] + builder = val[0] + builder.user_code = user_code + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 305) + def _reduce_68(val, _values, result) + sym = @grammar.find_symbol_by_id!(val[2]) + @prec_seen = true + builder = val[0] + builder.precedence_sym = sym + result = builder + + result + end +.,., + +# reduce 69 omitted + +# reduce 70 omitted + +# reduce 71 omitted + +# reduce 72 omitted + +module_eval(<<'.,.,', 'parser.y', 320) + def _reduce_73(val, _values, result) + result = [{tag: nil, tokens: val[0]}] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 324) + def _reduce_74(val, _values, result) + result = [{tag: val[0], tokens: val[1]}] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 328) + def _reduce_75(val, _values, result) + result = val[0].append({tag: val[1], tokens: val[2]}) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 331) + def _reduce_76(val, _values, result) + result = [val[0]] + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 332) + def _reduce_77(val, _values, result) + result = val[0].append(val[1]) + result + end +.,., + +# reduce 78 omitted + +# reduce 79 omitted + +module_eval(<<'.,.,', 'parser.y', 339) + def _reduce_80(val, _values, result) + begin_c_declaration("}") + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 343) + def _reduce_81(val, _values, result) + end_c_declaration + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 347) + def _reduce_82(val, _values, result) + result = val[0].append(val[3]) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 351) + def _reduce_83(val, _values, result) + begin_c_declaration("}") + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 355) + def _reduce_84(val, _values, result) + end_c_declaration + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 359) + def _reduce_85(val, _values, result) + result = [val[2]] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 364) + def _reduce_86(val, _values, result) + result = [{tag: nil, tokens: val[0]}] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 368) + def _reduce_87(val, _values, result) + result = [{tag: val[0], tokens: val[1]}] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 372) + def _reduce_88(val, _values, result) + result = val[0].append({tag: val[1], tokens: val[2]}) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 375) + def _reduce_89(val, _values, result) + result = [val[0]] + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 376) + def _reduce_90(val, _values, result) + result = val[0].append(val[1]) + result + end +.,., + +# reduce 91 omitted + +module_eval(<<'.,.,', 'parser.y', 380) + def _reduce_92(val, _values, result) + on_action_error("ident after %prec", val[0]) if @prec_seen + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 381) + def _reduce_93(val, _values, result) + on_action_error("char after %prec", val[0]) if @prec_seen + result + end +.,., + +# reduce 94 omitted + +# reduce 95 omitted + +# reduce 96 omitted + +# reduce 97 omitted + +module_eval(<<'.,.,', 'parser.y', 391) + def _reduce_98(val, _values, result) + lhs = val[0] + lhs.alias_name = val[1] + val[3].each do |builder| + builder.lhs = lhs + builder.complete_input + @grammar.add_rule_builder(builder) + end + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 402) + def _reduce_99(val, _values, result) + builder = val[0] + if !builder.line + builder.line = @lexer.line - 1 + end + result = [builder] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 410) + def _reduce_100(val, _values, result) + builder = val[2] + if !builder.line + builder.line = @lexer.line - 1 + end + result = val[0].append(builder) + + result + end +.,., + +# reduce 101 omitted + +module_eval(<<'.,.,', 'parser.y', 420) + def _reduce_102(val, _values, result) + reset_precs + result = Grammar::RuleBuilder.new(@rule_counter, @midrule_action_counter) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 425) + def _reduce_103(val, _values, result) + reset_precs + result = Grammar::RuleBuilder.new(@rule_counter, @midrule_action_counter) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 430) + def _reduce_104(val, _values, result) + token = val[1] + token.alias_name = val[2] + builder = val[0] + builder.add_rhs(token) + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 438) + def _reduce_105(val, _values, result) + token = Lrama::Lexer::Token::InstantiateRule.new(s_value: val[2], location: @lexer.location, args: [val[1]], lhs_tag: val[3]) + builder = val[0] + builder.add_rhs(token) + builder.line = val[1].first_line + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 446) + def _reduce_106(val, _values, result) + token = Lrama::Lexer::Token::InstantiateRule.new(s_value: val[1].s_value, location: @lexer.location, args: val[3], lhs_tag: val[5]) + builder = val[0] + builder.add_rhs(token) + builder.line = val[1].first_line + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 454) + def _reduce_107(val, _values, result) + if @prec_seen + on_action_error("multiple User_code after %prec", val[0]) if @code_after_prec + @code_after_prec = true + end + begin_c_declaration("}") + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 462) + def _reduce_108(val, _values, result) + end_c_declaration + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 466) + def _reduce_109(val, _values, result) + user_code = val[3] + user_code.alias_name = val[6] + user_code.tag = val[7] + builder = val[0] + builder.user_code = user_code + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 475) + def _reduce_110(val, _values, result) + sym = @grammar.find_symbol_by_id!(val[2]) + @prec_seen = true + builder = val[0] + builder.precedence_sym = sym + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 482) + def _reduce_111(val, _values, result) + result = "option" + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 483) + def _reduce_112(val, _values, result) + result = "nonempty_list" + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 484) + def _reduce_113(val, _values, result) + result = "list" + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 486) + def _reduce_114(val, _values, result) + result = [val[0]] + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 487) + def _reduce_115(val, _values, result) + result = val[0].append(val[2]) + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 488) + def _reduce_116(val, _values, result) + result = [Lrama::Lexer::Token::InstantiateRule.new(s_value: val[1].s_value, location: @lexer.location, args: val[0])] + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 489) + def _reduce_117(val, _values, result) + result = [Lrama::Lexer::Token::InstantiateRule.new(s_value: val[0].s_value, location: @lexer.location, args: val[2])] + result + end +.,., + +# reduce 118 omitted + +module_eval(<<'.,.,', 'parser.y', 492) + def _reduce_119(val, _values, result) + result = val[1].s_value + result + end +.,., + +# reduce 120 omitted + +# reduce 121 omitted + +module_eval(<<'.,.,', 'parser.y', 499) + def _reduce_122(val, _values, result) + begin_c_declaration('\Z') + @grammar.epilogue_first_lineno = @lexer.line + 1 + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 504) + def _reduce_123(val, _values, result) + end_c_declaration + @grammar.epilogue = val[2].s_value + + result + end +.,., + +# reduce 124 omitted + +# reduce 125 omitted + +# reduce 126 omitted + +# reduce 127 omitted + +# reduce 128 omitted + +module_eval(<<'.,.,', 'parser.y', 515) + def _reduce_129(val, _values, result) + result = [val[0]] + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 516) + def _reduce_130(val, _values, result) + result = val[0].append(val[1]) + result + end +.,., + +# reduce 131 omitted + +# reduce 132 omitted + +module_eval(<<'.,.,', 'parser.y', 521) + def _reduce_133(val, _values, result) + result = Lrama::Lexer::Token::Ident.new(s_value: val[0]) + result + end +.,., + +# reduce 134 omitted + +# reduce 135 omitted + +def _reduce_none(val, _values, result) + val[0] +end + + end # class Parser +end # module Lrama diff --git a/tool/lrama/lib/lrama/report.rb b/tool/lrama/lib/lrama/report.rb new file mode 100644 index 0000000000..650ac09d52 --- /dev/null +++ b/tool/lrama/lib/lrama/report.rb @@ -0,0 +1,2 @@ +require 'lrama/report/duration' +require 'lrama/report/profile' diff --git a/tool/lrama/lib/lrama/report/duration.rb b/tool/lrama/lib/lrama/report/duration.rb new file mode 100644 index 0000000000..7afe284f1a --- /dev/null +++ b/tool/lrama/lib/lrama/report/duration.rb @@ -0,0 +1,25 @@ +module Lrama + class Report + module Duration + def self.enable + @_report_duration_enabled = true + end + + def self.enabled? + !!@_report_duration_enabled + end + + def report_duration(method_name) + time1 = Time.now.to_f + result = yield + time2 = Time.now.to_f + + if Duration.enabled? + puts sprintf("%s %10.5f s", method_name, time2 - time1) + end + + return result + end + end + end +end diff --git a/tool/lrama/lib/lrama/report/profile.rb b/tool/lrama/lib/lrama/report/profile.rb new file mode 100644 index 0000000000..36156800a4 --- /dev/null +++ b/tool/lrama/lib/lrama/report/profile.rb @@ -0,0 +1,14 @@ +module Lrama + class Report + module Profile + # See "Profiling Lrama" in README.md for how to use. + def self.report_profile + require "stackprof" + + StackProf.run(mode: :cpu, raw: true, out: 'tmp/stackprof-cpu-myapp.dump') do + yield + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/state.rb b/tool/lrama/lib/lrama/state.rb new file mode 100644 index 0000000000..45bfe5acf6 --- /dev/null +++ b/tool/lrama/lib/lrama/state.rb @@ -0,0 +1,166 @@ +require "lrama/state/reduce" +require "lrama/state/reduce_reduce_conflict" +require "lrama/state/resolved_conflict" +require "lrama/state/shift" +require "lrama/state/shift_reduce_conflict" + +module Lrama + class State + attr_reader :id, :accessing_symbol, :kernels, :conflicts, :resolved_conflicts, + :default_reduction_rule, :closure, :items + attr_accessor :shifts, :reduces + + def initialize(id, accessing_symbol, kernels) + @id = id + @accessing_symbol = accessing_symbol + @kernels = kernels.freeze + @items = @kernels + # Manage relationships between items to state + # to resolve next state + @items_to_state = {} + @conflicts = [] + @resolved_conflicts = [] + @default_reduction_rule = nil + end + + def closure=(closure) + @closure = closure + @items = @kernels + @closure + end + + def non_default_reduces + reduces.reject do |reduce| + reduce.rule == @default_reduction_rule + end + end + + def compute_shifts_reduces + _shifts = {} + reduces = [] + items.each do |item| + # TODO: Consider what should be pushed + if item.end_of_rule? + reduces << Reduce.new(item) + else + key = item.next_sym + _shifts[key] ||= [] + _shifts[key] << item.new_by_next_position + end + end + + # It seems Bison 3.8.2 iterates transitions order by symbol number + shifts = _shifts.sort_by do |next_sym, new_items| + next_sym.number + end.map do |next_sym, new_items| + Shift.new(next_sym, new_items.flatten) + end + self.shifts = shifts.freeze + self.reduces = reduces.freeze + end + + def set_items_to_state(items, next_state) + @items_to_state[items] = next_state + end + + def set_look_ahead(rule, look_ahead) + reduce = reduces.find do |r| + r.rule == rule + end + + reduce.look_ahead = look_ahead + end + + # Returns array of [Shift, next_state] + def nterm_transitions + return @nterm_transitions if @nterm_transitions + + @nterm_transitions = [] + + shifts.each do |shift| + next if shift.next_sym.term? + + @nterm_transitions << [shift, @items_to_state[shift.next_items]] + end + + @nterm_transitions + end + + # Returns array of [Shift, next_state] + def term_transitions + return @term_transitions if @term_transitions + + @term_transitions = [] + + shifts.each do |shift| + next if shift.next_sym.nterm? + + @term_transitions << [shift, @items_to_state[shift.next_items]] + end + + @term_transitions + end + + def transitions + term_transitions + nterm_transitions + end + + def selected_term_transitions + term_transitions.reject do |shift, next_state| + shift.not_selected + end + end + + # Move to next state by sym + def transition(sym) + result = nil + + if sym.term? + term_transitions.each do |shift, next_state| + term = shift.next_sym + result = next_state if term == sym + end + else + nterm_transitions.each do |shift, next_state| + nterm = shift.next_sym + result = next_state if nterm == sym + end + end + + raise "Can not transit by #{sym} #{self}" if result.nil? + + result + end + + def find_reduce_by_item!(item) + reduces.find do |r| + r.item == item + end || (raise "reduce is not found. #{item}") + end + + def default_reduction_rule=(default_reduction_rule) + @default_reduction_rule = default_reduction_rule + + reduces.each do |r| + if r.rule == default_reduction_rule + r.default_reduction = true + end + end + end + + def has_conflicts? + !@conflicts.empty? + end + + def sr_conflicts + @conflicts.select do |conflict| + conflict.type == :shift_reduce + end + end + + def rr_conflicts + @conflicts.select do |conflict| + conflict.type == :reduce_reduce + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/reduce.rb b/tool/lrama/lib/lrama/state/reduce.rb new file mode 100644 index 0000000000..8ba51f45f2 --- /dev/null +++ b/tool/lrama/lib/lrama/state/reduce.rb @@ -0,0 +1,35 @@ +module Lrama + class State + class Reduce + # https://www.gnu.org/software/bison/manual/html_node/Default-Reductions.html + attr_reader :item, :look_ahead, :not_selected_symbols + attr_accessor :default_reduction + + def initialize(item) + @item = item + @look_ahead = nil + @not_selected_symbols = [] + end + + def rule + @item.rule + end + + def look_ahead=(look_ahead) + @look_ahead = look_ahead.freeze + end + + def add_not_selected_symbol(sym) + @not_selected_symbols << sym + end + + def selected_look_ahead + if @look_ahead + @look_ahead - @not_selected_symbols + else + [] + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/reduce_reduce_conflict.rb b/tool/lrama/lib/lrama/state/reduce_reduce_conflict.rb new file mode 100644 index 0000000000..0a0e4dc20a --- /dev/null +++ b/tool/lrama/lib/lrama/state/reduce_reduce_conflict.rb @@ -0,0 +1,9 @@ +module Lrama + class State + class ReduceReduceConflict < Struct.new(:symbols, :reduce1, :reduce2, keyword_init: true) + def type + :reduce_reduce + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/resolved_conflict.rb b/tool/lrama/lib/lrama/state/resolved_conflict.rb new file mode 100644 index 0000000000..02ea892147 --- /dev/null +++ b/tool/lrama/lib/lrama/state/resolved_conflict.rb @@ -0,0 +1,29 @@ +module Lrama + class State + # * symbol: A symbol under discussion + # * reduce: A reduce under discussion + # * which: For which a conflict is resolved. :shift, :reduce or :error (for nonassociative) + class ResolvedConflict < Struct.new(:symbol, :reduce, :which, :same_prec, keyword_init: true) + def report_message + s = symbol.display_name + r = reduce.rule.precedence_sym.display_name + case + when which == :shift && same_prec + msg = "resolved as #{which} (%right #{s})" + when which == :shift + msg = "resolved as #{which} (#{r} < #{s})" + when which == :reduce && same_prec + msg = "resolved as #{which} (%left #{s})" + when which == :reduce + msg = "resolved as #{which} (#{s} < #{r})" + when which == :error + msg = "resolved as an #{which} (%nonassoc #{s})" + else + raise "Unknown direction. #{self}" + end + + "Conflict between rule #{reduce.rule.id} and token #{s} #{msg}." + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/shift.rb b/tool/lrama/lib/lrama/state/shift.rb new file mode 100644 index 0000000000..2021eb61f6 --- /dev/null +++ b/tool/lrama/lib/lrama/state/shift.rb @@ -0,0 +1,13 @@ +module Lrama + class State + class Shift + attr_reader :next_sym, :next_items + attr_accessor :not_selected + + def initialize(next_sym, next_items) + @next_sym = next_sym + @next_items = next_items + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/shift_reduce_conflict.rb b/tool/lrama/lib/lrama/state/shift_reduce_conflict.rb new file mode 100644 index 0000000000..f80bd5f352 --- /dev/null +++ b/tool/lrama/lib/lrama/state/shift_reduce_conflict.rb @@ -0,0 +1,9 @@ +module Lrama + class State + class ShiftReduceConflict < Struct.new(:symbols, :shift, :reduce, keyword_init: true) + def type + :shift_reduce + end + end + end +end diff --git a/tool/lrama/lib/lrama/states.rb b/tool/lrama/lib/lrama/states.rb new file mode 100644 index 0000000000..290e996b82 --- /dev/null +++ b/tool/lrama/lib/lrama/states.rb @@ -0,0 +1,556 @@ +require "forwardable" +require "lrama/report/duration" +require "lrama/states/item" + +module Lrama + # States is passed to a template file + # + # "Efficient Computation of LALR(1) Look-Ahead Sets" + # https://dl.acm.org/doi/pdf/10.1145/69622.357187 + class States + extend Forwardable + include Lrama::Report::Duration + + def_delegators "@grammar", :symbols, :terms, :nterms, :rules, + :accept_symbol, :eof_symbol, :undef_symbol, :find_symbol_by_s_value! + + attr_reader :states, :reads_relation, :includes_relation, :lookback_relation + + def initialize(grammar, warning, trace_state: false) + @grammar = grammar + @warning = warning + @trace_state = trace_state + + @states = [] + + # `DR(p, A) = {t ∈ T | p -(A)-> r -(t)-> }` + # where p is state, A is nterm, t is term. + # + # `@direct_read_sets` is a hash whose + # key is [state.id, nterm.token_id], + # value is bitmap of term. + @direct_read_sets = {} + + # Reads relation on nonterminal transitions (pair of state and nterm) + # `(p, A) reads (r, C) iff p -(A)-> r -(C)-> and C =>* ε` + # where p, r are state, A, C are nterm. + # + # `@reads_relation` is a hash whose + # key is [state.id, nterm.token_id], + # value is array of [state.id, nterm.token_id]. + @reads_relation = {} + + # `@read_sets` is a hash whose + # key is [state.id, nterm.token_id], + # value is bitmap of term. + @read_sets = {} + + # `(p, A) includes (p', B) iff B -> βAγ, γ =>* ε, p' -(β)-> p` + # where p, p' are state, A, B are nterm, β, γ is sequence of symbol. + # + # `@includes_relation` is a hash whose + # key is [state.id, nterm.token_id], + # value is array of [state.id, nterm.token_id]. + @includes_relation = {} + + # `(q, A -> ω) lookback (p, A) iff p -(ω)-> q` + # where p, q are state, A -> ω is rule, A is nterm, ω is sequence of symbol. + # + # `@lookback_relation` is a hash whose + # key is [state.id, rule.id], + # value is array of [state.id, nterm.token_id]. + @lookback_relation = {} + + # `@follow_sets` is a hash whose + # key is [state.id, rule.id], + # value is bitmap of term. + @follow_sets = {} + + # `LA(q, A -> ω) = ∪{Follow(p, A) | (q, A -> ω) lookback (p, A)` + # + # `@la` is a hash whose + # key is [state.id, rule.id], + # value is bitmap of term. + @la = {} + end + + def compute + # Look Ahead Sets + report_duration(:compute_lr0_states) { compute_lr0_states } + report_duration(:compute_direct_read_sets) { compute_direct_read_sets } + report_duration(:compute_reads_relation) { compute_reads_relation } + report_duration(:compute_read_sets) { compute_read_sets } + report_duration(:compute_includes_relation) { compute_includes_relation } + report_duration(:compute_lookback_relation) { compute_lookback_relation } + report_duration(:compute_follow_sets) { compute_follow_sets } + report_duration(:compute_look_ahead_sets) { compute_look_ahead_sets } + + # Conflicts + report_duration(:compute_conflicts) { compute_conflicts } + + report_duration(:compute_default_reduction) { compute_default_reduction } + + check_conflicts + end + + def reporter + StatesReporter.new(self) + end + + def states_count + @states.count + end + + def direct_read_sets + @direct_read_sets.transform_values do |v| + bitmap_to_terms(v) + end + end + + def read_sets + @read_sets.transform_values do |v| + bitmap_to_terms(v) + end + end + + def follow_sets + @follow_sets.transform_values do |v| + bitmap_to_terms(v) + end + end + + def la + @la.transform_values do |v| + bitmap_to_terms(v) + end + end + + private + + def sr_conflicts + @states.flat_map(&:sr_conflicts) + end + + def rr_conflicts + @states.flat_map(&:rr_conflicts) + end + + def trace_state + if @trace_state + yield STDERR + end + end + + def create_state(accessing_symbol, kernels, states_created) + # A item can appear in some states, + # so need to use `kernels` (not `kernels.first`) as a key. + # + # For example... + # + # %% + # program: '+' strings_1 + # | '-' strings_2 + # ; + # + # strings_1: string_1 + # ; + # + # strings_2: string_1 + # | string_2 + # ; + # + # string_1: string + # ; + # + # string_2: string '+' + # ; + # + # string: tSTRING + # ; + # %% + # + # For these grammar, there are 2 states + # + # State A + # string_1: string • + # + # State B + # string_1: string • + # string_2: string • '+' + # + return [states_created[kernels], false] if states_created[kernels] + + state = State.new(@states.count, accessing_symbol, kernels) + @states << state + states_created[kernels] = state + + return [state, true] + end + + def setup_state(state) + # closure + closure = [] + visited = {} + queued = {} + items = state.kernels.dup + + items.each do |item| + queued[item] = true + end + + while (item = items.shift) do + visited[item] = true + + if (sym = item.next_sym) && sym.nterm? + @grammar.find_rules_by_symbol!(sym).each do |rule| + i = Item.new(rule: rule, position: 0) + next if queued[i] + closure << i + items << i + queued[i] = true + end + end + end + + state.closure = closure.sort_by {|i| i.rule.id } + + # Trace + trace_state do |out| + out << "Closure: input\n" + state.kernels.each do |item| + out << " #{item.display_rest}\n" + end + out << "\n\n" + out << "Closure: output\n" + state.items.each do |item| + out << " #{item.display_rest}\n" + end + out << "\n\n" + end + + # shift & reduce + state.compute_shifts_reduces + end + + def enqueue_state(states, state) + # Trace + previous = state.kernels.first.previous_sym + trace_state do |out| + out << sprintf("state_list_append (state = %d, symbol = %d (%s))", + @states.count, previous.number, previous.display_name) + end + + states << state + end + + def compute_lr0_states + # State queue + states = [] + states_created = {} + + state, _ = create_state(symbols.first, [Item.new(rule: @grammar.rules.first, position: 0)], states_created) + enqueue_state(states, state) + + while (state = states.shift) do + # Trace + # + # Bison 3.8.2 renders "(reached by "end-of-input")" for State 0 but + # I think it is not correct... + previous = state.kernels.first.previous_sym + trace_state do |out| + out << "Processing state #{state.id} (reached by #{previous.display_name})\n" + end + + setup_state(state) + + state.shifts.each do |shift| + new_state, created = create_state(shift.next_sym, shift.next_items, states_created) + state.set_items_to_state(shift.next_items, new_state) + enqueue_state(states, new_state) if created + end + end + end + + def nterm_transitions + a = [] + + @states.each do |state| + state.nterm_transitions.each do |shift, next_state| + nterm = shift.next_sym + a << [state, nterm, next_state] + end + end + + a + end + + def compute_direct_read_sets + @states.each do |state| + state.nterm_transitions.each do |shift, next_state| + nterm = shift.next_sym + + ary = next_state.term_transitions.map do |shift, _| + shift.next_sym.number + end + + key = [state.id, nterm.token_id] + @direct_read_sets[key] = Bitmap.from_array(ary) + end + end + end + + def compute_reads_relation + @states.each do |state| + state.nterm_transitions.each do |shift, next_state| + nterm = shift.next_sym + next_state.nterm_transitions.each do |shift2, _next_state2| + nterm2 = shift2.next_sym + if nterm2.nullable + key = [state.id, nterm.token_id] + @reads_relation[key] ||= [] + @reads_relation[key] << [next_state.id, nterm2.token_id] + end + end + end + end + end + + def compute_read_sets + sets = nterm_transitions.map do |state, nterm, next_state| + [state.id, nterm.token_id] + end + + @read_sets = Digraph.new(sets, @reads_relation, @direct_read_sets).compute + end + + # Execute transition of state by symbols + # then return final state. + def transition(state, symbols) + symbols.each do |sym| + state = state.transition(sym) + end + + state + end + + def compute_includes_relation + @states.each do |state| + state.nterm_transitions.each do |shift, next_state| + nterm = shift.next_sym + @grammar.find_rules_by_symbol!(nterm).each do |rule| + i = rule.rhs.count - 1 + + while (i > -1) do + sym = rule.rhs[i] + + break if sym.term? + state2 = transition(state, rule.rhs[0...i]) + # p' = state, B = nterm, p = state2, A = sym + key = [state2.id, sym.token_id] + # TODO: need to omit if state == state2 ? + @includes_relation[key] ||= [] + @includes_relation[key] << [state.id, nterm.token_id] + break if !sym.nullable + i -= 1 + end + end + end + end + end + + def compute_lookback_relation + @states.each do |state| + state.nterm_transitions.each do |shift, next_state| + nterm = shift.next_sym + @grammar.find_rules_by_symbol!(nterm).each do |rule| + state2 = transition(state, rule.rhs) + # p = state, A = nterm, q = state2, A -> ω = rule + key = [state2.id, rule.id] + @lookback_relation[key] ||= [] + @lookback_relation[key] << [state.id, nterm.token_id] + end + end + end + end + + def compute_follow_sets + sets = nterm_transitions.map do |state, nterm, next_state| + [state.id, nterm.token_id] + end + + @follow_sets = Digraph.new(sets, @includes_relation, @read_sets).compute + end + + def compute_look_ahead_sets + @states.each do |state| + rules.each do |rule| + ary = @lookback_relation[[state.id, rule.id]] + next if !ary + + ary.each do |state2_id, nterm_token_id| + # q = state, A -> ω = rule, p = state2, A = nterm + follows = @follow_sets[[state2_id, nterm_token_id]] + + next if follows == 0 + + key = [state.id, rule.id] + @la[key] ||= 0 + look_ahead = @la[key] | follows + @la[key] |= look_ahead + + # No risk of conflict when + # * the state only has single reduce + # * the state only has nterm_transitions (GOTO) + next if state.reduces.count == 1 && state.term_transitions.count == 0 + + state.set_look_ahead(rule, bitmap_to_terms(look_ahead)) + end + end + end + end + + def bitmap_to_terms(bit) + ary = Bitmap.to_array(bit) + ary.map do |i| + @grammar.find_symbol_by_number!(i) + end + end + + def compute_conflicts + compute_shift_reduce_conflicts + compute_reduce_reduce_conflicts + end + + def compute_shift_reduce_conflicts + states.each do |state| + state.shifts.each do |shift| + state.reduces.each do |reduce| + sym = shift.next_sym + + next unless reduce.look_ahead + next if !reduce.look_ahead.include?(sym) + + # Shift/Reduce conflict + shift_prec = sym.precedence + reduce_prec = reduce.item.rule.precedence + + # Can resolve only when both have prec + unless shift_prec && reduce_prec + state.conflicts << State::ShiftReduceConflict.new(symbols: [sym], shift: shift, reduce: reduce) + next + end + + case + when shift_prec < reduce_prec + # Reduce is selected + state.resolved_conflicts << State::ResolvedConflict.new(symbol: sym, reduce: reduce, which: :reduce) + shift.not_selected = true + next + when shift_prec > reduce_prec + # Shift is selected + state.resolved_conflicts << State::ResolvedConflict.new(symbol: sym, reduce: reduce, which: :shift) + reduce.add_not_selected_symbol(sym) + next + end + + # shift_prec == reduce_prec, then check associativity + case sym.precedence.type + when :precedence + # %precedence only specifies precedence and not specify associativity + # then a conflict is unresolved if precedence is same. + state.conflicts << State::ShiftReduceConflict.new(symbols: [sym], shift: shift, reduce: reduce) + next + when :right + # Shift is selected + state.resolved_conflicts << State::ResolvedConflict.new(symbol: sym, reduce: reduce, which: :shift, same_prec: true) + reduce.add_not_selected_symbol(sym) + next + when :left + # Reduce is selected + state.resolved_conflicts << State::ResolvedConflict.new(symbol: sym, reduce: reduce, which: :reduce, same_prec: true) + shift.not_selected = true + next + when :nonassoc + # Can not resolve + # + # nonassoc creates "run-time" error, precedence creates "compile-time" error. + # Then omit both the shift and reduce. + # + # https://www.gnu.org/software/bison/manual/html_node/Using-Precedence.html + state.resolved_conflicts << State::ResolvedConflict.new(symbol: sym, reduce: reduce, which: :error) + shift.not_selected = true + reduce.add_not_selected_symbol(sym) + else + raise "Unknown precedence type. #{sym}" + end + end + end + end + end + + def compute_reduce_reduce_conflicts + states.each do |state| + count = state.reduces.count + + for i in 0...count do + reduce1 = state.reduces[i] + next if reduce1.look_ahead.nil? + + for j in (i+1)...count do + reduce2 = state.reduces[j] + next if reduce2.look_ahead.nil? + + intersection = reduce1.look_ahead & reduce2.look_ahead + + if !intersection.empty? + state.conflicts << State::ReduceReduceConflict.new(symbols: intersection, reduce1: reduce1, reduce2: reduce2) + end + end + end + end + end + + def compute_default_reduction + states.each do |state| + next if state.reduces.empty? + # Do not set, if conflict exist + next if !state.conflicts.empty? + # Do not set, if shift with `error` exists. + next if state.shifts.map(&:next_sym).include?(@grammar.error_symbol) + + state.default_reduction_rule = state.reduces.map do |r| + [r.rule, r.rule.id, (r.look_ahead || []).count] + end.min_by do |rule, rule_id, count| + [-count, rule_id] + end.first + end + end + + def check_conflicts + sr_count = sr_conflicts.count + rr_count = rr_conflicts.count + + if @grammar.expect + + expected_sr_conflicts = @grammar.expect + expected_rr_conflicts = 0 + + if expected_sr_conflicts != sr_count + @warning.error("shift/reduce conflicts: #{sr_count} found, #{expected_sr_conflicts} expected") + end + + if expected_rr_conflicts != rr_count + @warning.error("reduce/reduce conflicts: #{rr_count} found, #{expected_rr_conflicts} expected") + end + else + if sr_count != 0 + @warning.warn("shift/reduce conflicts: #{sr_count} found") + end + + if rr_count != 0 + @warning.warn("reduce/reduce conflicts: #{rr_count} found") + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/states/item.rb b/tool/lrama/lib/lrama/states/item.rb new file mode 100644 index 0000000000..31b74b9d34 --- /dev/null +++ b/tool/lrama/lib/lrama/states/item.rb @@ -0,0 +1,81 @@ +# TODO: Validate position is not over rule rhs + +require "forwardable" + +module Lrama + class States + class Item < Struct.new(:rule, :position, keyword_init: true) + extend Forwardable + + def_delegators "rule", :lhs, :rhs + + # Optimization for States#setup_state + def hash + [rule_id, position].hash + end + + def rule_id + rule.id + end + + def empty_rule? + rule.empty_rule? + end + + def number_of_rest_symbols + rhs.count - position + end + + def next_sym + rhs[position] + end + + def next_next_sym + rhs[position + 1] + end + + def previous_sym + rhs[position - 1] + end + + def end_of_rule? + rhs.count == position + end + + def beginning_of_rule? + position == 0 + end + + def start_item? + rule.initial_rule? && beginning_of_rule? + end + + def new_by_next_position + Item.new(rule: rule, position: position + 1) + end + + def symbols_before_dot + rhs[0...position] + end + + def symbols_after_dot + rhs[position..-1] + end + + def to_s + "#{lhs.id.s_value}: #{display_name}" + end + + def display_name + r = rhs.map(&:display_name).insert(position, "•").join(" ") + "#{r} (rule #{rule_id})" + end + + # Right after position + def display_rest + r = rhs[position..-1].map(&:display_name).join(" ") + ". #{r} (rule #{rule_id})" + end + end + end +end diff --git a/tool/lrama/lib/lrama/states_reporter.rb b/tool/lrama/lib/lrama/states_reporter.rb new file mode 100644 index 0000000000..6f96cc6f65 --- /dev/null +++ b/tool/lrama/lib/lrama/states_reporter.rb @@ -0,0 +1,321 @@ +module Lrama + class StatesReporter + include Lrama::Report::Duration + + def initialize(states) + @states = states + end + + def report(io, **options) + report_duration(:report) do + _report(io, **options) + end + end + + private + + def _report(io, grammar: false, states: false, itemsets: false, lookaheads: false, solved: false, counterexamples: false, verbose: false) + # TODO: Unused terms + # TODO: Unused rules + + report_conflicts(io) + report_grammar(io) if grammar + report_states(io, itemsets, lookaheads, solved, counterexamples, verbose) + end + + def report_conflicts(io) + has_conflict = false + + @states.states.each do |state| + messages = [] + cs = state.conflicts.group_by(&:type) + if cs[:shift_reduce] + messages << "#{cs[:shift_reduce].count} shift/reduce" + end + + if cs[:reduce_reduce] + messages << "#{cs[:reduce_reduce].count} reduce/reduce" + end + + if !messages.empty? + has_conflict = true + io << "State #{state.id} conflicts: #{messages.join(', ')}\n" + end + end + + if has_conflict + io << "\n\n" + end + end + + def report_grammar(io) + io << "Grammar\n" + last_lhs = nil + + @states.rules.each do |rule| + if rule.empty_rule? + r = "ε" + else + r = rule.rhs.map(&:display_name).join(" ") + end + + if rule.lhs == last_lhs + io << sprintf("%5d %s| %s\n", rule.id, " " * rule.lhs.display_name.length, r) + else + io << "\n" + io << sprintf("%5d %s: %s\n", rule.id, rule.lhs.display_name, r) + end + + last_lhs = rule.lhs + end + io << "\n\n" + end + + def report_states(io, itemsets, lookaheads, solved, counterexamples, verbose) + if counterexamples + cex = Counterexamples.new(@states) + end + + @states.states.each do |state| + # Report State + io << "State #{state.id}\n\n" + + # Report item + last_lhs = nil + list = itemsets ? state.items : state.kernels + list.sort_by {|i| [i.rule_id, i.position] }.each do |item| + if item.empty_rule? + r = "ε •" + else + r = item.rhs.map(&:display_name).insert(item.position, "•").join(" ") + end + if item.lhs == last_lhs + l = " " * item.lhs.id.s_value.length + "|" + else + l = item.lhs.id.s_value + ":" + end + la = "" + if lookaheads && item.end_of_rule? + reduce = state.find_reduce_by_item!(item) + look_ahead = reduce.selected_look_ahead + if !look_ahead.empty? + la = " [#{look_ahead.map(&:display_name).join(", ")}]" + end + end + last_lhs = item.lhs + + io << sprintf("%5i %s %s%s\n", item.rule_id, l, r, la) + end + io << "\n" + + # Report shifts + tmp = state.term_transitions.reject do |shift, _| + shift.not_selected + end.map do |shift, next_state| + [shift.next_sym, next_state.id] + end + max_len = tmp.map(&:first).map(&:display_name).map(&:length).max + tmp.each do |term, state_id| + io << " #{term.display_name.ljust(max_len)} shift, and go to state #{state_id}\n" + end + io << "\n" if !tmp.empty? + + # Report error caused by %nonassoc + nl = false + tmp = state.resolved_conflicts.select do |resolved| + resolved.which == :error + end.map do |error| + error.symbol.display_name + end + max_len = tmp.map(&:length).max + tmp.each do |name| + nl = true + io << " #{name.ljust(max_len)} error (nonassociative)\n" + end + io << "\n" if !tmp.empty? + + # Report reduces + nl = false + max_len = state.non_default_reduces.flat_map(&:look_ahead).compact.map(&:display_name).map(&:length).max || 0 + max_len = [max_len, "$default".length].max if state.default_reduction_rule + ary = [] + + state.non_default_reduces.each do |reduce| + reduce.look_ahead.each do |term| + ary << [term, reduce] + end + end + + ary.sort_by do |term, reduce| + term.number + end.each do |term, reduce| + rule = reduce.item.rule + io << " #{term.display_name.ljust(max_len)} reduce using rule #{rule.id} (#{rule.lhs.display_name})\n" + nl = true + end + + if (r = state.default_reduction_rule) + nl = true + s = "$default".ljust(max_len) + + if r.initial_rule? + io << " #{s} accept\n" + else + io << " #{s} reduce using rule #{r.id} (#{r.lhs.display_name})\n" + end + end + io << "\n" if nl + + # Report nonterminal transitions + tmp = [] + max_len = 0 + state.nterm_transitions.each do |shift, next_state| + nterm = shift.next_sym + tmp << [nterm, next_state.id] + max_len = [max_len, nterm.id.s_value.length].max + end + tmp.uniq! + tmp.sort_by! do |nterm, state_id| + nterm.number + end + tmp.each do |nterm, state_id| + io << " #{nterm.id.s_value.ljust(max_len)} go to state #{state_id}\n" + end + io << "\n" if !tmp.empty? + + if solved + # Report conflict resolutions + state.resolved_conflicts.each do |resolved| + io << " #{resolved.report_message}\n" + end + io << "\n" if !state.resolved_conflicts.empty? + end + + if counterexamples && state.has_conflicts? + # Report counterexamples + examples = cex.compute(state) + examples.each do |example| + label0 = example.type == :shift_reduce ? "shift/reduce" : "reduce/reduce" + label1 = example.type == :shift_reduce ? "Shift derivation" : "First Reduce derivation" + label2 = example.type == :shift_reduce ? "Reduce derivation" : "Second Reduce derivation" + + io << " #{label0} conflict on token #{example.conflict_symbol.id.s_value}:\n" + io << " #{example.path1_item}\n" + io << " #{example.path2_item}\n" + io << " #{label1}\n" + example.derivations1.render_strings_for_report.each do |str| + io << " #{str}\n" + end + io << " #{label2}\n" + example.derivations2.render_strings_for_report.each do |str| + io << " #{str}\n" + end + end + end + + if verbose + # Report direct_read_sets + io << " [Direct Read sets]\n" + direct_read_sets = @states.direct_read_sets + @states.nterms.each do |nterm| + terms = direct_read_sets[[state.id, nterm.token_id]] + next if !terms + next if terms.empty? + + str = terms.map {|sym| sym.id.s_value }.join(", ") + io << " read #{nterm.id.s_value} shift #{str}\n" + end + io << "\n" + + # Report reads_relation + io << " [Reads Relation]\n" + @states.nterms.each do |nterm| + a = @states.reads_relation[[state.id, nterm.token_id]] + next if !a + + a.each do |state_id2, nterm_id2| + n = @states.nterms.find {|n| n.token_id == nterm_id2 } + io << " (State #{state_id2}, #{n.id.s_value})\n" + end + end + io << "\n" + + # Report read_sets + io << " [Read sets]\n" + read_sets = @states.read_sets + @states.nterms.each do |nterm| + terms = read_sets[[state.id, nterm.token_id]] + next if !terms + next if terms.empty? + + terms.each do |sym| + io << " #{sym.id.s_value}\n" + end + end + io << "\n" + + # Report includes_relation + io << " [Includes Relation]\n" + @states.nterms.each do |nterm| + a = @states.includes_relation[[state.id, nterm.token_id]] + next if !a + + a.each do |state_id2, nterm_id2| + n = @states.nterms.find {|n| n.token_id == nterm_id2 } + io << " (State #{state.id}, #{nterm.id.s_value}) -> (State #{state_id2}, #{n.id.s_value})\n" + end + end + io << "\n" + + # Report lookback_relation + io << " [Lookback Relation]\n" + @states.rules.each do |rule| + a = @states.lookback_relation[[state.id, rule.id]] + next if !a + + a.each do |state_id2, nterm_id2| + n = @states.nterms.find {|n| n.token_id == nterm_id2 } + io << " (Rule: #{rule}) -> (State #{state_id2}, #{n.id.s_value})\n" + end + end + io << "\n" + + # Report follow_sets + io << " [Follow sets]\n" + follow_sets = @states.follow_sets + @states.nterms.each do |nterm| + terms = follow_sets[[state.id, nterm.token_id]] + + next if !terms + + terms.each do |sym| + io << " #{nterm.id.s_value} -> #{sym.id.s_value}\n" + end + end + io << "\n" + + # Report LA + io << " [Look-Ahead Sets]\n" + tmp = [] + max_len = 0 + @states.rules.each do |rule| + syms = @states.la[[state.id, rule.id]] + next if !syms + + tmp << [rule, syms] + max_len = ([max_len] + syms.map {|s| s.id.s_value.length }).max + end + tmp.each do |rule, syms| + syms.each do |sym| + io << " #{sym.id.s_value.ljust(max_len)} reduce using rule #{rule.id} (#{rule.lhs.id.s_value})\n" + end + end + io << "\n" if !tmp.empty? + end + + # End of Report State + io << "\n" + end + end + end +end diff --git a/tool/lrama/lib/lrama/version.rb b/tool/lrama/lib/lrama/version.rb new file mode 100644 index 0000000000..ccd593f344 --- /dev/null +++ b/tool/lrama/lib/lrama/version.rb @@ -0,0 +1,3 @@ +module Lrama + VERSION = "0.6.5".freeze +end diff --git a/tool/lrama/lib/lrama/warning.rb b/tool/lrama/lib/lrama/warning.rb new file mode 100644 index 0000000000..3c99791ebf --- /dev/null +++ b/tool/lrama/lib/lrama/warning.rb @@ -0,0 +1,25 @@ +module Lrama + class Warning + attr_reader :errors, :warns + + def initialize(out = STDERR) + @out = out + @errors = [] + @warns = [] + end + + def error(message) + @out << message << "\n" + @errors << message + end + + def warn(message) + @out << message << "\n" + @warns << message + end + + def has_error? + !@errors.empty? + end + end +end diff --git a/tool/lrama/template/bison/_yacc.h b/tool/lrama/template/bison/_yacc.h new file mode 100644 index 0000000000..34ed6d81f5 --- /dev/null +++ b/tool/lrama/template/bison/_yacc.h @@ -0,0 +1,71 @@ +<%# b4_shared_declarations -%> + <%-# b4_cpp_guard_open([b4_spec_mapped_header_file]) -%> + <%- if output.spec_mapped_header_file -%> +#ifndef <%= output.b4_cpp_guard__b4_spec_mapped_header_file %> +# define <%= output.b4_cpp_guard__b4_spec_mapped_header_file %> + <%- end -%> + <%-# b4_declare_yydebug & b4_YYDEBUG_define -%> +/* Debug traces. */ +#ifndef YYDEBUG +# define YYDEBUG 0 +#endif +#if YYDEBUG && !defined(yydebug) +extern int yydebug; +#endif +<%= output.percent_code("requires") %> + + <%-# b4_token_enums_defines -%> +/* Token kinds. */ +#ifndef YYTOKENTYPE +# define YYTOKENTYPE + enum yytokentype + { +<%= output.token_enums -%> + }; + typedef enum yytokentype yytoken_kind_t; +#endif + + <%-# b4_declare_yylstype -%> + <%-# b4_value_type_define -%> +/* Value type. */ +#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED +union YYSTYPE +{ +#line <%= output.grammar.union.lineno %> "<%= output.grammar_file_path %>" +<%= output.grammar.union.braces_less_code %> +#line [@oline@] [@ofile@] + +}; +typedef union YYSTYPE YYSTYPE; +# define YYSTYPE_IS_TRIVIAL 1 +# define YYSTYPE_IS_DECLARED 1 +#endif + + <%-# b4_location_type_define -%> +/* Location type. */ +#if ! defined YYLTYPE && ! defined YYLTYPE_IS_DECLARED +typedef struct YYLTYPE YYLTYPE; +struct YYLTYPE +{ + int first_line; + int first_column; + int last_line; + int last_column; +}; +# define YYLTYPE_IS_DECLARED 1 +# define YYLTYPE_IS_TRIVIAL 1 +#endif + + + + + <%-# b4_declare_yyerror_and_yylex. Not supported -%> + <%-# b4_declare_yyparse -%> +int yyparse (<%= output.parse_param %>); + + +<%= output.percent_code("provides") %> + <%-# b4_cpp_guard_close([b4_spec_mapped_header_file]) -%> + <%- if output.spec_mapped_header_file -%> +#endif /* !<%= output.b4_cpp_guard__b4_spec_mapped_header_file %> */ + <%- end -%> diff --git a/tool/lrama/template/bison/yacc.c b/tool/lrama/template/bison/yacc.c new file mode 100644 index 0000000000..2d6753c282 --- /dev/null +++ b/tool/lrama/template/bison/yacc.c @@ -0,0 +1,2063 @@ +<%# b4_generated_by -%> +/* A Bison parser, made by Lrama <%= Lrama::VERSION %>. */ + +<%# b4_copyright -%> +/* Bison implementation for Yacc-like parsers in C + + Copyright (C) 1984, 1989-1990, 2000-2015, 2018-2021 Free Software Foundation, + Inc. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. */ + +/* As a special exception, you may create a larger work that contains + part or all of the Bison parser skeleton and distribute that work + under terms of your choice, so long as that work isn't itself a + parser generator using the skeleton or a modified version thereof + as a parser skeleton. Alternatively, if you modify or redistribute + the parser skeleton itself, you may (at your option) remove this + special exception, which will cause the skeleton and the resulting + Bison output files to be licensed under the GNU General Public + License without this special exception. + + This special exception was added by the Free Software Foundation in + version 2.2 of Bison. */ + +/* C LALR(1) parser skeleton written by Richard Stallman, by + simplifying the original so-called "semantic" parser. */ + +<%# b4_disclaimer -%> +/* DO NOT RELY ON FEATURES THAT ARE NOT DOCUMENTED in the manual, + especially those whose name start with YY_ or yy_. They are + private implementation details that can be changed or removed. */ + +/* All symbols defined below should begin with yy or YY, to avoid + infringing on user name space. This should be done even for local + variables, as they might otherwise be expanded by user macros. + There are some unavoidable exceptions within include files to + define necessary library symbols; they are noted "INFRINGES ON + USER NAME SPACE" below. */ + +<%# b4_identification -%> +/* Identify Bison output, and Bison version. */ +#define YYBISON 30802 + +/* Bison version string. */ +#define YYBISON_VERSION "3.8.2" + +/* Skeleton name. */ +#define YYSKELETON_NAME "<%= output.template_basename %>" + +/* Pure parsers. */ +#define YYPURE 1 + +/* Push parsers. */ +#define YYPUSH 0 + +/* Pull parsers. */ +#define YYPULL 1 + + +<%# b4_user_pre_prologue -%> +<%- if output.aux.prologue -%> +/* First part of user prologue. */ +#line <%= output.aux.prologue_first_lineno %> "<%= output.grammar_file_path %>" +<%= output.aux.prologue %> +#line [@oline@] [@ofile@] +<%- end -%> + +<%# b4_cast_define -%> +# ifndef YY_CAST +# ifdef __cplusplus +# define YY_CAST(Type, Val) static_cast<Type> (Val) +# define YY_REINTERPRET_CAST(Type, Val) reinterpret_cast<Type> (Val) +# else +# define YY_CAST(Type, Val) ((Type) (Val)) +# define YY_REINTERPRET_CAST(Type, Val) ((Type) (Val)) +# endif +# endif +<%# b4_null_define -%> +# ifndef YY_NULLPTR +# if defined __cplusplus +# if 201103L <= __cplusplus +# define YY_NULLPTR nullptr +# else +# define YY_NULLPTR 0 +# endif +# else +# define YY_NULLPTR ((void*)0) +# endif +# endif + +<%# b4_header_include_if -%> +<%- if output.include_header -%> +#include "<%= output.include_header %>" +<%- else -%> +/* Use api.header.include to #include this header + instead of duplicating it here. */ +<%= output.render_partial("bison/_yacc.h") %> +<%- end -%> +<%# b4_declare_symbol_enum -%> +/* Symbol kind. */ +enum yysymbol_kind_t +{ +<%= output.symbol_enum -%> +}; +typedef enum yysymbol_kind_t yysymbol_kind_t; + + + + +<%# b4_user_post_prologue -%> +<%# b4_c99_int_type_define -%> +#ifdef short +# undef short +#endif + +/* On compilers that do not define __PTRDIFF_MAX__ etc., make sure + <limits.h> and (if available) <stdint.h> are included + so that the code can choose integer types of a good width. */ + +#ifndef __PTRDIFF_MAX__ +# include <limits.h> /* INFRINGES ON USER NAME SPACE */ +# if defined __STDC_VERSION__ && 199901 <= __STDC_VERSION__ +# include <stdint.h> /* INFRINGES ON USER NAME SPACE */ +# define YY_STDINT_H +# endif +#endif + +/* Narrow types that promote to a signed type and that can represent a + signed or unsigned integer of at least N bits. In tables they can + save space and decrease cache pressure. Promoting to a signed type + helps avoid bugs in integer arithmetic. */ + +#ifdef __INT_LEAST8_MAX__ +typedef __INT_LEAST8_TYPE__ yytype_int8; +#elif defined YY_STDINT_H +typedef int_least8_t yytype_int8; +#else +typedef signed char yytype_int8; +#endif + +#ifdef __INT_LEAST16_MAX__ +typedef __INT_LEAST16_TYPE__ yytype_int16; +#elif defined YY_STDINT_H +typedef int_least16_t yytype_int16; +#else +typedef short yytype_int16; +#endif + +/* Work around bug in HP-UX 11.23, which defines these macros + incorrectly for preprocessor constants. This workaround can likely + be removed in 2023, as HPE has promised support for HP-UX 11.23 + (aka HP-UX 11i v2) only through the end of 2022; see Table 2 of + <https://h20195.www2.hpe.com/V2/getpdf.aspx/4AA4-7673ENW.pdf>. */ +#ifdef __hpux +# undef UINT_LEAST8_MAX +# undef UINT_LEAST16_MAX +# define UINT_LEAST8_MAX 255 +# define UINT_LEAST16_MAX 65535 +#endif + +#if defined __UINT_LEAST8_MAX__ && __UINT_LEAST8_MAX__ <= __INT_MAX__ +typedef __UINT_LEAST8_TYPE__ yytype_uint8; +#elif (!defined __UINT_LEAST8_MAX__ && defined YY_STDINT_H \ + && UINT_LEAST8_MAX <= INT_MAX) +typedef uint_least8_t yytype_uint8; +#elif !defined __UINT_LEAST8_MAX__ && UCHAR_MAX <= INT_MAX +typedef unsigned char yytype_uint8; +#else +typedef short yytype_uint8; +#endif + +#if defined __UINT_LEAST16_MAX__ && __UINT_LEAST16_MAX__ <= __INT_MAX__ +typedef __UINT_LEAST16_TYPE__ yytype_uint16; +#elif (!defined __UINT_LEAST16_MAX__ && defined YY_STDINT_H \ + && UINT_LEAST16_MAX <= INT_MAX) +typedef uint_least16_t yytype_uint16; +#elif !defined __UINT_LEAST16_MAX__ && USHRT_MAX <= INT_MAX +typedef unsigned short yytype_uint16; +#else +typedef int yytype_uint16; +#endif + +<%# b4_sizes_types_define -%> +#ifndef YYPTRDIFF_T +# if defined __PTRDIFF_TYPE__ && defined __PTRDIFF_MAX__ +# define YYPTRDIFF_T __PTRDIFF_TYPE__ +# define YYPTRDIFF_MAXIMUM __PTRDIFF_MAX__ +# elif defined PTRDIFF_MAX +# ifndef ptrdiff_t +# include <stddef.h> /* INFRINGES ON USER NAME SPACE */ +# endif +# define YYPTRDIFF_T ptrdiff_t +# define YYPTRDIFF_MAXIMUM PTRDIFF_MAX +# else +# define YYPTRDIFF_T long +# define YYPTRDIFF_MAXIMUM LONG_MAX +# endif +#endif + +#ifndef YYSIZE_T +# ifdef __SIZE_TYPE__ +# define YYSIZE_T __SIZE_TYPE__ +# elif defined size_t +# define YYSIZE_T size_t +# elif defined __STDC_VERSION__ && 199901 <= __STDC_VERSION__ +# include <stddef.h> /* INFRINGES ON USER NAME SPACE */ +# define YYSIZE_T size_t +# else +# define YYSIZE_T unsigned +# endif +#endif + +#define YYSIZE_MAXIMUM \ + YY_CAST (YYPTRDIFF_T, \ + (YYPTRDIFF_MAXIMUM < YY_CAST (YYSIZE_T, -1) \ + ? YYPTRDIFF_MAXIMUM \ + : YY_CAST (YYSIZE_T, -1))) + +#define YYSIZEOF(X) YY_CAST (YYPTRDIFF_T, sizeof (X)) + + +/* Stored state numbers (used for stacks). */ +typedef <%= output.int_type_for([output.yynstates - 1]) %> yy_state_t; + +/* State numbers in computations. */ +typedef int yy_state_fast_t; + +#ifndef YY_ +# if defined YYENABLE_NLS && YYENABLE_NLS +# if ENABLE_NLS +# include <libintl.h> /* INFRINGES ON USER NAME SPACE */ +# define YY_(Msgid) dgettext ("bison-runtime", Msgid) +# endif +# endif +# ifndef YY_ +# define YY_(Msgid) Msgid +# endif +#endif + + +<%# b4_attribute_define -%> +#ifndef YY_ATTRIBUTE_PURE +# if defined __GNUC__ && 2 < __GNUC__ + (96 <= __GNUC_MINOR__) +# define YY_ATTRIBUTE_PURE __attribute__ ((__pure__)) +# else +# define YY_ATTRIBUTE_PURE +# endif +#endif + +#ifndef YY_ATTRIBUTE_UNUSED +# if defined __GNUC__ && 2 < __GNUC__ + (7 <= __GNUC_MINOR__) +# define YY_ATTRIBUTE_UNUSED __attribute__ ((__unused__)) +# else +# define YY_ATTRIBUTE_UNUSED +# endif +#endif + +/* Suppress unused-variable warnings by "using" E. */ +#if ! defined lint || defined __GNUC__ +# define YY_USE(E) ((void) (E)) +#else +# define YY_USE(E) /* empty */ +#endif + +/* Suppress an incorrect diagnostic about yylval being uninitialized. */ +#if defined __GNUC__ && ! defined __ICC && 406 <= __GNUC__ * 100 + __GNUC_MINOR__ +# if __GNUC__ * 100 + __GNUC_MINOR__ < 407 +# define YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN \ + _Pragma ("GCC diagnostic push") \ + _Pragma ("GCC diagnostic ignored \"-Wuninitialized\"") +# else +# define YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN \ + _Pragma ("GCC diagnostic push") \ + _Pragma ("GCC diagnostic ignored \"-Wuninitialized\"") \ + _Pragma ("GCC diagnostic ignored \"-Wmaybe-uninitialized\"") +# endif +# define YY_IGNORE_MAYBE_UNINITIALIZED_END \ + _Pragma ("GCC diagnostic pop") +#else +# define YY_INITIAL_VALUE(Value) Value +#endif +#ifndef YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN +# define YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN +# define YY_IGNORE_MAYBE_UNINITIALIZED_END +#endif +#ifndef YY_INITIAL_VALUE +# define YY_INITIAL_VALUE(Value) /* Nothing. */ +#endif + +#if defined __cplusplus && defined __GNUC__ && ! defined __ICC && 6 <= __GNUC__ +# define YY_IGNORE_USELESS_CAST_BEGIN \ + _Pragma ("GCC diagnostic push") \ + _Pragma ("GCC diagnostic ignored \"-Wuseless-cast\"") +# define YY_IGNORE_USELESS_CAST_END \ + _Pragma ("GCC diagnostic pop") +#endif +#ifndef YY_IGNORE_USELESS_CAST_BEGIN +# define YY_IGNORE_USELESS_CAST_BEGIN +# define YY_IGNORE_USELESS_CAST_END +#endif + + +#define YY_ASSERT(E) ((void) (0 && (E))) + +#if 1 + +/* The parser invokes alloca or malloc; define the necessary symbols. */ + +# ifdef YYSTACK_USE_ALLOCA +# if YYSTACK_USE_ALLOCA +# ifdef __GNUC__ +# define YYSTACK_ALLOC __builtin_alloca +# elif defined __BUILTIN_VA_ARG_INCR +# include <alloca.h> /* INFRINGES ON USER NAME SPACE */ +# elif defined _AIX +# define YYSTACK_ALLOC __alloca +# elif defined _MSC_VER +# include <malloc.h> /* INFRINGES ON USER NAME SPACE */ +# define alloca _alloca +# else +# define YYSTACK_ALLOC alloca +# if ! defined _ALLOCA_H && ! defined EXIT_SUCCESS +# include <stdlib.h> /* INFRINGES ON USER NAME SPACE */ + /* Use EXIT_SUCCESS as a witness for stdlib.h. */ +# ifndef EXIT_SUCCESS +# define EXIT_SUCCESS 0 +# endif +# endif +# endif +# endif +# endif + +# ifdef YYSTACK_ALLOC + /* Pacify GCC's 'empty if-body' warning. */ +# define YYSTACK_FREE(Ptr) do { /* empty */; } while (0) +# ifndef YYSTACK_ALLOC_MAXIMUM + /* The OS might guarantee only one guard page at the bottom of the stack, + and a page size can be as small as 4096 bytes. So we cannot safely + invoke alloca (N) if N exceeds 4096. Use a slightly smaller number + to allow for a few compiler-allocated temporary stack slots. */ +# define YYSTACK_ALLOC_MAXIMUM 4032 /* reasonable circa 2006 */ +# endif +# else +# define YYSTACK_ALLOC YYMALLOC +# define YYSTACK_FREE YYFREE +# ifndef YYSTACK_ALLOC_MAXIMUM +# define YYSTACK_ALLOC_MAXIMUM YYSIZE_MAXIMUM +# endif +# if (defined __cplusplus && ! defined EXIT_SUCCESS \ + && ! ((defined YYMALLOC || defined malloc) \ + && (defined YYFREE || defined free))) +# include <stdlib.h> /* INFRINGES ON USER NAME SPACE */ +# ifndef EXIT_SUCCESS +# define EXIT_SUCCESS 0 +# endif +# endif +# ifndef YYMALLOC +# define YYMALLOC malloc +# if ! defined malloc && ! defined EXIT_SUCCESS +void *malloc (YYSIZE_T); /* INFRINGES ON USER NAME SPACE */ +# endif +# endif +# ifndef YYFREE +# define YYFREE free +# if ! defined free && ! defined EXIT_SUCCESS +void free (void *); /* INFRINGES ON USER NAME SPACE */ +# endif +# endif +# endif +#endif /* 1 */ + +#if (! defined yyoverflow \ + && (! defined __cplusplus \ + || (defined YYLTYPE_IS_TRIVIAL && YYLTYPE_IS_TRIVIAL \ + && defined YYSTYPE_IS_TRIVIAL && YYSTYPE_IS_TRIVIAL))) + +/* A type that is properly aligned for any stack member. */ +union yyalloc +{ + yy_state_t yyss_alloc; + YYSTYPE yyvs_alloc; + YYLTYPE yyls_alloc; +}; + +/* The size of the maximum gap between one aligned stack and the next. */ +# define YYSTACK_GAP_MAXIMUM (YYSIZEOF (union yyalloc) - 1) + +/* The size of an array large to enough to hold all stacks, each with + N elements. */ +# define YYSTACK_BYTES(N) \ + ((N) * (YYSIZEOF (yy_state_t) + YYSIZEOF (YYSTYPE) \ + + YYSIZEOF (YYLTYPE)) \ + + 2 * YYSTACK_GAP_MAXIMUM) + +# define YYCOPY_NEEDED 1 + +/* Relocate STACK from its old location to the new one. The + local variables YYSIZE and YYSTACKSIZE give the old and new number of + elements in the stack, and YYPTR gives the new location of the + stack. Advance YYPTR to a properly aligned location for the next + stack. */ +# define YYSTACK_RELOCATE(Stack_alloc, Stack) \ + do \ + { \ + YYPTRDIFF_T yynewbytes; \ + YYCOPY (&yyptr->Stack_alloc, Stack, yysize); \ + Stack = &yyptr->Stack_alloc; \ + yynewbytes = yystacksize * YYSIZEOF (*Stack) + YYSTACK_GAP_MAXIMUM; \ + yyptr += yynewbytes / YYSIZEOF (*yyptr); \ + } \ + while (0) + +#endif + +#if defined YYCOPY_NEEDED && YYCOPY_NEEDED +/* Copy COUNT objects from SRC to DST. The source and destination do + not overlap. */ +# ifndef YYCOPY +# if defined __GNUC__ && 1 < __GNUC__ +# define YYCOPY(Dst, Src, Count) \ + __builtin_memcpy (Dst, Src, YY_CAST (YYSIZE_T, (Count)) * sizeof (*(Src))) +# else +# define YYCOPY(Dst, Src, Count) \ + do \ + { \ + YYPTRDIFF_T yyi; \ + for (yyi = 0; yyi < (Count); yyi++) \ + (Dst)[yyi] = (Src)[yyi]; \ + } \ + while (0) +# endif +# endif +#endif /* !YYCOPY_NEEDED */ + +/* YYFINAL -- State number of the termination state. */ +#define YYFINAL <%= output.yyfinal %> +/* YYLAST -- Last index in YYTABLE. */ +#define YYLAST <%= output.yylast %> + +/* YYNTOKENS -- Number of terminals. */ +#define YYNTOKENS <%= output.yyntokens %> +/* YYNNTS -- Number of nonterminals. */ +#define YYNNTS <%= output.yynnts %> +/* YYNRULES -- Number of rules. */ +#define YYNRULES <%= output.yynrules %> +/* YYNSTATES -- Number of states. */ +#define YYNSTATES <%= output.yynstates %> + +/* YYMAXUTOK -- Last valid token kind. */ +#define YYMAXUTOK <%= output.yymaxutok %> + + +/* YYTRANSLATE(TOKEN-NUM) -- Symbol number corresponding to TOKEN-NUM + as returned by yylex, with out-of-bounds checking. */ +#define YYTRANSLATE(YYX) \ + (0 <= (YYX) && (YYX) <= YYMAXUTOK \ + ? YY_CAST (yysymbol_kind_t, yytranslate[YYX]) \ + : YYSYMBOL_YYUNDEF) + +/* YYTRANSLATE[TOKEN-NUM] -- Symbol number corresponding to TOKEN-NUM + as returned by yylex. */ +static const <%= output.int_type_for(output.context.yytranslate) %> yytranslate[] = +{ +<%= output.yytranslate %> +}; + +<%- if output.error_recovery -%> +/* YYTRANSLATE_INVERTED[SYMBOL-NUM] -- Token number corresponding to SYMBOL-NUM */ +static const <%= output.int_type_for(output.context.yytranslate_inverted) %> yytranslate_inverted[] = +{ +<%= output.yytranslate_inverted %> +}; +<%- end -%> +#if YYDEBUG +/* YYRLINE[YYN] -- Source line where rule number YYN was defined. */ +static const <%= output.int_type_for(output.context.yyrline) %> yyrline[] = +{ +<%= output.yyrline %> +}; +#endif + +/** Accessing symbol of state STATE. */ +#define YY_ACCESSING_SYMBOL(State) YY_CAST (yysymbol_kind_t, yystos[State]) + +#if 1 +/* The user-facing name of the symbol whose (internal) number is + YYSYMBOL. No bounds checking. */ +static const char *yysymbol_name (yysymbol_kind_t yysymbol) YY_ATTRIBUTE_UNUSED; + +/* YYTNAME[SYMBOL-NUM] -- String name of the symbol SYMBOL-NUM. + First, the terminals, then, starting at YYNTOKENS, nonterminals. */ +static const char *const yytname[] = +{ +<%= output.yytname %> +}; + +static const char * +yysymbol_name (yysymbol_kind_t yysymbol) +{ + return yytname[yysymbol]; +} +#endif + +#define YYPACT_NINF (<%= output.yypact_ninf %>) + +#define yypact_value_is_default(Yyn) \ + <%= output.table_value_equals(output.context.yypact, "Yyn", output.yypact_ninf, "YYPACT_NINF") %> + +#define YYTABLE_NINF (<%= output.yytable_ninf %>) + +#define yytable_value_is_error(Yyn) \ + <%= output.table_value_equals(output.context.yytable, "Yyn", output.yytable_ninf, "YYTABLE_NINF") %> + +<%# b4_parser_tables_define -%> +/* YYPACT[STATE-NUM] -- Index in YYTABLE of the portion describing + STATE-NUM. */ +static const <%= output.int_type_for(output.context.yypact) %> yypact[] = +{ +<%= output.int_array_to_string(output.context.yypact) %> +}; + +/* YYDEFACT[STATE-NUM] -- Default reduction number in state STATE-NUM. + Performed when YYTABLE does not specify something else to do. Zero + means the default is an error. */ +static const <%= output.int_type_for(output.context.yydefact) %> yydefact[] = +{ +<%= output.int_array_to_string(output.context.yydefact) %> +}; + +/* YYPGOTO[NTERM-NUM]. */ +static const <%= output.int_type_for(output.context.yypgoto) %> yypgoto[] = +{ +<%= output.int_array_to_string(output.context.yypgoto) %> +}; + +/* YYDEFGOTO[NTERM-NUM]. */ +static const <%= output.int_type_for(output.context.yydefgoto) %> yydefgoto[] = +{ +<%= output.int_array_to_string(output.context.yydefgoto) %> +}; + +/* YYTABLE[YYPACT[STATE-NUM]] -- What to do in state STATE-NUM. If + positive, shift that token. If negative, reduce the rule whose + number is the opposite. If YYTABLE_NINF, syntax error. */ +static const <%= output.int_type_for(output.context.yytable) %> yytable[] = +{ +<%= output.int_array_to_string(output.context.yytable) %> +}; + +static const <%= output.int_type_for(output.context.yycheck) %> yycheck[] = +{ +<%= output.int_array_to_string(output.context.yycheck) %> +}; + +/* YYSTOS[STATE-NUM] -- The symbol kind of the accessing symbol of + state STATE-NUM. */ +static const <%= output.int_type_for(output.context.yystos) %> yystos[] = +{ +<%= output.int_array_to_string(output.context.yystos) %> +}; + +/* YYR1[RULE-NUM] -- Symbol kind of the left-hand side of rule RULE-NUM. */ +static const <%= output.int_type_for(output.context.yyr1) %> yyr1[] = +{ +<%= output.int_array_to_string(output.context.yyr1) %> +}; + +/* YYR2[RULE-NUM] -- Number of symbols on the right-hand side of rule RULE-NUM. */ +static const <%= output.int_type_for(output.context.yyr2) %> yyr2[] = +{ +<%= output.int_array_to_string(output.context.yyr2) %> +}; + + +enum { YYENOMEM = -2 }; + +#define yyerrok (yyerrstatus = 0) +#define yyclearin (yychar = YYEMPTY) + +#define YYACCEPT goto yyacceptlab +#define YYABORT goto yyabortlab +#define YYERROR goto yyerrorlab +#define YYNOMEM goto yyexhaustedlab + + +#define YYRECOVERING() (!!yyerrstatus) + +#define YYBACKUP(Token, Value) \ + do \ + if (yychar == YYEMPTY) \ + { \ + yychar = (Token); \ + yylval = (Value); \ + YYPOPSTACK (yylen); \ + yystate = *yyssp; \ + goto yybackup; \ + } \ + else \ + { \ + yyerror (<%= output.yyerror_args %>, YY_("syntax error: cannot back up")); \ + YYERROR; \ + } \ + while (0) + +/* Backward compatibility with an undocumented macro. + Use YYerror or YYUNDEF. */ +#define YYERRCODE YYUNDEF + +<%# b4_yylloc_default_define -%> +/* YYLLOC_DEFAULT -- Set CURRENT to span from RHS[1] to RHS[N]. + If N is 0, then set CURRENT to the empty location which ends + the previous symbol: RHS[0] (always defined). */ + +#ifndef YYLLOC_DEFAULT +# define YYLLOC_DEFAULT(Current, Rhs, N) \ + do \ + if (N) \ + { \ + (Current).first_line = YYRHSLOC (Rhs, 1).first_line; \ + (Current).first_column = YYRHSLOC (Rhs, 1).first_column; \ + (Current).last_line = YYRHSLOC (Rhs, N).last_line; \ + (Current).last_column = YYRHSLOC (Rhs, N).last_column; \ + } \ + else \ + { \ + (Current).first_line = (Current).last_line = \ + YYRHSLOC (Rhs, 0).last_line; \ + (Current).first_column = (Current).last_column = \ + YYRHSLOC (Rhs, 0).last_column; \ + } \ + while (0) +#endif + +#define YYRHSLOC(Rhs, K) ((Rhs)[K]) + + +/* Enable debugging if requested. */ +#if YYDEBUG + +# ifndef YYFPRINTF +# include <stdio.h> /* INFRINGES ON USER NAME SPACE */ +# define YYFPRINTF fprintf +# endif + +# define YYDPRINTF(Args) \ +do { \ + if (yydebug) \ + YYFPRINTF Args; \ +} while (0) + + +<%# b4_yylocation_print_define -%> +/* YYLOCATION_PRINT -- Print the location on the stream. + This macro was not mandated originally: define only if we know + we won't break user code: when these are the locations we know. */ + +# ifndef YYLOCATION_PRINT + +# if defined YY_LOCATION_PRINT + + /* Temporary convenience wrapper in case some people defined the + undocumented and private YY_LOCATION_PRINT macros. */ +# define YYLOCATION_PRINT(File, Loc<%= output.user_args %>) YY_LOCATION_PRINT(File, *(Loc)<%= output.user_args %>) + +# elif defined YYLTYPE_IS_TRIVIAL && YYLTYPE_IS_TRIVIAL + +/* Print *YYLOCP on YYO. Private, do not rely on its existence. */ + +YY_ATTRIBUTE_UNUSED +static int +yy_location_print_ (FILE *yyo, YYLTYPE const * const yylocp) +{ + int res = 0; + int end_col = 0 != yylocp->last_column ? yylocp->last_column - 1 : 0; + if (0 <= yylocp->first_line) + { + res += YYFPRINTF (yyo, "%d", yylocp->first_line); + if (0 <= yylocp->first_column) + res += YYFPRINTF (yyo, ".%d", yylocp->first_column); + } + if (0 <= yylocp->last_line) + { + if (yylocp->first_line < yylocp->last_line) + { + res += YYFPRINTF (yyo, "-%d", yylocp->last_line); + if (0 <= end_col) + res += YYFPRINTF (yyo, ".%d", end_col); + } + else if (0 <= end_col && yylocp->first_column < end_col) + res += YYFPRINTF (yyo, "-%d", end_col); + } + return res; +} + +# define YYLOCATION_PRINT yy_location_print_ + + /* Temporary convenience wrapper in case some people defined the + undocumented and private YY_LOCATION_PRINT macros. */ +# define YY_LOCATION_PRINT(File, Loc<%= output.user_args %>) YYLOCATION_PRINT(File, &(Loc)<%= output.user_args %>) + +# else + +# define YYLOCATION_PRINT(File, Loc<%= output.user_args %>) ((void) 0) + /* Temporary convenience wrapper in case some people defined the + undocumented and private YY_LOCATION_PRINT macros. */ +# define YY_LOCATION_PRINT YYLOCATION_PRINT + +# endif +# endif /* !defined YYLOCATION_PRINT */ + + +# define YY_SYMBOL_PRINT(Title, Kind, Value, Location<%= output.user_args %>) \ +do { \ + if (yydebug) \ + { \ + YYFPRINTF (stderr, "%s ", Title); \ + yy_symbol_print (stderr, \ + Kind, Value, Location<%= output.user_args %>); \ + YYFPRINTF (stderr, "\n"); \ + } \ +} while (0) + + +<%# b4_yy_symbol_print_define -%> +/*-----------------------------------. +| Print this symbol's value on YYO. | +`-----------------------------------*/ + +static void +yy_symbol_value_print (FILE *yyo, + yysymbol_kind_t yykind, YYSTYPE const * const yyvaluep, YYLTYPE const * const yylocationp<%= output.user_formals %>) +{ + FILE *yyoutput = yyo; +<%= output.parse_param_use("yyoutput", "yylocationp") %> + if (!yyvaluep) + return; + YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN +<%# b4_symbol_actions(printer) -%> +switch (yykind) + { +<%= output.symbol_actions_for_printer -%> + default: + break; + } + YY_IGNORE_MAYBE_UNINITIALIZED_END +} + + +/*---------------------------. +| Print this symbol on YYO. | +`---------------------------*/ + +static void +yy_symbol_print (FILE *yyo, + yysymbol_kind_t yykind, YYSTYPE const * const yyvaluep, YYLTYPE const * const yylocationp<%= output.user_formals %>) +{ + YYFPRINTF (yyo, "%s %s (", + yykind < YYNTOKENS ? "token" : "nterm", yysymbol_name (yykind)); + + YYLOCATION_PRINT (yyo, yylocationp<%= output.user_args %>); + YYFPRINTF (yyo, ": "); + yy_symbol_value_print (yyo, yykind, yyvaluep, yylocationp<%= output.user_args %>); + YYFPRINTF (yyo, ")"); +} + +/*------------------------------------------------------------------. +| yy_stack_print -- Print the state stack from its BOTTOM up to its | +| TOP (included). | +`------------------------------------------------------------------*/ + +static void +yy_stack_print (yy_state_t *yybottom, yy_state_t *yytop<%= output.user_formals %>) +{ + YYFPRINTF (stderr, "Stack now"); + for (; yybottom <= yytop; yybottom++) + { + int yybot = *yybottom; + YYFPRINTF (stderr, " %d", yybot); + } + YYFPRINTF (stderr, "\n"); +} + +# define YY_STACK_PRINT(Bottom, Top<%= output.user_args %>) \ +do { \ + if (yydebug) \ + yy_stack_print ((Bottom), (Top)<%= output.user_args %>); \ +} while (0) + + +/*------------------------------------------------. +| Report that the YYRULE is going to be reduced. | +`------------------------------------------------*/ + +static void +yy_reduce_print (yy_state_t *yyssp, YYSTYPE *yyvsp, YYLTYPE *yylsp, + int yyrule<%= output.user_formals %>) +{ + int yylno = yyrline[yyrule]; + int yynrhs = yyr2[yyrule]; + int yyi; + YYFPRINTF (stderr, "Reducing stack by rule %d (line %d):\n", + yyrule - 1, yylno); + /* The symbols being reduced. */ + for (yyi = 0; yyi < yynrhs; yyi++) + { + YYFPRINTF (stderr, " $%d = ", yyi + 1); + yy_symbol_print (stderr, + YY_ACCESSING_SYMBOL (+yyssp[yyi + 1 - yynrhs]), + &yyvsp[(yyi + 1) - (yynrhs)], + &(yylsp[(yyi + 1) - (yynrhs)])<%= output.user_args %>); + YYFPRINTF (stderr, "\n"); + } +} + +# define YY_REDUCE_PRINT(Rule<%= output.user_args %>) \ +do { \ + if (yydebug) \ + yy_reduce_print (yyssp, yyvsp, yylsp, Rule<%= output.user_args %>); \ +} while (0) + +/* Nonzero means print parse trace. It is left uninitialized so that + multiple parsers can coexist. */ +#ifndef yydebug +int yydebug; +#endif +#else /* !YYDEBUG */ +# define YYDPRINTF(Args) ((void) 0) +# define YY_SYMBOL_PRINT(Title, Kind, Value, Location<%= output.user_args %>) +# define YY_STACK_PRINT(Bottom, Top<%= output.user_args %>) +# define YY_REDUCE_PRINT(Rule<%= output.user_args %>) +#endif /* !YYDEBUG */ + + +/* YYINITDEPTH -- initial size of the parser's stacks. */ +#ifndef YYINITDEPTH +# define YYINITDEPTH 200 +#endif + +/* YYMAXDEPTH -- maximum size the stacks can grow to (effective only + if the built-in stack extension method is used). + + Do not make this value too large; the results are undefined if + YYSTACK_ALLOC_MAXIMUM < YYSTACK_BYTES (YYMAXDEPTH) + evaluated with infinite-precision integer arithmetic. */ + +#ifndef YYMAXDEPTH +# define YYMAXDEPTH 10000 +#endif + + +/* Context of a parse error. */ +typedef struct +{ + yy_state_t *yyssp; + yysymbol_kind_t yytoken; + YYLTYPE *yylloc; +} yypcontext_t; + +/* Put in YYARG at most YYARGN of the expected tokens given the + current YYCTX, and return the number of tokens stored in YYARG. If + YYARG is null, return the number of expected tokens (guaranteed to + be less than YYNTOKENS). Return YYENOMEM on memory exhaustion. + Return 0 if there are more than YYARGN expected tokens, yet fill + YYARG up to YYARGN. */ +static int +yypcontext_expected_tokens (const yypcontext_t *yyctx, + yysymbol_kind_t yyarg[], int yyargn) +{ + /* Actual size of YYARG. */ + int yycount = 0; + int yyn = yypact[+*yyctx->yyssp]; + if (!yypact_value_is_default (yyn)) + { + /* Start YYX at -YYN if negative to avoid negative indexes in + YYCHECK. In other words, skip the first -YYN actions for + this state because they are default actions. */ + int yyxbegin = yyn < 0 ? -yyn : 0; + /* Stay within bounds of both yycheck and yytname. */ + int yychecklim = YYLAST - yyn + 1; + int yyxend = yychecklim < YYNTOKENS ? yychecklim : YYNTOKENS; + int yyx; + for (yyx = yyxbegin; yyx < yyxend; ++yyx) + if (yycheck[yyx + yyn] == yyx && yyx != YYSYMBOL_YYerror + && !yytable_value_is_error (yytable[yyx + yyn])) + { + if (!yyarg) + ++yycount; + else if (yycount == yyargn) + return 0; + else + yyarg[yycount++] = YY_CAST (yysymbol_kind_t, yyx); + } + } + if (yyarg && yycount == 0 && 0 < yyargn) + yyarg[0] = YYSYMBOL_YYEMPTY; + return yycount; +} + + + + +#ifndef yystrlen +# if defined __GLIBC__ && defined _STRING_H +# define yystrlen(S) (YY_CAST (YYPTRDIFF_T, strlen (S))) +# else +/* Return the length of YYSTR. */ +static YYPTRDIFF_T +yystrlen (const char *yystr) +{ + YYPTRDIFF_T yylen; + for (yylen = 0; yystr[yylen]; yylen++) + continue; + return yylen; +} +# endif +#endif + +#ifndef yystpcpy +# if defined __GLIBC__ && defined _STRING_H && defined _GNU_SOURCE +# define yystpcpy stpcpy +# else +/* Copy YYSRC to YYDEST, returning the address of the terminating '\0' in + YYDEST. */ +static char * +yystpcpy (char *yydest, const char *yysrc) +{ + char *yyd = yydest; + const char *yys = yysrc; + + while ((*yyd++ = *yys++) != '\0') + continue; + + return yyd - 1; +} +# endif +#endif + +#ifndef yytnamerr +/* Copy to YYRES the contents of YYSTR after stripping away unnecessary + quotes and backslashes, so that it's suitable for yyerror. The + heuristic is that double-quoting is unnecessary unless the string + contains an apostrophe, a comma, or backslash (other than + backslash-backslash). YYSTR is taken from yytname. If YYRES is + null, do not copy; instead, return the length of what the result + would have been. */ +static YYPTRDIFF_T +yytnamerr (char *yyres, const char *yystr) +{ + if (*yystr == '"') + { + YYPTRDIFF_T yyn = 0; + char const *yyp = yystr; + for (;;) + switch (*++yyp) + { + case '\'': + case ',': + goto do_not_strip_quotes; + + case '\\': + if (*++yyp != '\\') + goto do_not_strip_quotes; + else + goto append; + + append: + default: + if (yyres) + yyres[yyn] = *yyp; + yyn++; + break; + + case '"': + if (yyres) + yyres[yyn] = '\0'; + return yyn; + } + do_not_strip_quotes: ; + } + + if (yyres) + return yystpcpy (yyres, yystr) - yyres; + else + return yystrlen (yystr); +} +#endif + + +static int +yy_syntax_error_arguments (const yypcontext_t *yyctx, + yysymbol_kind_t yyarg[], int yyargn) +{ + /* Actual size of YYARG. */ + int yycount = 0; + /* There are many possibilities here to consider: + - If this state is a consistent state with a default action, then + the only way this function was invoked is if the default action + is an error action. In that case, don't check for expected + tokens because there are none. + - The only way there can be no lookahead present (in yychar) is if + this state is a consistent state with a default action. Thus, + detecting the absence of a lookahead is sufficient to determine + that there is no unexpected or expected token to report. In that + case, just report a simple "syntax error". + - Don't assume there isn't a lookahead just because this state is a + consistent state with a default action. There might have been a + previous inconsistent state, consistent state with a non-default + action, or user semantic action that manipulated yychar. + - Of course, the expected token list depends on states to have + correct lookahead information, and it depends on the parser not + to perform extra reductions after fetching a lookahead from the + scanner and before detecting a syntax error. Thus, state merging + (from LALR or IELR) and default reductions corrupt the expected + token list. However, the list is correct for canonical LR with + one exception: it will still contain any token that will not be + accepted due to an error action in a later state. + */ + if (yyctx->yytoken != YYSYMBOL_YYEMPTY) + { + int yyn; + if (yyarg) + yyarg[yycount] = yyctx->yytoken; + ++yycount; + yyn = yypcontext_expected_tokens (yyctx, + yyarg ? yyarg + 1 : yyarg, yyargn - 1); + if (yyn == YYENOMEM) + return YYENOMEM; + else + yycount += yyn; + } + return yycount; +} + +/* Copy into *YYMSG, which is of size *YYMSG_ALLOC, an error message + about the unexpected token YYTOKEN for the state stack whose top is + YYSSP. + + Return 0 if *YYMSG was successfully written. Return -1 if *YYMSG is + not large enough to hold the message. In that case, also set + *YYMSG_ALLOC to the required number of bytes. Return YYENOMEM if the + required number of bytes is too large to store. */ +static int +yysyntax_error (YYPTRDIFF_T *yymsg_alloc, char **yymsg, + const yypcontext_t *yyctx<%= output.user_formals %>) +{ + enum { YYARGS_MAX = 5 }; + /* Internationalized format string. */ + const char *yyformat = YY_NULLPTR; + /* Arguments of yyformat: reported tokens (one for the "unexpected", + one per "expected"). */ + yysymbol_kind_t yyarg[YYARGS_MAX]; + /* Cumulated lengths of YYARG. */ + YYPTRDIFF_T yysize = 0; + + /* Actual size of YYARG. */ + int yycount = yy_syntax_error_arguments (yyctx, yyarg, YYARGS_MAX); + if (yycount == YYENOMEM) + return YYENOMEM; + + switch (yycount) + { +#define YYCASE_(N, S) \ + case N: \ + yyformat = S; \ + break + default: /* Avoid compiler warnings. */ + YYCASE_(0, YY_("syntax error")); + YYCASE_(1, YY_("syntax error, unexpected %s")); + YYCASE_(2, YY_("syntax error, unexpected %s, expecting %s")); + YYCASE_(3, YY_("syntax error, unexpected %s, expecting %s or %s")); + YYCASE_(4, YY_("syntax error, unexpected %s, expecting %s or %s or %s")); + YYCASE_(5, YY_("syntax error, unexpected %s, expecting %s or %s or %s or %s")); +#undef YYCASE_ + } + + /* Compute error message size. Don't count the "%s"s, but reserve + room for the terminator. */ + yysize = yystrlen (yyformat) - 2 * yycount + 1; + { + int yyi; + for (yyi = 0; yyi < yycount; ++yyi) + { + YYPTRDIFF_T yysize1 + = yysize + yytnamerr (YY_NULLPTR, yytname[yyarg[yyi]]); + if (yysize <= yysize1 && yysize1 <= YYSTACK_ALLOC_MAXIMUM) + yysize = yysize1; + else + return YYENOMEM; + } + } + + if (*yymsg_alloc < yysize) + { + *yymsg_alloc = 2 * yysize; + if (! (yysize <= *yymsg_alloc + && *yymsg_alloc <= YYSTACK_ALLOC_MAXIMUM)) + *yymsg_alloc = YYSTACK_ALLOC_MAXIMUM; + return -1; + } + + /* Avoid sprintf, as that infringes on the user's name space. + Don't have undefined behavior even if the translation + produced a string with the wrong number of "%s"s. */ + { + char *yyp = *yymsg; + int yyi = 0; + while ((*yyp = *yyformat) != '\0') + if (*yyp == '%' && yyformat[1] == 's' && yyi < yycount) + { + yyp += yytnamerr (yyp, yytname[yyarg[yyi++]]); + yyformat += 2; + } + else + { + ++yyp; + ++yyformat; + } + } + return 0; +} + +<%# b4_yydestruct_define %> +/*-----------------------------------------------. +| Release the memory associated to this symbol. | +`-----------------------------------------------*/ + +static void +yydestruct (const char *yymsg, + yysymbol_kind_t yykind, YYSTYPE *yyvaluep, YYLTYPE *yylocationp<%= output.user_formals %>) +{ +<%= output.parse_param_use("yyvaluep", "yylocationp") %> + if (!yymsg) + yymsg = "Deleting"; + YY_SYMBOL_PRINT (yymsg, yykind, yyvaluep, yylocationp<%= output.user_args %>); + + YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN + switch (yykind) + { +<%= output.symbol_actions_for_destructor -%> + default: + break; + } + YY_IGNORE_MAYBE_UNINITIALIZED_END +} + + + +<%- if output.error_recovery -%> +#ifndef YYMAXREPAIR +# define YYMAXREPAIR(<%= output.parse_param_name %>) (3) +#endif + +#ifndef YYERROR_RECOVERY_ENABLED +# define YYERROR_RECOVERY_ENABLED(<%= output.parse_param_name %>) (1) +#endif + +enum yy_repair_type { + insert, + delete, + shift, +}; + +struct yy_repair { + enum yy_repair_type type; + yysymbol_kind_t term; +}; +typedef struct yy_repair yy_repair; + +struct yy_repairs { + /* For debug */ + int id; + /* For breadth-first traversing */ + struct yy_repairs *next; + YYPTRDIFF_T stack_length; + /* Bottom of states */ + yy_state_t *states; + /* Top of states */ + yy_state_t *state; + /* repair length */ + int repair_length; + /* */ + struct yy_repairs *prev_repair; + struct yy_repair repair; +}; +typedef struct yy_repairs yy_repairs; + +struct yy_term { + yysymbol_kind_t kind; + YYSTYPE value; + YYLTYPE location; +}; +typedef struct yy_term yy_term; + +struct yy_repair_terms { + int id; + int length; + yy_term terms[]; +}; +typedef struct yy_repair_terms yy_repair_terms; + +static void +yy_error_token_initialize (yysymbol_kind_t yykind, YYSTYPE * const yyvaluep, YYLTYPE * const yylocationp<%= output.user_formals %>) +{ + YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN +switch (yykind) + { +<%= output.symbol_actions_for_error_token -%> + default: + break; + } + YY_IGNORE_MAYBE_UNINITIALIZED_END +} + +static yy_repair_terms * +yy_create_repair_terms(yy_repairs *reps<%= output.user_formals %>) +{ + yy_repairs *r = reps; + yy_repair_terms *rep_terms; + int count = 0; + + while (r->prev_repair) + { + count++; + r = r->prev_repair; + } + + rep_terms = (yy_repair_terms *) YYMALLOC (sizeof (yy_repair_terms) + sizeof (yy_term) * count); + rep_terms->id = reps->id; + rep_terms->length = count; + + r = reps; + while (r->prev_repair) + { + rep_terms->terms[count-1].kind = r->repair.term; + count--; + r = r->prev_repair; + } + + return rep_terms; +} + +static void +yy_print_repairs(yy_repairs *reps<%= output.user_formals %>) +{ + yy_repairs *r = reps; + + YYDPRINTF ((stderr, + "id: %d, repair_length: %d, repair_state: %d, prev_repair_id: %d\n", + reps->id, reps->repair_length, *reps->state, reps->prev_repair->id)); + + while (r->prev_repair) + { + YYDPRINTF ((stderr, "%s ", yysymbol_name (r->repair.term))); + r = r->prev_repair; + } + + YYDPRINTF ((stderr, "\n")); +} + +static void +yy_print_repair_terms(yy_repair_terms *rep_terms<%= output.user_formals %>) +{ + for (int i = 0; i < rep_terms->length; i++) + YYDPRINTF ((stderr, "%s ", yysymbol_name (rep_terms->terms[i].kind))); + + YYDPRINTF ((stderr, "\n")); +} + +static void +yy_free_repairs(yy_repairs *reps<%= output.user_formals %>) +{ + while (reps) + { + yy_repairs *r = reps; + reps = reps->next; + YYFREE (r->states); + YYFREE (r); + } +} + +static int +yy_process_repairs(yy_repairs *reps, yysymbol_kind_t token) +{ + int yyn; + int yystate = *reps->state; + int yylen = 0; + yysymbol_kind_t yytoken = token; + + goto yyrecover_backup; + +yyrecover_newstate: + // TODO: check reps->stack_length + reps->state += 1; + *reps->state = (yy_state_t) yystate; + + +yyrecover_backup: + yyn = yypact[yystate]; + if (yypact_value_is_default (yyn)) + goto yyrecover_default; + + /* "Reading a token" */ + if (yytoken == YYSYMBOL_YYEMPTY) + return 1; + + yyn += yytoken; + if (yyn < 0 || YYLAST < yyn || yycheck[yyn] != yytoken) + goto yyrecover_default; + yyn = yytable[yyn]; + if (yyn <= 0) + { + if (yytable_value_is_error (yyn)) + goto yyrecover_errlab; + yyn = -yyn; + goto yyrecover_reduce; + } + + /* shift */ + yystate = yyn; + yytoken = YYSYMBOL_YYEMPTY; + goto yyrecover_newstate; + + +yyrecover_default: + yyn = yydefact[yystate]; + if (yyn == 0) + goto yyrecover_errlab; + goto yyrecover_reduce; + + +yyrecover_reduce: + yylen = yyr2[yyn]; + /* YYPOPSTACK */ + reps->state -= yylen; + yylen = 0; + + { + const int yylhs = yyr1[yyn] - YYNTOKENS; + const int yyi = yypgoto[yylhs] + *reps->state; + yystate = (0 <= yyi && yyi <= YYLAST && yycheck[yyi] == *reps->state + ? yytable[yyi] + : yydefgoto[yylhs]); + } + + goto yyrecover_newstate; + +yyrecover_errlab: + return 0; +} + +static yy_repair_terms * +yyrecover(yy_state_t *yyss, yy_state_t *yyssp, int yychar<%= output.user_formals %>) +{ + yysymbol_kind_t yytoken = YYTRANSLATE (yychar); + yy_repair_terms *rep_terms = YY_NULLPTR; + int count = 0; + + yy_repairs *head = (yy_repairs *) YYMALLOC (sizeof (yy_repairs)); + yy_repairs *current = head; + yy_repairs *tail = head; + YYPTRDIFF_T stack_length = yyssp - yyss + 1; + + head->id = count; + head->next = 0; + head->stack_length = stack_length; + head->states = (yy_state_t *) YYMALLOC (sizeof (yy_state_t) * (stack_length)); + head->state = head->states + (yyssp - yyss); + YYCOPY (head->states, yyss, stack_length); + head->repair_length = 0; + head->prev_repair = 0; + + stack_length = (stack_length * 2 > 100) ? (stack_length * 2) : 100; + count++; + + while (current) + { + int yystate = *current->state; + int yyn = yypact[yystate]; + /* See also: yypcontext_expected_tokens */ + if (!yypact_value_is_default (yyn)) + { + int yyxbegin = yyn < 0 ? -yyn : 0; + int yychecklim = YYLAST - yyn + 1; + int yyxend = yychecklim < YYNTOKENS ? yychecklim : YYNTOKENS; + int yyx; + for (yyx = yyxbegin; yyx < yyxend; ++yyx) + { + if (yyx != YYSYMBOL_YYerror) + { + if (current->repair_length + 1 > YYMAXREPAIR(<%= output.parse_param_name %>)) + continue; + + yy_repairs *new = (yy_repairs *) YYMALLOC (sizeof (yy_repairs)); + new->id = count; + new->next = 0; + new->stack_length = stack_length; + new->states = (yy_state_t *) YYMALLOC (sizeof (yy_state_t) * (stack_length)); + new->state = new->states + (current->state - current->states); + YYCOPY (new->states, current->states, current->state - current->states + 1); + new->repair_length = current->repair_length + 1; + new->prev_repair = current; + new->repair.type = insert; + new->repair.term = (yysymbol_kind_t) yyx; + + /* Process PDA assuming next token is yyx */ + if (! yy_process_repairs (new, yyx)) + { + YYFREE (new); + continue; + } + + tail->next = new; + tail = new; + count++; + + if (yyx == yytoken) + { + rep_terms = yy_create_repair_terms (current<%= output.user_args %>); + YYDPRINTF ((stderr, "repair_terms found. id: %d, length: %d\n", rep_terms->id, rep_terms->length)); + yy_print_repairs (current<%= output.user_args %>); + yy_print_repair_terms (rep_terms<%= output.user_args %>); + + goto done; + } + + YYDPRINTF ((stderr, + "New repairs is enqueued. count: %d, yystate: %d, yyx: %d\n", + count, yystate, yyx)); + yy_print_repairs (new<%= output.user_args %>); + } + } + } + + current = current->next; + } + +done: + + yy_free_repairs(head<%= output.user_args %>); + + if (!rep_terms) + { + YYDPRINTF ((stderr, "repair_terms not found\n")); + } + + return rep_terms; +} +<%- end -%> + + + +/*----------. +| yyparse. | +`----------*/ + +int +yyparse (<%= output.parse_param %>) +{ +<%# b4_declare_scanner_communication_variables -%> +/* Lookahead token kind. */ +int yychar; + + +/* The semantic value of the lookahead symbol. */ +/* Default value used for initialization, for pacifying older GCCs + or non-GCC compilers. */ +YY_INITIAL_VALUE (static const YYSTYPE yyval_default;) +YYSTYPE yylval YY_INITIAL_VALUE (= yyval_default); + +/* Location data for the lookahead symbol. */ +static const YYLTYPE yyloc_default +# if defined YYLTYPE_IS_TRIVIAL && YYLTYPE_IS_TRIVIAL + = { 1, 1, 1, 1 } +# endif +; +YYLTYPE yylloc = yyloc_default; + +<%# b4_declare_parser_state_variables -%> + /* Number of syntax errors so far. */ + int yynerrs = 0; + YY_USE (yynerrs); /* Silence compiler warning. */ + + yy_state_fast_t yystate = 0; + /* Number of tokens to shift before error messages enabled. */ + int yyerrstatus = 0; + + /* Refer to the stacks through separate pointers, to allow yyoverflow + to reallocate them elsewhere. */ + + /* Their size. */ + YYPTRDIFF_T yystacksize = YYINITDEPTH; + + /* The state stack: array, bottom, top. */ + yy_state_t yyssa[YYINITDEPTH]; + yy_state_t *yyss = yyssa; + yy_state_t *yyssp = yyss; + + /* The semantic value stack: array, bottom, top. */ + YYSTYPE yyvsa[YYINITDEPTH]; + YYSTYPE *yyvs = yyvsa; + YYSTYPE *yyvsp = yyvs; + + /* The location stack: array, bottom, top. */ + YYLTYPE yylsa[YYINITDEPTH]; + YYLTYPE *yyls = yylsa; + YYLTYPE *yylsp = yyls; + + int yyn; + /* The return value of yyparse. */ + int yyresult; + /* Lookahead symbol kind. */ + yysymbol_kind_t yytoken = YYSYMBOL_YYEMPTY; + /* The variables used to return semantic value and location from the + action routines. */ + YYSTYPE yyval; + YYLTYPE yyloc; + + /* The locations where the error started and ended. */ + YYLTYPE yyerror_range[3]; +<%- if output.error_recovery -%> + yy_repair_terms *rep_terms = 0; + yy_term term_backup; + int rep_terms_index; + int yychar_backup; +<%- end -%> + + /* Buffer for error messages, and its allocated size. */ + char yymsgbuf[128]; + char *yymsg = yymsgbuf; + YYPTRDIFF_T yymsg_alloc = sizeof yymsgbuf; + +#define YYPOPSTACK(N) (yyvsp -= (N), yyssp -= (N), yylsp -= (N)) + + /* The number of symbols on the RHS of the reduced rule. + Keep to zero when no symbol should be popped. */ + int yylen = 0; + + YYDPRINTF ((stderr, "Starting parse\n")); + + yychar = YYEMPTY; /* Cause a token to be read. */ + + +<%# b4_user_initial_action -%> +<%= output.user_initial_action("/* User initialization code. */") %> +#line [@oline@] [@ofile@] + + yylsp[0] = yylloc; + goto yysetstate; + + +/*------------------------------------------------------------. +| yynewstate -- push a new state, which is found in yystate. | +`------------------------------------------------------------*/ +yynewstate: + /* In all cases, when you get here, the value and location stacks + have just been pushed. So pushing a state here evens the stacks. */ + yyssp++; + + +/*--------------------------------------------------------------------. +| yysetstate -- set current state (the top of the stack) to yystate. | +`--------------------------------------------------------------------*/ +yysetstate: + YYDPRINTF ((stderr, "Entering state %d\n", yystate)); + YY_ASSERT (0 <= yystate && yystate < YYNSTATES); + YY_IGNORE_USELESS_CAST_BEGIN + *yyssp = YY_CAST (yy_state_t, yystate); + YY_IGNORE_USELESS_CAST_END + YY_STACK_PRINT (yyss, yyssp<%= output.user_args %>); + + if (yyss + yystacksize - 1 <= yyssp) +#if !defined yyoverflow && !defined YYSTACK_RELOCATE + YYNOMEM; +#else + { + /* Get the current used size of the three stacks, in elements. */ + YYPTRDIFF_T yysize = yyssp - yyss + 1; + +# if defined yyoverflow + { + /* Give user a chance to reallocate the stack. Use copies of + these so that the &'s don't force the real ones into + memory. */ + yy_state_t *yyss1 = yyss; + YYSTYPE *yyvs1 = yyvs; + YYLTYPE *yyls1 = yyls; + + /* Each stack pointer address is followed by the size of the + data in use in that stack, in bytes. This used to be a + conditional around just the two extra args, but that might + be undefined if yyoverflow is a macro. */ + yyoverflow (YY_("memory exhausted"), + &yyss1, yysize * YYSIZEOF (*yyssp), + &yyvs1, yysize * YYSIZEOF (*yyvsp), + &yyls1, yysize * YYSIZEOF (*yylsp), + &yystacksize); + yyss = yyss1; + yyvs = yyvs1; + yyls = yyls1; + } +# else /* defined YYSTACK_RELOCATE */ + /* Extend the stack our own way. */ + if (YYMAXDEPTH <= yystacksize) + YYNOMEM; + yystacksize *= 2; + if (YYMAXDEPTH < yystacksize) + yystacksize = YYMAXDEPTH; + + { + yy_state_t *yyss1 = yyss; + union yyalloc *yyptr = + YY_CAST (union yyalloc *, + YYSTACK_ALLOC (YY_CAST (YYSIZE_T, YYSTACK_BYTES (yystacksize)))); + if (! yyptr) + YYNOMEM; + YYSTACK_RELOCATE (yyss_alloc, yyss); + YYSTACK_RELOCATE (yyvs_alloc, yyvs); + YYSTACK_RELOCATE (yyls_alloc, yyls); +# undef YYSTACK_RELOCATE + if (yyss1 != yyssa) + YYSTACK_FREE (yyss1); + } +# endif + + yyssp = yyss + yysize - 1; + yyvsp = yyvs + yysize - 1; + yylsp = yyls + yysize - 1; + + YY_IGNORE_USELESS_CAST_BEGIN + YYDPRINTF ((stderr, "Stack size increased to %ld\n", + YY_CAST (long, yystacksize))); + YY_IGNORE_USELESS_CAST_END + + if (yyss + yystacksize - 1 <= yyssp) + YYABORT; + } +#endif /* !defined yyoverflow && !defined YYSTACK_RELOCATE */ + + + if (yystate == YYFINAL) + YYACCEPT; + + goto yybackup; + + +/*-----------. +| yybackup. | +`-----------*/ +yybackup: + /* Do appropriate processing given the current state. Read a + lookahead token if we need one and don't already have one. */ + + /* First try to decide what to do without reference to lookahead token. */ + yyn = yypact[yystate]; + if (yypact_value_is_default (yyn)) + goto yydefault; + + /* Not known => get a lookahead token if don't already have one. */ + +<%- if output.error_recovery -%> + if (YYERROR_RECOVERY_ENABLED(<%= output.parse_param_name %>)) + { + if (yychar == YYEMPTY && rep_terms) + { + + if (rep_terms_index < rep_terms->length) + { + YYDPRINTF ((stderr, "An error recovery token is used\n")); + yy_term term = rep_terms->terms[rep_terms_index]; + yytoken = term.kind; + yylval = term.value; + yylloc = term.location; + yychar = yytranslate_inverted[yytoken]; + YY_SYMBOL_PRINT ("Next error recovery token is", yytoken, &yylval, &yylloc<%= output.user_args %>); + rep_terms_index++; + } + else + { + YYDPRINTF ((stderr, "Error recovery is completed\n")); + yytoken = term_backup.kind; + yylval = term_backup.value; + yylloc = term_backup.location; + yychar = yychar_backup; + YY_SYMBOL_PRINT ("Next token is", yytoken, &yylval, &yylloc<%= output.user_args %>); + + YYFREE (rep_terms); + rep_terms = 0; + yychar_backup = 0; + } + } + } +<%- end -%> + /* YYCHAR is either empty, or end-of-input, or a valid lookahead. */ + if (yychar == YYEMPTY) + { + YYDPRINTF ((stderr, "Reading a token\n")); + yychar = yylex <%= output.yylex_formals %>; + } + + if (yychar <= <%= output.eof_symbol.id.s_value %>) + { + yychar = <%= output.eof_symbol.id.s_value %>; + yytoken = <%= output.eof_symbol.enum_name %>; + YYDPRINTF ((stderr, "Now at end of input.\n")); + } + else if (yychar == <%= output.error_symbol.id.s_value %>) + { + /* The scanner already issued an error message, process directly + to error recovery. But do not keep the error token as + lookahead, it is too special and may lead us to an endless + loop in error recovery. */ + yychar = <%= output.undef_symbol.id.s_value %>; + yytoken = <%= output.error_symbol.enum_name %>; + yyerror_range[1] = yylloc; + goto yyerrlab1; + } + else + { + yytoken = YYTRANSLATE (yychar); + YY_SYMBOL_PRINT ("Next token is", yytoken, &yylval, &yylloc<%= output.user_args %>); + } + + /* If the proper action on seeing token YYTOKEN is to reduce or to + detect an error, take that action. */ + yyn += yytoken; + if (yyn < 0 || YYLAST < yyn || yycheck[yyn] != yytoken) + goto yydefault; + yyn = yytable[yyn]; + if (yyn <= 0) + { + if (yytable_value_is_error (yyn)) + goto yyerrlab; + yyn = -yyn; + goto yyreduce; + } + + /* Count tokens shifted since error; after three, turn off error + status. */ + if (yyerrstatus) + yyerrstatus--; + + /* Shift the lookahead token. */ + YY_SYMBOL_PRINT ("Shifting", yytoken, &yylval, &yylloc<%= output.user_args %>); + yystate = yyn; + YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN + *++yyvsp = yylval; + YY_IGNORE_MAYBE_UNINITIALIZED_END + *++yylsp = yylloc; +<%= output.after_shift_function("/* %after-shift code. */") %> + + /* Discard the shifted token. */ + yychar = YYEMPTY; + goto yynewstate; + + +/*-----------------------------------------------------------. +| yydefault -- do the default action for the current state. | +`-----------------------------------------------------------*/ +yydefault: + yyn = yydefact[yystate]; + if (yyn == 0) + goto yyerrlab; + goto yyreduce; + + +/*-----------------------------. +| yyreduce -- do a reduction. | +`-----------------------------*/ +yyreduce: + /* yyn is the number of a rule to reduce with. */ + yylen = yyr2[yyn]; + + /* If YYLEN is nonzero, implement the default value of the action: + '$$ = $1'. + + Otherwise, the following line sets YYVAL to garbage. + This behavior is undocumented and Bison + users should not rely upon it. Assigning to YYVAL + unconditionally makes the parser a bit smaller, and it avoids a + GCC warning that YYVAL may be used uninitialized. */ + yyval = yyvsp[1-yylen]; +<%= output.before_reduce_function("/* %before-reduce function. */") %> + + /* Default location. */ + YYLLOC_DEFAULT (yyloc, (yylsp - yylen), yylen); + yyerror_range[1] = yyloc; + YY_REDUCE_PRINT (yyn<%= output.user_args %>); + switch (yyn) + { +<%= output.user_actions -%> + + default: break; + } + /* User semantic actions sometimes alter yychar, and that requires + that yytoken be updated with the new translation. We take the + approach of translating immediately before every use of yytoken. + One alternative is translating here after every semantic action, + but that translation would be missed if the semantic action invokes + YYABORT, YYACCEPT, or YYERROR immediately after altering yychar or + if it invokes YYBACKUP. In the case of YYABORT or YYACCEPT, an + incorrect destructor might then be invoked immediately. In the + case of YYERROR or YYBACKUP, subsequent parser actions might lead + to an incorrect destructor call or verbose syntax error message + before the lookahead is translated. */ + YY_SYMBOL_PRINT ("-> $$ =", YY_CAST (yysymbol_kind_t, yyr1[yyn]), &yyval, &yyloc<%= output.user_args %>); + + YYPOPSTACK (yylen); +<%= output.after_reduce_function("/* %after-reduce function. */") %> + yylen = 0; + + *++yyvsp = yyval; + *++yylsp = yyloc; + + /* Now 'shift' the result of the reduction. Determine what state + that goes to, based on the state we popped back to and the rule + number reduced by. */ + { + const int yylhs = yyr1[yyn] - YYNTOKENS; + const int yyi = yypgoto[yylhs] + *yyssp; + yystate = (0 <= yyi && yyi <= YYLAST && yycheck[yyi] == *yyssp + ? yytable[yyi] + : yydefgoto[yylhs]); + } + + goto yynewstate; + + +/*--------------------------------------. +| yyerrlab -- here on detecting error. | +`--------------------------------------*/ +yyerrlab: + /* Make sure we have latest lookahead translation. See comments at + user semantic actions for why this is necessary. */ + yytoken = yychar == YYEMPTY ? YYSYMBOL_YYEMPTY : YYTRANSLATE (yychar); + /* If not already recovering from an error, report this error. */ + if (!yyerrstatus) + { + ++yynerrs; + { + yypcontext_t yyctx + = {yyssp, yytoken, &yylloc}; + char const *yymsgp = YY_("syntax error"); + int yysyntax_error_status; + yysyntax_error_status = yysyntax_error (&yymsg_alloc, &yymsg, &yyctx<%= output.user_args %>); + if (yysyntax_error_status == 0) + yymsgp = yymsg; + else if (yysyntax_error_status == -1) + { + if (yymsg != yymsgbuf) + YYSTACK_FREE (yymsg); + yymsg = YY_CAST (char *, + YYSTACK_ALLOC (YY_CAST (YYSIZE_T, yymsg_alloc))); + if (yymsg) + { + yysyntax_error_status + = yysyntax_error (&yymsg_alloc, &yymsg, &yyctx<%= output.user_args %>); + yymsgp = yymsg; + } + else + { + yymsg = yymsgbuf; + yymsg_alloc = sizeof yymsgbuf; + yysyntax_error_status = YYENOMEM; + } + } + yyerror (<%= output.yyerror_args %>, yymsgp); + if (yysyntax_error_status == YYENOMEM) + YYNOMEM; + } + } + + yyerror_range[1] = yylloc; + if (yyerrstatus == 3) + { + /* If just tried and failed to reuse lookahead token after an + error, discard it. */ + + if (yychar <= <%= output.eof_symbol.id.s_value %>) + { + /* Return failure if at end of input. */ + if (yychar == <%= output.eof_symbol.id.s_value %>) + YYABORT; + } + else + { + yydestruct ("Error: discarding", + yytoken, &yylval, &yylloc<%= output.user_args %>); + yychar = YYEMPTY; + } + } + + /* Else will try to reuse lookahead token after shifting the error + token. */ + goto yyerrlab1; + + +/*---------------------------------------------------. +| yyerrorlab -- error raised explicitly by YYERROR. | +`---------------------------------------------------*/ +yyerrorlab: + /* Pacify compilers when the user code never invokes YYERROR and the + label yyerrorlab therefore never appears in user code. */ + if (0) + YYERROR; + ++yynerrs; + + /* Do not reclaim the symbols of the rule whose action triggered + this YYERROR. */ + YYPOPSTACK (yylen); +<%= output.after_pop_stack_function("yylen", "/* %after-pop-stack function. */") %> + yylen = 0; + YY_STACK_PRINT (yyss, yyssp<%= output.user_args %>); + yystate = *yyssp; + goto yyerrlab1; + + +/*-------------------------------------------------------------. +| yyerrlab1 -- common code for both syntax error and YYERROR. | +`-------------------------------------------------------------*/ +yyerrlab1: +<%- if output.error_recovery -%> + if (YYERROR_RECOVERY_ENABLED(<%= output.parse_param_name %>)) + { + rep_terms = yyrecover (yyss, yyssp, yychar<%= output.user_args %>); + if (rep_terms) + { + for (int i = 0; i < rep_terms->length; i++) + { + yy_term *term = &rep_terms->terms[i]; + yy_error_token_initialize (term->kind, &term->value, &term->location<%= output.user_args %>); + } + + yychar_backup = yychar; + /* Can be packed into (the tail of) rep_terms? */ + term_backup.kind = yytoken; + term_backup.value = yylval; + term_backup.location = yylloc; + rep_terms_index = 0; + yychar = YYEMPTY; + + goto yybackup; + } + } +<%- end -%> + yyerrstatus = 3; /* Each real token shifted decrements this. */ + + /* Pop stack until we find a state that shifts the error token. */ + for (;;) + { + yyn = yypact[yystate]; + if (!yypact_value_is_default (yyn)) + { + yyn += YYSYMBOL_YYerror; + if (0 <= yyn && yyn <= YYLAST && yycheck[yyn] == YYSYMBOL_YYerror) + { + yyn = yytable[yyn]; + if (0 < yyn) + break; + } + } + + /* Pop the current state because it cannot handle the error token. */ + if (yyssp == yyss) + YYABORT; + + yyerror_range[1] = *yylsp; + yydestruct ("Error: popping", + YY_ACCESSING_SYMBOL (yystate), yyvsp, yylsp<%= output.user_args %>); + YYPOPSTACK (1); +<%= output.after_pop_stack_function(1, "/* %after-pop-stack function. */") %> + yystate = *yyssp; + YY_STACK_PRINT (yyss, yyssp<%= output.user_args %>); + } + + YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN + *++yyvsp = yylval; + YY_IGNORE_MAYBE_UNINITIALIZED_END + + yyerror_range[2] = yylloc; + ++yylsp; + YYLLOC_DEFAULT (*yylsp, yyerror_range, 2); + + /* Shift the error token. */ + YY_SYMBOL_PRINT ("Shifting", YY_ACCESSING_SYMBOL (yyn), yyvsp, yylsp<%= output.user_args %>); +<%= output.after_shift_error_token_function("/* %after-shift-error-token code. */") %> + + yystate = yyn; + goto yynewstate; + + +/*-------------------------------------. +| yyacceptlab -- YYACCEPT comes here. | +`-------------------------------------*/ +yyacceptlab: + yyresult = 0; + goto yyreturnlab; + + +/*-----------------------------------. +| yyabortlab -- YYABORT comes here. | +`-----------------------------------*/ +yyabortlab: + yyresult = 1; + goto yyreturnlab; + + +/*-----------------------------------------------------------. +| yyexhaustedlab -- YYNOMEM (memory exhaustion) comes here. | +`-----------------------------------------------------------*/ +yyexhaustedlab: + yyerror (<%= output.yyerror_args %>, YY_("memory exhausted")); + yyresult = 2; + goto yyreturnlab; + + +/*----------------------------------------------------------. +| yyreturnlab -- parsing is finished, clean up and return. | +`----------------------------------------------------------*/ +yyreturnlab: + if (yychar != YYEMPTY) + { + /* Make sure we have latest lookahead translation. See comments at + user semantic actions for why this is necessary. */ + yytoken = YYTRANSLATE (yychar); + yydestruct ("Cleanup: discarding lookahead", + yytoken, &yylval, &yylloc<%= output.user_args %>); + } + /* Do not reclaim the symbols of the rule whose action triggered + this YYABORT or YYACCEPT. */ + YYPOPSTACK (yylen); + YY_STACK_PRINT (yyss, yyssp<%= output.user_args %>); + while (yyssp != yyss) + { + yydestruct ("Cleanup: popping", + YY_ACCESSING_SYMBOL (+*yyssp), yyvsp, yylsp<%= output.user_args %>); + YYPOPSTACK (1); + } +#ifndef yyoverflow + if (yyss != yyssa) + YYSTACK_FREE (yyss); +#endif + if (yymsg != yymsgbuf) + YYSTACK_FREE (yymsg); + return yyresult; +} + +<%# b4_percent_code_get([[epilogue]]) -%> +<%- if output.aux.epilogue -%> +#line <%= output.aux.epilogue_first_lineno - 1 %> "<%= output.grammar_file_path %>" +<%= output.aux.epilogue -%> +<%- end -%> + diff --git a/tool/lrama/template/bison/yacc.h b/tool/lrama/template/bison/yacc.h new file mode 100644 index 0000000000..848dbf5961 --- /dev/null +++ b/tool/lrama/template/bison/yacc.h @@ -0,0 +1,40 @@ +<%# b4_generated_by -%> +/* A Bison parser, made by Lrama <%= Lrama::VERSION %>. */ + +<%# b4_copyright -%> +/* Bison interface for Yacc-like parsers in C + + Copyright (C) 1984, 1989-1990, 2000-2015, 2018-2021 Free Software Foundation, + Inc. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. */ + +/* As a special exception, you may create a larger work that contains + part or all of the Bison parser skeleton and distribute that work + under terms of your choice, so long as that work isn't itself a + parser generator using the skeleton or a modified version thereof + as a parser skeleton. Alternatively, if you modify or redistribute + the parser skeleton itself, you may (at your option) remove this + special exception, which will cause the skeleton and the resulting + Bison output files to be licensed under the GNU General Public + License without this special exception. + + This special exception was added by the Free Software Foundation in + version 2.2 of Bison. */ + +<%# b4_disclaimer -%> +/* DO NOT RELY ON FEATURES THAT ARE NOT DOCUMENTED in the manual, + especially those whose name start with YY_ or yy_. They are + private implementation details that can be changed or removed. */ +<%= output.render_partial("bison/_yacc.h") %> diff --git a/tool/m4/ruby_default_arch.m4 b/tool/m4/ruby_default_arch.m4 index 03e52f7776..2f25ba81ee 100644 --- a/tool/m4/ruby_default_arch.m4 +++ b/tool/m4/ruby_default_arch.m4 @@ -1,11 +1,21 @@ dnl -*- Autoconf -*- AC_DEFUN([RUBY_DEFAULT_ARCH], [ +# Set ARCH_FLAG for different width but family CPU AC_MSG_CHECKING([arch option]) -AS_CASE([$1], - [arm64], [], - [*64], [ARCH_FLAG=-m64], - [[i[3-6]86]], [ARCH_FLAG=-m32], - [AC_MSG_ERROR(unknown target architecture: $target_archs)] - ) -AC_MSG_RESULT([$ARCH_FLAG]) +AS_CASE([$1:"$host_cpu"], + [arm64:arm*], [ARCH_FLAG=-m64], + [arm*:arm*], [ARCH_FLAG=-m32], + [x86_64:[i[3-6]86]], [ARCH_FLAG=-m64], + [x64:x86_64], [], + [[i[3-6]86]:x86_64], [ARCH_FLAG=-m32], + [ppc64:ppc*], [ARCH_FLAG=-m64], + [ppc*:ppc64], [ARCH_FLAG=-m32], + [ + ARCH_FLAG= + for flag in "-arch "$1 -march=$1; do + _RUBY_TRY_CFLAGS([$]flag, [ARCH_FLAG="[$]flag"]) + test x"$ARCH_FLAG" = x || break + done] +) +AC_MSG_RESULT([${ARCH_FLAG:-'(none)'}]) ])dnl diff --git a/tool/m4/ruby_shared_gc.m4 b/tool/m4/ruby_shared_gc.m4 new file mode 100644 index 0000000000..a27b9b8505 --- /dev/null +++ b/tool/m4/ruby_shared_gc.m4 @@ -0,0 +1,19 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_SHARED_GC],[ +AC_ARG_WITH(shared-gc, + AS_HELP_STRING([--with-shared-gc], + [Enable replacement of Ruby's GC from a shared library.]), + [with_shared_gc=$withval], [unset with_shared_gc] +) + +AC_SUBST([with_shared_gc]) +AC_MSG_CHECKING([if Ruby is build with shared GC support]) +AS_IF([test "$with_shared_gc" = "yes"], [ + AC_MSG_RESULT([yes]) + AC_DEFINE([USE_SHARED_GC], [1]) +], [ + AC_MSG_RESULT([no]) + with_shared_gc="no" + AC_DEFINE([USE_SHARED_GC], [0]) +]) +])dnl diff --git a/tool/m4/ruby_stack_grow_direction.m4 b/tool/m4/ruby_stack_grow_direction.m4 index a4d205cc3c..8c6fdd5722 100644 --- a/tool/m4/ruby_stack_grow_direction.m4 +++ b/tool/m4/ruby_stack_grow_direction.m4 @@ -3,7 +3,7 @@ AC_DEFUN([RUBY_STACK_GROW_DIRECTION], [ AS_VAR_PUSHDEF([stack_grow_dir], [rb_cv_stack_grow_dir_$1]) AC_CACHE_CHECK(stack growing direction on $1, stack_grow_dir, [ AS_CASE(["$1"], -[m68*|x86*|x64|i?86|ppc*|sparc*|alpha*], [ $2=-1], +[m68*|x86*|x64|i?86|ppc*|sparc*|alpha*|arm*|aarch*], [ $2=-1], [hppa*], [ $2=+1], [ AC_RUN_IFELSE([AC_LANG_SOURCE([[ diff --git a/tool/m4/ruby_try_cflags.m4 b/tool/m4/ruby_try_cflags.m4 index 672f4f8e51..b74718fe5e 100644 --- a/tool/m4/ruby_try_cflags.m4 +++ b/tool/m4/ruby_try_cflags.m4 @@ -6,14 +6,19 @@ m4_version_prereq([2.70], [], [ m4_defun([AC_LANG_PROGRAM(C)], m4_bpatsubst(m4_defn([AC_LANG_PROGRAM(C)]), [main ()], [main (void)])) ])dnl dnl -AC_DEFUN([RUBY_TRY_CFLAGS], [ - AC_MSG_CHECKING([whether ]$1[ is accepted as CFLAGS]) +AC_DEFUN([_RUBY_TRY_CFLAGS], [ RUBY_WERROR_FLAG([ CFLAGS="[$]CFLAGS $1" AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[$4]], [[$5]])], + [$2], [$3]) + ])dnl +])dnl +AC_DEFUN([RUBY_TRY_CFLAGS], [ + AC_MSG_CHECKING([whether ]$1[ is accepted as CFLAGS])dnl + _RUBY_TRY_CFLAGS([$1], [$2 AC_MSG_RESULT(yes)], [$3 - AC_MSG_RESULT(no)]) - ]) + AC_MSG_RESULT(no)], + [$4], [$5]) ])dnl diff --git a/tool/m4/ruby_universal_arch.m4 b/tool/m4/ruby_universal_arch.m4 index c8914c88d9..d3e0dd0b47 100644 --- a/tool/m4/ruby_universal_arch.m4 +++ b/tool/m4/ruby_universal_arch.m4 @@ -17,7 +17,7 @@ AS_IF([test ${target_archs+set}], [ cpu=$archs cpu=`echo $cpu | sed 's/-.*-.*//'` universal_binary="${universal_binary+$universal_binary,}$cpu" - universal_archnames="${universal_archnames} ${archs}=${cpu}" + universal_archnames="${universal_archnames:+$universal_archnames }${archs}=${cpu}" ARCH_FLAG="${ARCH_FLAG+$ARCH_FLAG }-arch $archs" ]) done @@ -40,7 +40,7 @@ AS_IF([test ${target_archs+set}], [ AS_IF([$CC $CFLAGS $ARCH_FLAG -o conftest conftest.c > /dev/null 2>&1], [ rm -fr conftest.* ], [test -z "$ARCH_FLAG"], [ - RUBY_DEFAULT_ARCH("$target_archs") + RUBY_DEFAULT_ARCH($target_archs) ]) ]) target_cpu=${target_archs} @@ -73,7 +73,7 @@ EOF sed -n 's/^"processor-name=\(.*\)"/\1/p'` target="$target_cpu${target}" AC_MSG_RESULT([$target_cpu]) - ]) + ]) ]) target_archs="$target_cpu" ]) diff --git a/tool/m4/ruby_wasm_tools.m4 b/tool/m4/ruby_wasm_tools.m4 index d58de88ec8..efc017e771 100644 --- a/tool/m4/ruby_wasm_tools.m4 +++ b/tool/m4/ruby_wasm_tools.m4 @@ -9,13 +9,17 @@ AC_DEFUN([RUBY_WASM_TOOLS], AC_SUBST(wasmoptflags) : ${wasmoptflags=-O3} - AC_MSG_CHECKING([wheather \$WASI_SDK_PATH is set]) - AS_IF([test x"${WASI_SDK_PATH}" = x], [AC_MSG_RESULT([no])], [ + AC_MSG_CHECKING([whether \$WASI_SDK_PATH is set]) + AS_IF([test x"${WASI_SDK_PATH}" = x], [ + AC_MSG_RESULT([no]) + AC_MSG_ERROR([WASI_SDK_PATH environment variable is required]) + ], [ AC_MSG_RESULT([yes]) - CC="${WASI_SDK_PATH}/bin/clang" - LD="${WASI_SDK_PATH}/bin/clang" - AR="${WASI_SDK_PATH}/bin/llvm-ar" - RANLIB="${WASI_SDK_PATH}/bin/llvm-ranlib" + CC="${CC:-${WASI_SDK_PATH}/bin/clang}" + LD="${LD:-${WASI_SDK_PATH}/bin/clang}" + AR="${AR:-${WASI_SDK_PATH}/bin/llvm-ar}" + RANLIB="${RANLIB:-${WASI_SDK_PATH}/bin/llvm-ranlib}" + OBJCOPY="${OBJCOPY:-${WASI_SDK_PATH}/bin/llvm-objcopy}" ]) ]) ])dnl diff --git a/tool/make-snapshot b/tool/make-snapshot index f91ab8855f..7446f18578 100755 --- a/tool/make-snapshot +++ b/tool/make-snapshot @@ -22,6 +22,7 @@ $keep_temp ||= nil $patch_file ||= nil $packages ||= nil $digests ||= nil +$no7z ||= nil $tooldir = File.expand_path("..", __FILE__) $unicode_version = nil if ($unicode_version ||= nil) == "" $colorize = Colorize.new @@ -37,8 +38,6 @@ options: -packages=PKG[,...] make PKG packages (#{PACKAGES.keys.join(", ")}) -digests=ALG[,...] show ALG digests (#{DIGESTS.join(", ")}) -unicode_version=VER Unicode version to generate encodings - -svn[=URL] make snapshot from SVN repository - (#{SVNURL}) -help, --help show this message version: master, trunk, stable, branches/*, tags/*, X.Y, X.Y.Z, X.Y.Z-pL @@ -68,13 +67,12 @@ if mflags = ENV["GNUMAKEFLAGS"] and /\A-(\S*)j\d*/ =~ mflags ENV["GNUMAKEFLAGS"] = (mflags unless mflags.empty?) end ENV["LC_ALL"] = ENV["LANG"] = "C" -SVNURL = URI.parse("https://svn.ruby-lang.org/repos/ruby/") # https git clone is disabled at git.ruby-lang.org/ruby.git. GITURL = URI.parse("https://github.com/ruby/ruby.git") RUBY_VERSION_PATTERN = /^\#define\s+RUBY_VERSION\s+"([\d.]+)"/ ENV["VPATH"] ||= "include/ruby" -YACC = ENV["YACC"] ||= "bison" +YACC = ENV["YACC"] ||= "#{$tooldir}/lrama/exe/lrama" ENV["BASERUBY"] ||= "ruby" ENV["RUBY"] ||= "ruby" ENV["MV"] ||= "mv" @@ -146,7 +144,7 @@ unless destdir = ARGV.shift end revisions = ARGV.empty? ? [nil] : ARGV -if $exported +if defined?($exported) abort "#{File.basename $0}: -exported option is deprecated; use -srcdir instead" end @@ -294,7 +292,7 @@ def package(vcs, rev, destdir, tmp = nil) if info = vcs.get_revisions(url) modified = info[2] else - modified = Time.now - 10 + _, _, modified = VCS::Null.new(nil).get_revisions(url) end if !revision and info revision = info @@ -345,12 +343,8 @@ def package(vcs, rev, destdir, tmp = nil) v = v[0] end - open("#{v}/revision.h", "wb") {|f| - short = vcs.short_revision(revision) - f.puts "#define RUBY_REVISION #{short.inspect}" - unless short == revision - f.puts "#define RUBY_FULL_REVISION #{revision.inspect}" - end + File.open("#{v}/revision.h", "wb") {|f| + f.puts vcs.revision_header(revision, modified) } version ||= (versionhdr = IO.read("#{v}/version.h"))[RUBY_VERSION_PATTERN, 1] version ||= @@ -370,7 +364,7 @@ def package(vcs, rev, destdir, tmp = nil) end elsif prerelease versionhdr ||= IO.read("#{v}/version.h") - versionhdr.sub!(/^\#define\s+RUBY_PATCHLEVEL_STR\s+"\K.+?(?=")/, tag) + versionhdr.sub!(/^\#\s*define\s+RUBY_PATCHLEVEL_STR\s+"\K.+?(?=")/, tag) or raise "no match of RUBY_PATCHLEVEL_STR to replace" IO.write("#{v}/version.h", versionhdr) else tag ||= vcs.revision_name(revision) @@ -390,7 +384,21 @@ def package(vcs, rev, destdir, tmp = nil) puts $colorize.fail("patching failed") return end - def (clean = []).add(n) push(n); n end + + class << (clean = []) + def add(n) push(n) + n + end + def create(file, content = "", &block) + add(file) + if block + File.open(file, "wb", &block) + else + File.binwrite(file, content) + end + end + end + Dir.chdir(v) do unless File.exist?("ChangeLog") vcs.export_changelog(url, nil, revision, "ChangeLog") @@ -411,7 +419,7 @@ def package(vcs, rev, destdir, tmp = nil) puts end - File.open(clean.add("cross.rb"), "w") do |f| + clean.create("cross.rb") do |f| f.puts "Object.__send__(:remove_const, :CROSS_COMPILING) if defined?(CROSS_COMPILING)" f.puts "CROSS_COMPILING=true" f.puts "Object.__send__(:remove_const, :RUBY_PLATFORM)" @@ -419,6 +427,7 @@ def package(vcs, rev, destdir, tmp = nil) f.puts "Object.__send__(:remove_const, :RUBY_VERSION)" f.puts "RUBY_VERSION='#{version}'" end + puts "cross.rb:", File.read("cross.rb").gsub(/^/, "> "), "" if $VERBOSE unless File.exist?("configure") print "creating configure..." unless system([ENV["AUTOCONF"]]*2) @@ -438,14 +447,13 @@ def package(vcs, rev, destdir, tmp = nil) rescue Errno::ENOENT # use fallback file end - File.open(clean.add("config.status"), "w") {|f| - f.print status - } + clean.create("config.status", status) + clean.create("noarch-fake.rb", "require_relative 'cross'\n") FileUtils.mkpath(hdrdir = "#{extout}/include/ruby") - File.open("#{hdrdir}/config.h", "w") {} + File.binwrite("#{hdrdir}/config.h", "") FileUtils.mkpath(defaults = "#{extout}/rubygems/defaults") - File.open("#{defaults}/operating_system.rb", "w") {} - File.open("#{defaults}/ruby.rb", "w") {} + File.binwrite("#{defaults}/operating_system.rb", "") + File.binwrite("#{defaults}/ruby.rb", "") miniruby = ENV['MINIRUBY'] + " -I. -I#{extout} -rcross" baseruby = ENV["BASERUBY"] mk = (IO.read("template/Makefile.in") rescue IO.read("Makefile.in")). @@ -481,11 +489,9 @@ update-gems: $(UNICODE_SRC_DATA_DIR)/.unicode-tables.time: touch-unicode-files: APPEND - open(clean.add("Makefile"), "w") do |f| - f.puts mk - end - File.open(clean.add("revision.tmp"), "w") {} - File.open(clean.add(".revision.time"), "w") {} + clean.create("Makefile", mk) + clean.create("revision.tmp") + clean.create(".revision.time") ENV["CACHE_SAVE"] = "no" make = MAKE.new(args) return unless make.run("update-download") @@ -512,8 +518,7 @@ touch-unicode-files: end vcs.after_export(".") if exported clean.concat(Dir.glob("ext/**/autom4te.cache")) - FileUtils.rm_rf(clean) unless $keep_temp - FileUtils.rm_rf(".downloaded-cache") + clean.add(".downloaded-cache") if File.exist?("gems/bundled_gems") gems = Dir.glob("gems/*.gem") gems -= File.readlines("gems/bundled_gems").map {|line| @@ -521,10 +526,11 @@ touch-unicode-files: name, version, _ = line.split(' ') "gems/#{name}-#{version}.gem" } - FileUtils.rm_f(gems) + clean.concat(gems) else - FileUtils.rm_rf("gems") + clean.add("gems") end + FileUtils.rm_rf(clean) if modified touch_all(modified, "**/*/", 0) do |name, stat| stat.mtime > modified @@ -593,20 +599,18 @@ ensure Dir.chdir(pwd) end -if [$srcdir, ($svn||=nil), ($git||=nil)].compact.size > 1 - abort "#{File.basename $0}: -srcdir, -svn, and -git are exclusive" +if [$srcdir, ($git||=nil)].compact.size > 1 + abort "#{File.basename $0}: -srcdir and -git are exclusive" end if $srcdir vcs = VCS.detect($srcdir) -elsif $svn - vcs = VCS::SVN.new($svn == true ? SVNURL : URI.parse($svn)) elsif $git abort "#{File.basename $0}: use -srcdir with cloned local repository" else begin vcs = VCS.detect(File.expand_path("../..", __FILE__)) rescue VCS::NotFoundError - vcs = VCS::SVN.new(SVNURL) + abort "#{File.expand_path("../..", __FILE__)}: cannot find git repository" end end @@ -619,7 +623,7 @@ revisions.collect {|rev| package(vcs, rev, destdir, tmp)}.flatten.each do |name| success = false next end - str = open(name, "rb") {|f| f.read} + str = File.binread(name) pathname = Pathname(name) basename = pathname.basename.to_s extname = pathname.extname.sub(/\A\./, '') diff --git a/tool/make_hgraph.rb b/tool/make_hgraph.rb index 0f388814dd..174fa5dd2f 100644 --- a/tool/make_hgraph.rb +++ b/tool/make_hgraph.rb @@ -83,13 +83,12 @@ module ObjectSpace def self.module_refenreces_image klass, file dot = module_refenreces_dot(klass) - img = nil - IO.popen("dot -Tpng", 'r+'){|io| + img = IO.popen(%W"dot -Tpng", 'r+b') {|io| # io.puts dot io.close_write - img = io.read + io.read } - open(File.expand_path(file), 'w+'){|f| f.puts img} + File.binwrite(file, img) end end diff --git a/tool/merger.rb b/tool/merger.rb index d38f00b0fd..0d9957074f 100755 --- a/tool/merger.rb +++ b/tool/merger.rb @@ -57,11 +57,11 @@ class << Merger yield if block_given? STDERR.puts "\e[1;33m#{str} ([y]es|[a]bort|[r]etry#{'|[e]dit' if editfile})\e[0m" case STDIN.gets - when /\Aa/i then exit + when /\Aa/i then exit 1 when /\Ar/i then redo when /\Ay/i then break when /\Ae/i then system(ENV['EDITOR'], editfile) - else exit + else exit 1 end end end @@ -204,7 +204,7 @@ class << Merger if svn_mode? command = %w[svn diff --diff-cmd=diff -x -upw] else - command = %w[git diff --color] + command = %w[git diff --color HEAD] end IO.popen(command + [file].compact, &:read) end @@ -295,7 +295,8 @@ else tickets = '' end - revstr = ARGV[0].delete('^, :\-0-9a-fA-F') + revstr = ARGV[0].gsub(%r!https://github\.com/ruby/ruby/commit/|https://bugs\.ruby-lang\.org/projects/ruby-master/repository/git/revisions/!, '') + revstr = revstr.delete('^, :\-0-9a-fA-F') revs = revstr.split(/[,\s]+/) commit_message = '' @@ -323,9 +324,12 @@ else end patch = resp.body.sub(/^diff --git a\/version\.h b\/version\.h\nindex .*\n--- a\/version\.h\n\+\+\+ b\/version\.h\n@@ .* @@\n(?:[-\+ ].*\n|\n)+/, '') - message = "\n\n#{(patch[/^Subject: (.*)\n\ndiff --git/m, 1] || "Message not found for revision: #{git_rev}\n")}" + message = "#{(patch[/^Subject: (.*)\n---\n /m, 1] || "Message not found for revision: #{git_rev}\n")}" + message.gsub!(/\G(.*)\n( .*)/, "\\1\\2") + message = "\n\n#{message}" + puts '+ git apply' - IO.popen(['git', 'apply'], 'wb') { |f| f.write(patch) } + IO.popen(['git', 'apply', '--3way'], 'wb') { |f| f.write(patch) } else default_merge_branch = (%r{^URL: .*/branches/ruby_1_8_} =~ `svn info` ? 'branches/ruby_1_8' : 'trunk') svn_src = "#{Merger::REPOS}#{ARGV[1] || default_merge_branch}" diff --git a/tool/missing-baseruby.bat b/tool/missing-baseruby.bat new file mode 100755 index 0000000000..87a9857e06 --- /dev/null +++ b/tool/missing-baseruby.bat @@ -0,0 +1,19 @@ +:"" == " +@echo off || ( + :warn + echo>&2.%~1 + goto :eof + :abort + exit /b 1 +)||( +:)"||( + s = %^# +) +: ; call() { local call=${1#:}; shift; $call "$@"; } +: ; warn() { echo "$1" >&2; } +: ; abort () { exit 1; } + +call :warn "executable host ruby is required. use --with-baseruby option." +call :warn "Note that BASERUBY must be Ruby 3.0.0 or later." +call :abort +: || (:^; abort if RUBY_VERSION < s[%r"warn .*Ruby ([\d.]+)(?:\.0)?",1]) diff --git a/tool/mjit_archflag.sh b/tool/mjit_archflag.sh deleted file mode 100644 index 082fb4bcd0..0000000000 --- a/tool/mjit_archflag.sh +++ /dev/null @@ -1,40 +0,0 @@ -# -*- sh -*- - -quote() { - printf "#${indent}define $1" - shift - ${1+printf} ${1+' "%s"'$sep} ${1+"$@"} - echo -} - -archs="" -arch_flag="" - -parse_arch_flags() { - for arch in $1; do - archs="${archs:+$archs }${arch%=*}" - done - - while shift && [ "$#" -gt 0 ]; do - case "$1" in - -arch) - shift - archs="${archs:+$archs }$1" - ;; - *) - arch_flag="${arch_flag:+${arch_flag} }$1" - ;; - esac - done -} - -define_arch_flags() { - ${archs:+echo} ${archs:+'#if 0'} - for arch in $archs; do - echo "#elif defined __${arch}__" - quote "MJIT_ARCHFLAG " -arch "${arch}" - done - ${archs:+echo} ${archs:+'#else'} - quote "MJIT_ARCHFLAG /* ${arch_flag:-no flag} */" ${arch_flag} - ${archs:+echo} ${archs:+'#endif'} -} diff --git a/tool/mjit_tabs.rb b/tool/mjit_tabs.rb deleted file mode 100644 index edcbf6cfcb..0000000000 --- a/tool/mjit_tabs.rb +++ /dev/null @@ -1,67 +0,0 @@ -# frozen_string_literal: true -# This is a script to run a command in ARGV, expanding tabs in some files -# included by vm.c to normalize indentation of MJIT header. You can enable -# this feature by passing `--without-mjit-tabs` in configure. -# -# Note that preprocessor of GCC converts a hard tab to one spaces, where -# we expect it to be shown as 8 spaces. To obviate this script, we need -# to convert all tabs to spaces in these files. - -require 'fileutils' - -EXPAND_TARGETS = %w[ - vm*.* - include/ruby/ruby.h -] - -# These files have no hard tab indentations. Skip normalizing these files from the glob result. -SKIPPED_FILES = %w[ - vm_callinfo.h - vm_debug.h - vm_exec.h - vm_opts.h - vm_sync.h - vm_sync.c -] - -srcdir = File.expand_path('..', __dir__) -targets = EXPAND_TARGETS.flat_map { |t| Dir.glob(File.join(srcdir, t)) } - SKIPPED_FILES.map { |f| File.join(srcdir, f) } -sources = {} -mtimes = {} - -mjit_tabs, *command = ARGV - -targets.each do |target| - next if mjit_tabs != 'false' - unless File.writable?(target) - puts "tool/mjit_tabs.rb: Skipping #{target.dump} as it's not writable." - next - end - source = File.read(target) - begin - expanded = source.gsub(/^\t+/) { |tab| ' ' * 8 * tab.length } - rescue ArgumentError # invalid byte sequence in UTF-8 (Travis, RubyCI) - puts "tool/mjit_tabs.rb: Skipping #{target.dump} as the encoding is #{source.encoding}." - next - end - - sources[target] = source - mtimes[target] = File.mtime(target) - - if sources[target] == expanded - puts "#{target.dump} has no hard tab indentation. This should be ignored in tool/mjit_tabs.rb." - end - File.write(target, expanded) - FileUtils.touch(target, mtime: mtimes[target]) -end - -result = system(*command) - -targets.each do |target| - if sources.key?(target) - File.write(target, sources[target]) - FileUtils.touch(target, mtime: mtimes.fetch(target)) - end -end - -exit result diff --git a/tool/mk_builtin_loader.rb b/tool/mk_builtin_loader.rb index 23e6a01017..4abd497f0e 100644 --- a/tool/mk_builtin_loader.rb +++ b/tool/mk_builtin_loader.rb @@ -4,6 +4,10 @@ require 'ripper' require 'stringio' require_relative 'ruby_vm/helpers/c_escape' +SUBLIBS = {} +REQUIRED = {} +BUILTIN_ATTRS = %w[leaf inline_block use_block] + def string_literal(lit, str = []) while lit case lit.first @@ -22,6 +26,17 @@ def string_literal(lit, str = []) end end +# e.g. [:symbol_literal, [:symbol, [:@ident, "inline", [19, 21]]]] +def symbol_literal(lit) + symbol_literal, symbol_lit = lit + raise "#{lit.inspect} was not :symbol_literal" if symbol_literal != :symbol_literal + symbol, ident_lit = symbol_lit + raise "#{symbol_lit.inspect} was not :symbol" if symbol != :symbol + ident, symbol_name, = ident_lit + raise "#{ident.inspect} was not :@ident" if ident != :@ident + symbol_name +end + def inline_text argc, arg1 raise "argc (#{argc}) of inline! should be 1" unless argc == 1 arg1 = string_literal(arg1) @@ -29,6 +44,16 @@ def inline_text argc, arg1 arg1.join("").rstrip end +def inline_attrs(args) + raise "args was empty" if args.empty? + args.each do |arg| + attr = symbol_literal(arg) + unless BUILTIN_ATTRS.include?(attr) + raise "attr (#{attr}) was not in: #{BUILTIN_ATTRS.join(', ')}" + end + end +end + def make_cfunc_name inlines, name, lineno case name when /\[\]/ @@ -84,7 +109,7 @@ def collect_builtin base, tree, name, bs, inlines, locals = nil tree = tree[2] next when :method_add_arg - _, mid, (_, (_, args)) = tree + _method_add_arg, mid, (_arg_paren, args) = tree case mid.first when :call _, recv, sep, mid = mid @@ -93,6 +118,11 @@ def collect_builtin base, tree, name, bs, inlines, locals = nil else mid = nil end + # w/ trailing comma: [[:method_add_arg, ...]] + # w/o trailing comma: [:args_add_block, [[:method_add_arg, ...]], false] + if args && args.first == :args_add_block + args = args[1] + end when :vcall _, mid = tree when :command # FCALL @@ -130,15 +160,13 @@ def collect_builtin base, tree, name, bs, inlines, locals = nil if /(.+)[\!\?]\z/ =~ func_name case $1 when 'attr' - text = inline_text(argc, args.first) - if text != 'inline' - raise "Only 'inline' is allowed to be annotated (but got: '#{text}')" - end + # Compile-time validation only. compile.c will parse them. + inline_attrs(args) break when 'cstmt' text = inline_text argc, args.first - func_name = "_bi#{inlines.size}" + func_name = "_bi#{lineno}" cfunc_name = make_cfunc_name(inlines, name, lineno) inlines[cfunc_name] = [lineno, text, locals, func_name] argc -= 1 @@ -146,7 +174,7 @@ def collect_builtin base, tree, name, bs, inlines, locals = nil text = inline_text argc, args.first code = "return #{text};" - func_name = "_bi#{inlines.size}" + func_name = "_bi#{lineno}" cfunc_name = make_cfunc_name(inlines, name, lineno) locals = [] if $1 == 'cconst' @@ -174,6 +202,21 @@ def collect_builtin base, tree, name, bs, inlines, locals = nil end bs[func_name] = [argc, cfunc_name] if func_name + elsif /\Arequire(?:_relative)\z/ =~ mid and args.size == 1 and + (arg1 = args[0])[0] == :string_literal and + (arg1 = arg1[1])[0] == :string_content and + (arg1 = arg1[1])[0] == :@tstring_content and + sublib = arg1[1] + if File.exist?(f = File.join(@dir, sublib)+".rb") + puts "- #{@base}.rb requires #{sublib}" + if REQUIRED[sublib] + warn "!!! #{sublib} is required from #{REQUIRED[sublib]} already; ignored" + else + REQUIRED[sublib] = @base + (SUBLIBS[@base] ||= []) << sublib + end + ARGV.push(f) + end end break unless tree = args end @@ -220,11 +263,19 @@ end def generate_cexpr(ofile, lineno, line_file, body_lineno, text, locals, func_name) f = StringIO.new + + # Avoid generating fetches of lvars we don't need. This is imperfect as it + # will match text inside strings or other false positives. + local_candidates = text.scan(/[a-zA-Z_][a-zA-Z0-9_]*/) + f.puts '{' lineno += 1 - locals.reverse_each.with_index{|param, i| + # locals is nil outside methods + locals&.reverse_each&.with_index{|param, i| next unless Symbol === param - f.puts "MAYBE_UNUSED(const VALUE) #{param} = rb_vm_lvar(ec, #{-3 - i});" + next unless local_candidates.include?(param.to_s) + f.puts "VALUE *const #{param}__ptr = (VALUE *)&ec->cfp->ep[#{-3 - i}];" + f.puts "MAYBE_UNUSED(const VALUE) #{param} = *#{param}__ptr;" lineno += 1 } f.puts "#line #{body_lineno} \"#{line_file}\"" @@ -242,7 +293,9 @@ def generate_cexpr(ofile, lineno, line_file, body_lineno, text, locals, func_nam end def mk_builtin_header file + @dir = File.dirname(file) base = File.basename(file, '.rb') + @base = base ofile = "#{file}inc" # bs = { func_name => argc } @@ -251,10 +304,10 @@ def mk_builtin_header file collect_builtin(base, Ripper.sexp(code), 'top', bs = {}, inlines = {}) begin - f = open(ofile, 'w') - rescue Errno::EACCES + f = File.open(ofile, 'w') + rescue SystemCallError # EACCES, EPERM, EROFS, etc. # Fall back to the current directory - f = open(File.basename(ofile), 'w') + f = File.open(File.basename(ofile), 'w') end begin if File::ALT_SEPARATOR @@ -293,43 +346,13 @@ def mk_builtin_header file end } - bs.each_pair{|func, (argc, cfunc_name)| - decl = ', VALUE' * argc - argv = argc \ - . times \ - . map {|i|", argv[#{i}]"} \ - . join('') - f.puts %'static void' - f.puts %'mjit_compile_invokebuiltin_for_#{func}(FILE *f, long index, unsigned stack_size, bool inlinable_p)' - f.puts %'{' - f.puts %' fprintf(f, " VALUE self = GET_SELF();\\n");' - f.puts %' fprintf(f, " typedef VALUE (*func)(rb_execution_context_t *, VALUE#{decl});\\n");' - if inlines.has_key? cfunc_name - body_lineno, text, locals, func_name = inlines[cfunc_name] - lineno, str = generate_cexpr(ofile, lineno, line_file, body_lineno, text, locals, func_name) - f.puts %' if (inlinable_p) {' - str.gsub(/^(?!#)/, ' ').each_line {|i| - j = RubyVM::CEscape.rstring2cstr(i).dup - j.sub!(/^ return\b/ , ' val =') - f.printf(%' fprintf(f, "%%s", %s);\n', j) - } - f.puts(%' return;') - f.puts(%' }') + if SUBLIBS[base] + f.puts "// sub libraries" + SUBLIBS[base].each do |sub| + f.puts %[#include #{(sub+".rbinc").dump}] end - if argc > 0 - f.puts %' if (index == -1) {' - f.puts %' fprintf(f, " const VALUE *argv = &stack[%d];\\n", stack_size - #{argc});' - f.puts %' }' - f.puts %' else {' - f.puts %' fprintf(f, " const unsigned int lnum = ISEQ_BODY(GET_ISEQ())->local_table_size;\\n");' - f.puts %' fprintf(f, " const VALUE *argv = GET_EP() - lnum - VM_ENV_DATA_SIZE + 1 + %ld;\\n", index);' - f.puts %' }' - end - f.puts %' fprintf(f, " func f = (func)%"PRIuVALUE"; /* == #{cfunc_name} */\\n", (VALUE)#{cfunc_name});' - f.puts %' fprintf(f, " val = f(ec, self#{argv});\\n");' - f.puts %'}' f.puts - } + end f.puts "void Init_builtin_#{base}(void)" f.puts "{" @@ -338,9 +361,9 @@ def mk_builtin_header file f.puts " // table definition" f.puts " static const struct rb_builtin_function #{table}[] = {" bs.each.with_index{|(func, (argc, cfunc_name)), i| - f.puts " RB_BUILTIN_FUNCTION(#{i}, #{func}, #{cfunc_name}, #{argc}, mjit_compile_invokebuiltin_for_#{func})," + f.puts " RB_BUILTIN_FUNCTION(#{i}, #{func}, #{cfunc_name}, #{argc})," } - f.puts " RB_BUILTIN_FUNCTION(-1, NULL, NULL, 0, 0)," + f.puts " RB_BUILTIN_FUNCTION(-1, NULL, NULL, 0)," f.puts " };" f.puts @@ -354,6 +377,14 @@ def mk_builtin_header file } f.puts "COMPILER_WARNING_POP" + if SUBLIBS[base] + f.puts + f.puts " // sub libraries" + SUBLIBS[base].each do |sub| + f.puts " Init_builtin_#{sub}();" + end + end + f.puts f.puts " // load" f.puts " rb_load_with_builtin_functions(#{base.dump}, #{table});" diff --git a/tool/mkconfig.rb b/tool/mkconfig.rb index 120b90850d..55e781a28e 100755 --- a/tool/mkconfig.rb +++ b/tool/mkconfig.rb @@ -63,8 +63,8 @@ File.foreach "config.status" do |line| when /^(?:X|(?:MINI|RUN|(?:HAVE_)?BASE|BOOTSTRAP|BTEST)RUBY(?:_COMMAND)?$)/; next when /^INSTALLDOC|TARGET$/; next when /^DTRACE/; next - when /^MJIT_(CC|SUPPORT)$/; # pass - when /^MJIT_/; next + when /^RJIT_(CC|SUPPORT)$/; # pass + when /^RJIT_/; next when /^(?:MAJOR|MINOR|TEENY)$/; vars[name] = val; next when /^LIBRUBY_D?LD/; next when /^RUBY_INSTALL_NAME$/; next vars[name] = (install_name = val).dup if $install_name @@ -122,8 +122,16 @@ File.foreach "config.status" do |line| universal, val = val, 'universal' if universal when /^arch$/ if universal - platform = val.sub(/universal/, %q[#{arch && universal[/(?:\A|\s)#{Regexp.quote(arch)}=(\S+)/, 1] || RUBY_PLATFORM[/\A[^-]*/]}]) + platform = val.sub(/universal/, '$(arch)') end + when /^target_cpu$/ + if universal + val = 'cpu' + end + when /^target$/ + val = '"$(target_cpu)-$(target_vendor)-$(target_os)"' + when /^host(?:_(?:os|vendor|cpu|alias))?$/ + val = %["$(#{name.sub(/^host/, 'target')})"] when /^includedir$/ val = '"$(SDKROOT)"'+val if /darwin/ =~ arch end @@ -185,17 +193,19 @@ print " # Ruby installed directory.\n" print " TOPDIR = File.dirname(__FILE__).chomp!(#{relative_archdir.dump})\n" print " # DESTDIR on make install.\n" print " DESTDIR = ", (drive ? "TOPDIR && TOPDIR[/\\A[a-z]:/i] || " : ""), "'' unless defined? DESTDIR\n" -print <<'ARCH' if universal +print <<"UNIVERSAL", <<'ARCH' if universal + universal = #{universal} +UNIVERSAL arch_flag = ENV['ARCHFLAGS'] || ((e = ENV['RC_ARCHS']) && e.split.uniq.map {|a| "-arch #{a}"}.join(' ')) arch = arch_flag && arch_flag[/\A\s*-arch\s+(\S+)\s*\z/, 1] + cpu = arch && universal[/(?:\A|\s)#{Regexp.quote(arch)}=(\S+)/, 1] || RUBY_PLATFORM[/\A[^-]*/] ARCH -print " universal = #{universal}\n" if universal print " # The hash configurations stored.\n" print " CONFIG = {}\n" print " CONFIG[\"DESTDIR\"] = DESTDIR\n" versions = {} -IO.foreach(File.join(srcdir, "version.h")) do |l| +File.foreach(File.join(srcdir, "version.h")) do |l| m = /^\s*#\s*define\s+RUBY_(PATCHLEVEL)\s+(-?\d+)/.match(l) if m versions[m[1]] = m[2] @@ -216,7 +226,7 @@ IO.foreach(File.join(srcdir, "version.h")) do |l| end end if versions.size != 4 - IO.foreach(File.join(srcdir, "include/ruby/version.h")) do |l| + File.foreach(File.join(srcdir, "include/ruby/version.h")) do |l| m = /^\s*#\s*define\s+RUBY_API_VERSION_(\w+)\s+(-?\d+)/.match(l) if m versions[m[1]] ||= m[2] @@ -268,7 +278,7 @@ EOS print <<EOS if $unicode_emoji_version CONFIG["UNICODE_EMOJI_VERSION"] = #{$unicode_emoji_version.dump} EOS -print <<EOS if /darwin/ =~ arch +print prefix.start_with?("/System/") ? <<EOS : <<EOS if /darwin/ =~ arch if sdkroot = ENV["SDKROOT"] sdkroot = sdkroot.dup elsif File.exist?(File.join(CONFIG["prefix"], "include")) || @@ -279,6 +289,8 @@ print <<EOS if /darwin/ =~ arch end CONFIG["SDKROOT"] = sdkroot EOS + CONFIG["SDKROOT"] = "" +EOS print <<EOS CONFIG["platform"] = #{platform || '"$(arch)"'} CONFIG["archdir"] = "$(rubyarchdir)" diff --git a/tool/mkrunnable.rb b/tool/mkrunnable.rb index 3b71b0751b..3a62fea80f 100755 --- a/tool/mkrunnable.rb +++ b/tool/mkrunnable.rb @@ -6,6 +6,7 @@ require './rbconfig' require 'fileutils' +require_relative 'lib/path' case ARGV[0] when "-n" @@ -18,91 +19,7 @@ else include FileUtils end -module Mswin - def ln_safe(src, dest, *opt) - cmd = ["mklink", dest.tr("/", "\\"), src.tr("/", "\\")] - cmd[1, 0] = opt - return if system("cmd", "/c", *cmd) - # TODO: use RUNAS or something - puts cmd.join(" ") - end - - def ln_dir_safe(src, dest) - ln_safe(src, dest, "/d") - end -end - -def clean_link(src, dest) - begin - link = File.readlink(dest) - rescue - else - return if link == src - File.unlink(dest) - end - yield src, dest -end - -def ln_safe(src, dest) - ln_sf(src, dest) -rescue Errno::ENOENT - # Windows disallows to create broken symboic links, probably because - # it is a kind of reparse points. - raise if File.exist?(src) -end - -alias ln_dir_safe ln_safe - -case RUBY_PLATFORM -when /linux|darwin|solaris/ - def ln_exe(src, dest) - ln(src, dest, force: true) - end -else - alias ln_exe ln_safe -end - -if !File.respond_to?(:symlink) && /mingw|mswin/ =~ (CROSS_COMPILING || RUBY_PLATFORM) - extend Mswin -end - -def clean_path(path) - path = "#{path}/".gsub(/(\A|\/)(?:\.\/)+/, '\1').tr_s('/', '/') - nil while path.sub!(/[^\/]+\/\.\.\//, '') - path -end - -def relative_path_from(path, base) - path = clean_path(path) - base = clean_path(base) - path, base = [path, base].map{|s|s.split("/")} - until path.empty? or base.empty? or path[0] != base[0] - path.shift - base.shift - end - path, base = [path, base].map{|s|s.join("/")} - if /(\A|\/)\.\.\// =~ base - File.expand_path(path) - else - base.gsub!(/[^\/]+/, '..') - File.join(base, path) - end -end - -def ln_relative(src, dest, executable = false) - return if File.identical?(src, dest) - parent = File.dirname(dest) - File.directory?(parent) or mkdir_p(parent) - return ln_exe(src, dest) if executable - clean_link(relative_path_from(src, parent), dest) {|s, d| ln_safe(s, d)} -end - -def ln_dir_relative(src, dest) - return if File.identical?(src, dest) - parent = File.dirname(dest) - File.directory?(parent) or mkdir_p(parent) - clean_link(relative_path_from(src, parent), dest) {|s, d| ln_dir_safe(s, d)} -end +include Path config = RbConfig::MAKEFILE_CONFIG.merge("prefix" => ".", "exec_prefix" => ".") config.each_value {|s| RbConfig.expand(s, config)} @@ -117,18 +34,22 @@ vendordir = config["vendordir"] rubylibdir = config["rubylibdir"] rubyarchdir = config["rubyarchdir"] archdir = "#{extout}/#{arch}" -[bindir, libdir, archdir].uniq.each do |dir| +exedir = libdirname == "archlibdir" ? "#{config["libexecdir"]}/#{arch}/bin" : bindir +[exedir, libdir, archdir].uniq.each do |dir| File.directory?(dir) or mkdir_p(dir) end +unless exedir == bindir + ln_dir_relative(exedir, bindir) +end exeext = config["EXEEXT"] ruby_install_name = config["ruby_install_name"] rubyw_install_name = config["rubyw_install_name"] goruby_install_name = "go" + ruby_install_name -[ruby_install_name, rubyw_install_name, goruby_install_name].map do |ruby| +[ruby_install_name, rubyw_install_name, goruby_install_name].each do |ruby| if ruby and !ruby.empty? ruby += exeext - ln_relative(ruby, "#{bindir}/#{ruby}", true) + ln_relative(ruby, "#{exedir}/#{ruby}", true) end end so = config["LIBRUBY_SO"] diff --git a/tool/outdate-bundled-gems.rb b/tool/outdate-bundled-gems.rb new file mode 100755 index 0000000000..c82d31d743 --- /dev/null +++ b/tool/outdate-bundled-gems.rb @@ -0,0 +1,190 @@ +#!/usr/bin/ruby +require 'fileutils' +require 'rubygems' + +fu = FileUtils::Verbose + +until ARGV.empty? + case ARGV.first + when '--' + ARGV.shift + break + when '-n', '--dry-run', '--dryrun' + ## -n, --dry-run Don't remove + fu = FileUtils::DryRun + when /\A--make=/ + # just to run when `make -n` + when /\A--mflags=(.*)/ + fu = FileUtils::DryRun if /\A-\S*n/ =~ $1 + when /\A--gem[-_]platform=(.*)/im + ## --gem-platform=PLATFORM Platform in RubyGems style + gem_platform = $1 + ruby_platform = nil + when /\A--ruby[-_]platform=(.*)/im + ## --ruby-platform=PLATFORM Platform in Ruby style + ruby_platform = $1 + gem_platform = nil + when /\A--ruby[-_]version=(.*)/im + ## --ruby-version=VERSION Ruby version to keep + ruby_version = $1 + when /\A--only=(?:(curdir|srcdir)|all)\z/im + ## --only=(curdir|srcdir|all) Specify directory to remove gems from + only = $1&.downcase + when /\A--all\z/im + ## --all Remove all gems not only bundled gems + all = true + when /\A--help\z/im + ## --help Print this message + puts "Usage: #$0 [options] [srcdir]" + File.foreach(__FILE__) do |line| + line.sub!(/^ *## /, "") or next + break if line.chomp!.empty? + opt, desc = line.split(/ {2,}/, 2) + printf " %-28s %s\n", opt, desc + end + exit + when /\A-/ + raise "#{$0}: unknown option: #{ARGV.first}" + else + break + end + ## + ARGV.shift +end + +gem_platform ||= Gem::Platform.new(ruby_platform).to_s if ruby_platform + +class Removal + attr_reader :base + + def initialize(base = nil) + @base = (File.join(base, "/") if base) + @remove = {} + end + + def prefixed(name) + @base ? File.join(@base, name) : name + end + + def stripped(name) + if @base && name.start_with?(@base) + name[@base.size..-1] + else + name + end + end + + def slash(name) + name.sub(%r[[^/]\K\z], '/') + end + + def exist?(name) + !@remove.fetch(name) {|k| @remove[k] = !File.exist?(prefixed(name))} + end + def directory?(name) + !@remove.fetch(slash(name)) {|k| @remove[k] = !File.directory?(prefixed(name))} + end + + def unlink(name) + @remove[stripped(name)] = :rm_f + end + def rmdir(name) + @remove[slash(stripped(name))] = :rm_rf + end + + def glob(pattern, *rest) + Dir.glob(prefixed(pattern), *rest) {|n| + yield stripped(n) + } + end + + def sorted + @remove.sort_by {|k, | [-k.count("/"), k]} + end + + def each_file + sorted.each {|k, v| yield prefixed(k) if v == :rm_f} + end + + def each_directory + sorted.each {|k, v| yield prefixed(k) if v == :rm_rf} + end +end + +srcdir = Removal.new(ARGV.shift) +curdir = !srcdir.base || File.identical?(srcdir.base, ".") ? srcdir : Removal.new + +bundled = File.readlines("#{srcdir.base}gems/bundled_gems"). + grep(/^(\w\S+)\s+\S+(?:\s+\S+\s+(\S+))?/) {$~.captures}.to_h rescue nil + +srcdir.glob(".bundle/gems/*/") do |dir| + base = File.basename(dir) + next if !all && bundled && !bundled.key?(base[/\A.+(?=-)/]) + unless srcdir.exist?("gems/#{base}.gem") + srcdir.rmdir(dir) + end +end + +srcdir.glob(".bundle/.timestamp/*.revision") do |file| + unless bundled&.fetch(File.basename(file, ".revision"), nil) + srcdir.unlink(file) + end +end + +srcdir.glob(".bundle/specifications/*.gemspec") do |spec| + unless srcdir.directory?(".bundle/gems/#{File.basename(spec, '.gemspec')}/") + srcdir.unlink(spec) + end +end + +curdir.glob(".bundle/specifications/*.gemspec") do |spec| + unless srcdir.directory?(".bundle/gems/#{File.basename(spec, '.gemspec')}") + curdir.unlink(spec) + end +end + +curdir.glob(".bundle/gems/*/") do |dir| + base = File.basename(dir) + unless curdir.exist?(".bundle/specifications/#{base}.gemspec") or + curdir.exist?("#{dir}/.bundled.#{base}.gemspec") + curdir.rmdir(dir) + end +end + +curdir.glob(".bundle/{extensions,.timestamp}/*/") do |dir| + unless gem_platform and File.fnmatch?(gem_platform, File.basename(dir)) + curdir.rmdir(dir) + end +end + +if gem_platform + curdir.glob(".bundle/{extensions,.timestamp}/#{gem_platform}/*/") do |dir| + unless ruby_version and File.fnmatch?(ruby_version, File.basename(dir, '-static')) + curdir.rmdir(dir) + end + end +end + +if ruby_version + curdir.glob(".bundle/extensions/#{gem_platform || '*'}/#{ruby_version}/*/") do |dir| + unless curdir.exist?(".bundle/specifications/#{File.basename(dir)}.gemspec") + curdir.rmdir(dir) + end + end + + curdir.glob(".bundle/.timestamp/#{gem_platform || '*'}/#{ruby_version}/.*.time") do |stamp| + dir = stamp[%r[/\.([^/]+)\.time\z], 1].gsub('.-.', '/')[%r[\A[^/]+/[^/]+]] + unless curdir.directory?(File.join(".bundle", dir)) + curdir.unlink(stamp) + end + end +end + +unless only == "curdir" + srcdir.each_file {|f| fu.rm_f(f)} + srcdir.each_directory {|d| fu.rm_rf(d)} +end +unless only == "srcdir" or curdir.equal?(srcdir) + curdir.each_file {|f| fu.rm_f(f)} + curdir.each_directory {|d| fu.rm_rf(d)} +end diff --git a/tool/pure_parser.rb b/tool/pure_parser.rb deleted file mode 100755 index 21c87cc5d6..0000000000 --- a/tool/pure_parser.rb +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/ruby -pi.bak -BEGIN { - # pathological setting - ENV['LANG'] = ENV['LC_MESSAGES'] = ENV['LC_ALL'] = 'C' - - require_relative 'lib/colorize' - - colorize = Colorize.new - file = ARGV.shift - begin - version = IO.popen(ARGV+%w[--version], "rb", &:read) - rescue Errno::ENOENT - abort "Failed to run `#{colorize.fail ARGV.join(' ')}'; You may have to install it." - end - unless /\Abison .* (\d+)\.\d+/ =~ version - puts colorize.fail("not bison") - exit - end - exit if $1.to_i >= 3 - ARGV.clear - ARGV.push(file) -} -$_.sub!(/^%define\s+api\.pure/, '%pure-parser') -$_.sub!(/^%define\s+.*/, '') diff --git a/tool/rbinstall.rb b/tool/rbinstall.rb index c944ef74da..63f4beb943 100755 --- a/tool/rbinstall.rb +++ b/tool/rbinstall.rb @@ -28,6 +28,7 @@ begin rescue LoadError $" << "zlib.rb" end +require_relative 'lib/path' INDENT = " "*36 STDOUT.sync = true @@ -96,6 +97,20 @@ def parse_args(argv = ARGV) opt.on('--gnumake') {gnumake = true} opt.on('--debug-symbols=SUFFIX', /\w+/) {|name| $debug_symbols = ".#{name}"} + unless $install_procs.empty? + w = (w = ENV["COLUMNS"] and (w = w.to_i) > 80) ? w - 30 : 50 + opt.on("\n""Types for --install and --exclude:") + mesg = +" " + $install_procs.each_key do |t| + if mesg.size + t.size > w + opt.on(mesg) + mesg = +" " + end + mesg << " " << t.to_s + end + opt.on(mesg) + end + opt.order!(argv) do |v| case v when /\AINSTALL[-_]([-\w]+)=(.*)/ @@ -143,7 +158,7 @@ def parse_args(argv = ARGV) if $installed_list ||= $mflags.defined?('INSTALLED_LIST') RbConfig.expand($installed_list, RbConfig::CONFIG) - $installed_list = open($installed_list, "ab") + $installed_list = File.open($installed_list, "ab") $installed_list.sync = true end @@ -205,15 +220,20 @@ def ln_sf(src, dest) end $made_dirs = {} + +def dir_creating(dir) + $made_dirs.fetch(dir) do + $made_dirs[dir] = true + $installed_list.puts(File.join(dir, "")) if $installed_list + yield if defined?(yield) + end +end + def makedirs(dirs) dirs = fu_list(dirs) dirs.collect! do |dir| realdir = with_destdir(dir) - realdir unless $made_dirs.fetch(dir) do - $made_dirs[dir] = true - $installed_list.puts(File.join(dir, "")) if $installed_list - File.directory?(realdir) - end + realdir unless dir_creating(dir) {File.directory?(realdir)} end.compact! super(dirs, :mode => $dir_mode) unless dirs.empty? end @@ -291,11 +311,11 @@ def install_recursive(srcdir, dest, options = {}) end def open_for_install(path, mode) - data = open(realpath = with_destdir(path), "rb") {|f| f.read} rescue nil + data = File.binread(realpath = with_destdir(path)) rescue nil newdata = yield unless $dryrun unless newdata == data - open(realpath, "wb", mode) {|f| f.write newdata} + File.open(realpath, "wb", mode) {|f| f.write newdata} end File.chmod(mode, realpath) end @@ -346,6 +366,13 @@ rubyw_install_name = CONFIG["rubyw_install_name"] goruby_install_name = "go" + ruby_install_name bindir = CONFIG["bindir", true] +if CONFIG["libdirname"] == "archlibdir" + libexecdir = MAKEFILE_CONFIG["archlibdir"].dup + unless libexecdir.sub!(/\$\(lib\K(?=dir\))/) {"exec"} + libexecdir = "$(libexecdir)/$(arch)" + end + archbindir = RbConfig.expand(libexecdir) + "/bin" +end libdir = CONFIG[CONFIG.fetch("libdirname", "libdir"), true] rubyhdrdir = CONFIG["rubyhdrdir", true] archhdrdir = CONFIG["rubyarchhdrdir"] || (rubyhdrdir + "/" + CONFIG['arch']) @@ -369,106 +396,6 @@ load_relative = CONFIG["LIBRUBY_RELATIVE"] == 'yes' rdoc_noinst = %w[created.rid] -install?(:local, :arch, :bin, :'bin-arch') do - prepare "binary commands", bindir - - install ruby_install_name+exeext, bindir, :mode => $prog_mode, :strip => $strip - if rubyw_install_name and !rubyw_install_name.empty? - install rubyw_install_name+exeext, bindir, :mode => $prog_mode, :strip => $strip - end - # emcc produces ruby and ruby.wasm, the first is a JavaScript file of runtime support - # to load and execute the second .wasm file. Both are required to execute ruby - if RUBY_PLATFORM =~ /emscripten/ and File.exist? ruby_install_name+".wasm" - install ruby_install_name+".wasm", bindir, :mode => $prog_mode, :strip => $strip - end - if File.exist? goruby_install_name+exeext - install goruby_install_name+exeext, bindir, :mode => $prog_mode, :strip => $strip - end - if enable_shared and dll != lib - install dll, bindir, :mode => $prog_mode, :strip => $strip - end -end - -install?(:local, :arch, :lib, :'lib-arch') do - prepare "base libraries", libdir - - install lib, libdir, :mode => $prog_mode, :strip => $strip unless lib == arc - install arc, libdir, :mode => $data_mode unless CONFIG["INSTALL_STATIC_LIBRARY"] == "no" - if dll == lib and dll != arc - for link in CONFIG["LIBRUBY_ALIASES"].split - [File.basename(dll)] - ln_sf(dll, File.join(libdir, link)) - end - end - - prepare "arch files", archlibdir - install "rbconfig.rb", archlibdir, :mode => $data_mode - if CONFIG["ARCHFILE"] - for file in CONFIG["ARCHFILE"].split - install file, archlibdir, :mode => $data_mode - end - end -end - -install?(:local, :arch, :data) do - pc = CONFIG["ruby_pc"] - if pc and File.file?(pc) and File.size?(pc) - prepare "pkgconfig data", pkgconfigdir = File.join(libdir, "pkgconfig") - install pc, pkgconfigdir, :mode => $data_mode - end -end - -install?(:ext, :arch, :'ext-arch') do - prepare "extension objects", archlibdir - noinst = %w[-* -*/] | (CONFIG["no_install_files"] || "").split - install_recursive("#{$extout}/#{CONFIG['arch']}", archlibdir, :no_install => noinst, :mode => $prog_mode, :strip => $strip) - prepare "extension objects", sitearchlibdir - prepare "extension objects", vendorarchlibdir - if extso = File.read("exts.mk")[/^EXTSO[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] and - !(extso = extso.gsub(/\\\n/, '').split).empty? - libpathenv = CONFIG["LIBPATHENV"] - dest = CONFIG[!libpathenv || libpathenv == "PATH" ? "bindir" : "libdir"] - prepare "external libraries", dest - for file in extso - install file, dest, :mode => $prog_mode - end - end -end -install?(:ext, :arch, :hdr, :'arch-hdr', :'hdr-arch') do - prepare "extension headers", archhdrdir - install_recursive("#{$extout}/include/#{CONFIG['arch']}", archhdrdir, :glob => "*.h", :mode => $data_mode) - install_recursive("#{$extout}/include/#{CONFIG['arch']}", archhdrdir, :glob => "rb_mjit_header-*.obj", :mode => $data_mode) - install_recursive("#{$extout}/include/#{CONFIG['arch']}", archhdrdir, :glob => "rb_mjit_header-*.pch", :mode => $data_mode) -end -install?(:ext, :comm, :'ext-comm') do - prepare "extension scripts", rubylibdir - install_recursive("#{$extout}/common", rubylibdir, :mode => $data_mode) - prepare "extension scripts", sitelibdir - prepare "extension scripts", vendorlibdir -end -install?(:ext, :comm, :hdr, :'comm-hdr', :'hdr-comm') do - hdrdir = rubyhdrdir + "/ruby" - prepare "extension headers", hdrdir - install_recursive("#{$extout}/include/ruby", hdrdir, :glob => "*.h", :mode => $data_mode) -end - -install?(:doc, :rdoc) do - if $rdocdir - ridatadir = File.join(CONFIG['ridir'], CONFIG['ruby_version'], "system") - prepare "rdoc", ridatadir - install_recursive($rdocdir, ridatadir, :no_install => rdoc_noinst, :mode => $data_mode) - end -end -install?(:doc, :html) do - if $htmldir - prepare "html-docs", docdir - install_recursive($htmldir, docdir+"/html", :no_install => rdoc_noinst, :mode => $data_mode) - end -end -install?(:doc, :capi) do - prepare "capi-docs", docdir - install_recursive "doc/capi", docdir+"/capi", :mode => $data_mode -end - prolog_script = <<EOS bindir="#{load_relative ? '${0%/*}' : bindir.gsub(/\"/, '\\\\"')}" EOS @@ -550,7 +477,7 @@ $script_installer = Class.new(installer) do def install(src, cmd) cmd = cmd.sub(/[^\/]*\z/m) {|n| transform(n)} - shebang, body = open(src, "rb") do |f| + shebang, body = File.open(src, "rb") do |f| next f.gets, f.read end shebang or raise "empty file - #{src}" @@ -584,126 +511,6 @@ $script_installer = Class.new(installer) do break new(ruby_shebang, ruby_bin, ruby_install_name, nil, trans) end -install?(:local, :comm, :bin, :'bin-comm') do - prepare "command scripts", bindir - - install_recursive(File.join(srcdir, "bin"), bindir, :maxdepth => 1) do |src, cmd| - $script_installer.install(src, cmd) - end -end - -install?(:local, :comm, :lib) do - prepare "library scripts", rubylibdir - noinst = %w[*.txt *.rdoc *.gemspec] - install_recursive(File.join(srcdir, "lib"), rubylibdir, :no_install => noinst, :mode => $data_mode) -end - -install?(:local, :comm, :hdr, :'comm-hdr') do - prepare "common headers", rubyhdrdir - - noinst = [] - unless RUBY_PLATFORM =~ /mswin|mingw|bccwin/ - noinst << "win32.h" - end - noinst = nil if noinst.empty? - install_recursive(File.join(srcdir, "include"), rubyhdrdir, :no_install => noinst, :glob => "*.{h,hpp}", :mode => $data_mode) -end - -install?(:local, :comm, :man) do - mdocs = Dir["#{srcdir}/man/*.[1-9]"] - prepare "manpages", mandir, ([] | mdocs.collect {|mdoc| mdoc[/\d+$/]}).sort.collect {|sec| "man#{sec}"} - - case $mantype - when /\.(?:(gz)|bz2)\z/ - compress = $1 ? "gzip" : "bzip2" - suffix = $& - end - mandir = File.join(mandir, "man") - has_goruby = File.exist?(goruby_install_name+exeext) - require File.join(srcdir, "tool/mdoc2man.rb") if /\Adoc\b/ !~ $mantype - mdocs.each do |mdoc| - next unless File.file?(mdoc) and open(mdoc){|fh| fh.read(1) == '.'} - base = File.basename(mdoc) - if base == "goruby.1" - next unless has_goruby - end - - destdir = mandir + (section = mdoc[/\d+$/]) - destname = ruby_install_name.sub(/ruby/, base.chomp(".#{section}")) - destfile = File.join(destdir, "#{destname}.#{section}") - - if /\Adoc\b/ =~ $mantype - if compress - w = open(mdoc) {|f| - stdin = STDIN.dup - STDIN.reopen(f) - begin - destfile << suffix - IO.popen(compress, &:read) - ensure - STDIN.reopen(stdin) - stdin.close - end - } - open_for_install(destfile, $data_mode) {w} - else - install mdoc, destfile, :mode => $data_mode - end - else - class << (w = []) - alias print push - end - if File.basename(mdoc).start_with?('bundle') || - File.basename(mdoc).start_with?('gemfile') - w = File.read(mdoc) - else - open(mdoc) {|r| Mdoc2Man.mdoc2man(r, w)} - w = w.join("") - end - if compress - require 'tmpdir' - Dir.mktmpdir("man") {|d| - dest = File.join(d, File.basename(destfile)) - File.open(dest, "wb") {|f| f.write w} - if system(compress, dest) - w = File.open(dest+suffix, "rb") {|f| f.read} - destfile << suffix - end - } - end - open_for_install(destfile, $data_mode) {w} - end - end -end - -install?(:dbg, :nodefault) do - prepare "debugger commands", bindir - prepare "debugger scripts", rubylibdir - conf = RbConfig::MAKEFILE_CONFIG.merge({"prefix"=>"${prefix#/}"}) - Dir.glob(File.join(srcdir, "template/ruby-*db.in")) do |src| - cmd = $script_installer.transform(File.basename(src, ".in")) - open_for_install(File.join(bindir, cmd), $script_mode) { - RbConfig.expand(File.read(src), conf) - } - end - install File.join(srcdir, "misc/lldb_cruby.py"), File.join(rubylibdir, "lldb_cruby.py") - install File.join(srcdir, ".gdbinit"), File.join(rubylibdir, "gdbinit") - if $debug_symbols - { - ruby_install_name => bindir, - rubyw_install_name => bindir, - goruby_install_name => bindir, - dll => libdir, - }.each do |src, dest| - next if src.empty? - src += $debug_symbols - if File.directory?(src) - install_recursive src, File.join(dest, src) - end - end - end -end - module RbInstall def self.no_write(options = nil) u = File.umask(0022) @@ -742,47 +549,108 @@ module RbInstall end def collect - ruby_libraries.sort + requirable_features.sort + end + + private + + def features_from_makefile(makefile_path) + makefile = File.read(makefile_path) + + name = makefile[/^TARGET[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] + return [] if name.nil? || name.empty? + + feature = makefile[/^DLLIB[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] + feature = feature.sub("$(TARGET)", name) + + target_prefix = makefile[/^target_prefix[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] + feature = File.join(target_prefix.delete_prefix("/"), feature) unless target_prefix.empty? + + Array(feature) end class Ext < self - def skip_install?(files) + def requirable_features # install ext only when it's configured - !File.exist?("#{$ext_build_dir}/#{relative_base}/Makefile") + return [] unless File.exist?(makefile_path) + + ruby_features + ext_features end - def ruby_libraries - Dir.glob("lib/**/*.rb", base: "#{srcdir}/ext/#{relative_base}") + private + + def ruby_features + Dir.glob("**/*.rb", base: "#{makefile_dir}/lib") + end + + def ext_features + features_from_makefile(makefile_path) + end + + def makefile_path + if File.exist?("#{makefile_dir}/Makefile") + "#{makefile_dir}/Makefile" + else + # for out-of-place build + "#{$ext_build_dir}/#{relative_base}/Makefile" + end + end + + def makefile_dir + "#{root}/#{relative_base}" + end + + def root + File.expand_path($ext_build_dir, srcdir) end end class Lib < self - def skip_install?(files) - files.empty? + def requirable_features + ruby_features + ext_features end - def ruby_libraries + private + + def ruby_features gemname = File.basename(gemspec, ".gemspec") base = relative_base || gemname # for lib/net/net-smtp.gemspec if m = /.*(?=-(.*)\z)/.match(gemname) base = File.join(base, *m.to_a.select {|n| !base.include?(n)}) end - files = Dir.glob("lib/#{base}{.rb,/**/*.rb}", base: srcdir) + files = Dir.glob("#{base}{.rb,/**/*.rb}", base: root) if !relative_base and files.empty? # no files at the toplevel # pseudo gem like ruby2_keywords - files << "lib/#{gemname}.rb" + files << "#{gemname}.rb" end case gemname when "net-http" - files << "lib/net/https.rb" + files << "net/https.rb" when "optparse" - files << "lib/optionparser.rb" + files << "optionparser.rb" end files end + + def ext_features + loaded_gemspec = load_gemspec("#{root}/#{gemspec}") + extension = loaded_gemspec.extensions.first + return [] unless extension + + extconf = File.expand_path(extension, srcdir) + ext_build_dir = File.dirname(extconf) + makefile_path = "#{ext_build_dir}/Makefile" + return [] unless File.exist?(makefile_path) + + features_from_makefile(makefile_path) + end + + def root + "#{srcdir}/lib" + end end end end @@ -820,10 +688,7 @@ module RbInstall end end - class GemInstaller < Gem::Installer - end - - class UnpackedInstaller < GemInstaller + class UnpackedInstaller < Gem::Installer def write_cache_file end @@ -847,11 +712,6 @@ module RbInstall super end - def generate_bin_script(filename, bindir) - return if same_bin_script?(filename, bindir) - super - end - def same_bin_script?(filename, bindir) path = File.join(bindir, formatted_program_filename(filename)) begin @@ -870,11 +730,10 @@ module RbInstall super unless $dryrun $installed_list.puts(without_destdir(default_spec_file)) if $installed_list end - end - class GemInstaller def install spec.post_install_message = nil + dir_creating(without_destdir(gem_dir)) RbInstall.no_write(options) {super} end @@ -883,6 +742,7 @@ module RbInstall end def generate_bin_script(filename, bindir) + return if same_bin_script?(filename, bindir) name = formatted_program_filename(filename) unless $dryrun super @@ -904,29 +764,23 @@ module RbInstall end end -# :startdoc: - -install?(:ext, :comm, :gem, :'default-gems', :'default-gems-comm') do - install_default_gem('lib', srcdir, bindir) -end -install?(:ext, :arch, :gem, :'default-gems', :'default-gems-arch') do - install_default_gem('ext', srcdir, bindir) -end - def load_gemspec(file, base = nil) file = File.realpath(file) code = File.read(file, encoding: "utf-8:-") + + files = [] + Dir.glob("**/*", File::FNM_DOTMATCH, base: base) do |n| + case File.basename(n); when ".", ".."; next; end + next if File.directory?(File.join(base, n)) + files << n.dump + end if base code.gsub!(/(?:`git[^\`]*`|%x\[git[^\]]*\])\.split\([^\)]*\)/m) do - files = [] - if base - Dir.glob("**/*", File::FNM_DOTMATCH, base: base) do |n| - case File.basename(n); when ".", ".."; next; end - next if File.directory?(File.join(base, n)) - files << n.dump - end - end "[" + files.join(", ") + "]" end + code.gsub!(/IO\.popen\(.*git.*?\)/) do + "[" + files.join(", ") + "] || itself" + end + spec = eval(code, binding, file) unless Gem::Specification === spec raise TypeError, "[#{file}] isn't a Gem::Specification (#{spec.class} instead)." @@ -964,7 +818,7 @@ def install_default_gem(dir, srcdir, bindir) spec = load_gemspec("#{base}/#{src}") file_collector = RbInstall::Specs::FileCollector.for(srcdir, dir, src) files = file_collector.collect - if file_collector.skip_install?(files) + if files.empty? next end spec.files = files @@ -987,6 +841,261 @@ def install_default_gem(dir, srcdir, bindir) end end +# :startdoc: + +install?(:local, :arch, :bin, :'bin-arch') do + prepare "binary commands", (dest = archbindir || bindir) + + def (bins = []).add(name) + push(name) + name + end + + install bins.add(ruby_install_name+exeext), dest, :mode => $prog_mode, :strip => $strip + if rubyw_install_name and !rubyw_install_name.empty? + install bins.add(rubyw_install_name+exeext), dest, :mode => $prog_mode, :strip => $strip + end + # emcc produces ruby and ruby.wasm, the first is a JavaScript file of runtime support + # to load and execute the second .wasm file. Both are required to execute ruby + if RUBY_PLATFORM =~ /emscripten/ and File.exist? ruby_install_name+".wasm" + install bins.add(ruby_install_name+".wasm"), dest, :mode => $prog_mode, :strip => $strip + end + if File.exist? goruby_install_name+exeext + install bins.add(goruby_install_name+exeext), dest, :mode => $prog_mode, :strip => $strip + end + if enable_shared and dll != lib + install bins.add(dll), dest, :mode => $prog_mode, :strip => $strip + end + if archbindir + prepare "binary command links", bindir + relpath = Path.relative(archbindir, bindir) + bins.each do |f| + ln_sf(File.join(relpath, f), File.join(bindir, f)) + end + end +end + +install?(:local, :arch, :lib, :'lib-arch') do + prepare "base libraries", libdir + + install lib, libdir, :mode => $prog_mode, :strip => $strip unless lib == arc + install arc, libdir, :mode => $data_mode unless CONFIG["INSTALL_STATIC_LIBRARY"] == "no" + if dll == lib and dll != arc + for link in CONFIG["LIBRUBY_ALIASES"].split - [File.basename(dll)] + ln_sf(dll, File.join(libdir, link)) + end + end + + prepare "arch files", archlibdir + install "rbconfig.rb", archlibdir, :mode => $data_mode + if CONFIG["ARCHFILE"] + for file in CONFIG["ARCHFILE"].split + install file, archlibdir, :mode => $data_mode + end + end +end + +install?(:local, :arch, :data) do + pc = CONFIG["ruby_pc"] + if pc and File.file?(pc) and File.size?(pc) + prepare "pkgconfig data", pkgconfigdir = File.join(libdir, "pkgconfig") + install pc, pkgconfigdir, :mode => $data_mode + if (pkgconfig_base = CONFIG["libdir", true]) != libdir + prepare "pkgconfig data link", File.join(pkgconfig_base, "pkgconfig") + ln_sf(File.join("..", Path.relative(pkgconfigdir, pkgconfig_base), pc), + File.join(pkgconfig_base, "pkgconfig", pc)) + end + end +end + +install?(:ext, :arch, :'ext-arch') do + prepare "extension objects", archlibdir + noinst = %w[-* -*/] | (CONFIG["no_install_files"] || "").split + install_recursive("#{$extout}/#{CONFIG['arch']}", archlibdir, :no_install => noinst, :mode => $prog_mode, :strip => $strip) + prepare "extension objects", sitearchlibdir + prepare "extension objects", vendorarchlibdir + if extso = File.read("exts.mk")[/^EXTSO[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] and + !(extso = extso.gsub(/\\\n/, '').split).empty? + libpathenv = CONFIG["LIBPATHENV"] + dest = CONFIG[!libpathenv || libpathenv == "PATH" ? "bindir" : "libdir"] + prepare "external libraries", dest + for file in extso + install file, dest, :mode => $prog_mode + end + end +end + +install?(:ext, :arch, :hdr, :'arch-hdr', :'hdr-arch') do + prepare "extension headers", archhdrdir + install_recursive("#{$extout}/include/#{CONFIG['arch']}", archhdrdir, :glob => "*.h", :mode => $data_mode) + install_recursive("#{$extout}/include/#{CONFIG['arch']}", archhdrdir, :glob => "rb_rjit_header-*.obj", :mode => $data_mode) + install_recursive("#{$extout}/include/#{CONFIG['arch']}", archhdrdir, :glob => "rb_rjit_header-*.pch", :mode => $data_mode) +end + +install?(:ext, :comm, :'ext-comm') do + prepare "extension scripts", rubylibdir + install_recursive("#{$extout}/common", rubylibdir, :mode => $data_mode) + prepare "extension scripts", sitelibdir + prepare "extension scripts", vendorlibdir +end + +install?(:ext, :comm, :hdr, :'comm-hdr', :'hdr-comm') do + hdrdir = rubyhdrdir + "/ruby" + prepare "extension headers", hdrdir + install_recursive("#{$extout}/include/ruby", hdrdir, :glob => "*.h", :mode => $data_mode) +end + +install?(:doc, :rdoc) do + if $rdocdir + ridatadir = File.join(CONFIG['ridir'], CONFIG['ruby_version'], "system") + prepare "rdoc", ridatadir + install_recursive($rdocdir, ridatadir, :no_install => rdoc_noinst, :mode => $data_mode) + end +end + +install?(:doc, :html) do + if $htmldir + prepare "html-docs", docdir + install_recursive($htmldir, docdir+"/html", :no_install => rdoc_noinst, :mode => $data_mode) + end +end + +install?(:doc, :capi) do + prepare "capi-docs", docdir + install_recursive "doc/capi", docdir+"/capi", :mode => $data_mode +end + +install?(:local, :comm, :bin, :'bin-comm') do + prepare "command scripts", bindir + + install_recursive(File.join(srcdir, "bin"), bindir, :maxdepth => 1) do |src, cmd| + $script_installer.install(src, cmd) + end +end + +install?(:local, :comm, :lib) do + prepare "library scripts", rubylibdir + noinst = %w[*.txt *.rdoc *.gemspec] + install_recursive(File.join(srcdir, "lib"), rubylibdir, :no_install => noinst, :mode => $data_mode) +end + +install?(:local, :comm, :hdr, :'comm-hdr') do + prepare "common headers", rubyhdrdir + + noinst = [] + unless RUBY_PLATFORM =~ /mswin|mingw|bccwin/ + noinst << "win32.h" + end + noinst = nil if noinst.empty? + install_recursive(File.join(srcdir, "include"), rubyhdrdir, :no_install => noinst, :glob => "*.{h,hpp}", :mode => $data_mode) +end + +install?(:local, :comm, :man) do + mdocs = Dir["#{srcdir}/man/*.[1-9]"] + prepare "manpages", mandir, ([] | mdocs.collect {|mdoc| mdoc[/\d+$/]}).sort.collect {|sec| "man#{sec}"} + + case $mantype + when /\.(?:(gz)|bz2)\z/ + compress = $1 ? "gzip" : "bzip2" + suffix = $& + end + mandir = File.join(mandir, "man") + has_goruby = File.exist?(goruby_install_name+exeext) + require File.join(srcdir, "tool/mdoc2man.rb") if /\Adoc\b/ !~ $mantype + mdocs.each do |mdoc| + next unless File.file?(mdoc) and File.read(mdoc, 1) == '.' + base = File.basename(mdoc) + if base == "goruby.1" + next unless has_goruby + end + + destdir = mandir + (section = mdoc[/\d+$/]) + destname = ruby_install_name.sub(/ruby/, base.chomp(".#{section}")) + destfile = File.join(destdir, "#{destname}.#{section}") + + if /\Adoc\b/ =~ $mantype + if compress + begin + w = IO.popen(compress, "rb", in: mdoc, &:read) + rescue + else + destfile << suffix + end + end + if w + open_for_install(destfile, $data_mode) {w} + else + install mdoc, destfile, :mode => $data_mode + end + else + class << (w = []) + alias print push + end + if File.basename(mdoc).start_with?('bundle') || + File.basename(mdoc).start_with?('gemfile') + w = File.read(mdoc) + else + File.open(mdoc) {|r| Mdoc2Man.mdoc2man(r, w)} + w = w.join("") + end + if compress + begin + w = IO.popen(compress, "r+b") do |f| + Thread.start {f.write w; f.close_write} + f.read + end + rescue + else + destfile << suffix + end + end + open_for_install(destfile, $data_mode) {w} + end + end +end + +install?(:dbg, :nodefault) do + prepare "debugger commands", bindir + prepare "debugger scripts", rubylibdir + conf = MAKEFILE_CONFIG.merge({"prefix"=>"${prefix#/}"}) + Dir.glob(File.join(srcdir, "template/ruby-*db.in")) do |src| + cmd = $script_installer.transform(File.basename(src, ".in")) + open_for_install(File.join(bindir, cmd), $script_mode) { + RbConfig.expand(File.read(src), conf) + } + end + Dir.glob(File.join(srcdir, "misc/lldb_*")) do |src| + if File.directory?(src) + install_recursive src, File.join(rubylibdir, File.basename(src)) + else + install src, rubylibdir + end + end + install File.join(srcdir, ".gdbinit"), File.join(rubylibdir, "gdbinit") + if $debug_symbols + { + ruby_install_name => archbindir || bindir, + rubyw_install_name => archbindir || bindir, + goruby_install_name => archbindir || bindir, + dll => libdir, + }.each do |src, dest| + next if src.empty? + src += $debug_symbols + if File.directory?(src) + install_recursive src, File.join(dest, src) + end + end + end +end + +install?(:ext, :comm, :gem, :'default-gems', :'default-gems-comm') do + install_default_gem('lib', srcdir, bindir) +end + +install?(:ext, :arch, :gem, :'default-gems', :'default-gems-arch') do + install_default_gem('ext', srcdir, bindir) +end + install?(:ext, :comm, :gem, :'bundled-gems') do gem_dir = Gem.default_dir install_dir = with_destdir(gem_dir) @@ -996,6 +1105,7 @@ install?(:ext, :comm, :gem, :'bundled-gems') do end installed_gems = {} + skipped = {} options = { :install_dir => install_dir, :bin_dir => with_destdir(bindir), @@ -1022,15 +1132,38 @@ install?(:ext, :comm, :gem, :'bundled-gems') do File.foreach("#{srcdir}/gems/bundled_gems") do |name| next if /^\s*(?:#|$)/ =~ name next unless /^(\S+)\s+(\S+).*/ =~ name + gem = $1 gem_name = "#$1-#$2" - path = "#{srcdir}/.bundle/specifications/#{gem_name}.gemspec" + # Try to find the original gemspec file + path = "#{srcdir}/.bundle/gems/#{gem_name}/#{gem}.gemspec" unless File.exist?(path) + # Try to find the gemspec file for C ext gems + # ex .bundle/gems/debug-1.7.1/debug-1.7.1.gemspec + # This gemspec keep the original dependencies path = "#{srcdir}/.bundle/gems/#{gem_name}/#{gem_name}.gemspec" - next unless File.exist?(path) + unless File.exist?(path) + # Try to find the gemspec file for gems that hasn't own gemspec + path = "#{srcdir}/.bundle/specifications/#{gem_name}.gemspec" + unless File.exist?(path) + skipped[gem_name] = "gemspec not found" + next + end + end end spec = load_gemspec(path, "#{srcdir}/.bundle/gems/#{gem_name}") - next unless spec.platform == Gem::Platform::RUBY - next unless spec.full_name == gem_name + unless spec.platform == Gem::Platform::RUBY + skipped[gem_name] = "not ruby platform (#{spec.platform})" + next + end + unless spec.full_name == gem_name + skipped[gem_name] = "full name unmatch #{spec.full_name}" + next + end + # Skip install C ext bundled gem if it is build failed or not found + if !spec.extensions.empty? && !File.exist?("#{build_dir}/#{gem_name}/gem.build_complete") + skipped[gem_name] = "extensions not found or build failed #{spec.full_name}" + next + end spec.extension_dir = "#{extensions_dir}/#{spec.full_name}" package = RbInstall::DirPackage.new spec ins = RbInstall::UnpackedInstaller.new(package, options) @@ -1048,7 +1181,11 @@ install?(:ext, :comm, :gem, :'bundled-gems') do install installed_gems, gem_dir+"/cache" end unless gems.empty? - puts "skipped bundled gems: #{gems.join(' ')}" + skipped.default = "not found in bundled_gems" + puts "skipped bundled gems:" + gems.each do |gem| + printf " %-32s%s\n", File.basename(gem), skipped[gem] + end end end @@ -1068,6 +1205,7 @@ installs = $install.map do |inst| end installs.flatten! installs -= $exclude.map {|exc| $install_procs[exc]}.flatten +puts "Installing to #$destdir" unless installs.empty? installs.each do |block| dir = Dir.pwd begin diff --git a/tool/rbs_skip_tests b/tool/rbs_skip_tests new file mode 100644 index 0000000000..56a864d193 --- /dev/null +++ b/tool/rbs_skip_tests @@ -0,0 +1,61 @@ +# Running tests of RBS gem may fail because of various reasons. +# You can skip tests of RBS gem using this file, instead of pushing a new commit to `ruby/rbs` repository. +# +# The most frequently seen reason is the incompatibilities introduced to the unreleased version, including +# +# * Strict argument type check is introduced +# * A required method parameter is added +# * A method/class is removed +# +# Feel free to skip the tests with this file for that case. +# +# Syntax: +# +# $(test-case-name) ` ` $(optional comment) # Skipping single test case +# $(test-class-name) ` ` $(optional comment) # Skipping a test class +# + +test_replicate(EncodingTest) the method was removed in 3.3 + +test_collection_install(RBS::CliTest) running tests without Bundler +test_collection_install_frozen(RBS::CliTest) running tests without Bundler +test_collection_install_gemspec(RBS::CliTest) running tests without Bundler +test_collection_update(RBS::CliTest) running tests without Bundler + +test_loading_from_rbs_collection__gem_version_mismatch(RBS::EnvironmentLoaderTest) running test without rbs-amber testing gem + +test_defs(RBS::RbPrototypeTest) Numeric Nodes are added +test_defs_return_type(RBS::RbPrototypeTest) Numeric Nodes are added +test_defs_return_type_with_block(RBS::RbPrototypeTest) Numeric Nodes are added +test_defs_return_type_with_if(RBS::RbPrototypeTest) Numeric Nodes are added +test_endless_method_definition(RBS::RbPrototypeTest) Numeric Nodes are added +test_literal_to_type(RBS::RbPrototypeTest) Numeric Nodes are added +test_literal_types(RBS::RbPrototypeTest) Numeric Nodes are added +test_accessibility(RBS::RbPrototypeTest) Symbol Node is added +test_aliases(RBS::RbPrototypeTest) Symbol Node is added +test_comments(RBS::RbPrototypeTest) Symbol Node is added +test_const(RBS::RbPrototypeTest) Symbol Node is added +test_meta_programming(RBS::RbPrototypeTest) Symbol Node is added +test_module_function(RBS::RbPrototypeTest) Symbol Node is added +test_all(RBS::RbiPrototypeTest) Symbol Node is added +test_block_args(RBS::RbiPrototypeTest) Symbol Node is added +test_implicit_block(RBS::RbiPrototypeTest) Symbol Node is added +test_non_parameter_type_member(RBS::RbiPrototypeTest) Symbol Node is added +test_noreturn(RBS::RbiPrototypeTest) Symbol Node is added +test_optional_block(RBS::RbiPrototypeTest) Symbol Node is added +test_overloading(RBS::RbiPrototypeTest) Symbol Node is added +test_parameter(RBS::RbiPrototypeTest) Symbol Node is added +test_parameter_type_member_variance(RBS::RbiPrototypeTest) Symbol Node is added +test_tuple(RBS::RbiPrototypeTest) Symbol Node is added +test_untyped_block(RBS::RbiPrototypeTest) Symbol Node is added +test_argument_forwarding(RBS::RbPrototypeTest) `...` args handling is changed + +test_TOPDIR(RbConfigSingletonTest) `TOPDIR` is `nil` during CI while RBS type is declared as `String` + +test_aref(FiberSingletonTest) the method should not accept String keys + +NetSingletonTest depending on external resources +NetInstanceTest depending on external resources +TestHTTPRequest depending on external resources +TestSingletonNetHTTPResponse depending on external resources +TestInstanceNetHTTPResponse depending on external resources diff --git a/tool/rbuninstall.rb b/tool/rbuninstall.rb index f0c286012c..60f5241a4f 100755 --- a/tool/rbuninstall.rb +++ b/tool/rbuninstall.rb @@ -21,15 +21,33 @@ BEGIN { end $dirs = [] $files = [] + COLUMNS = $tty && (ENV["COLUMNS"]&.to_i || begin require 'io/console/size'; rescue; else IO.console_size&.at(1); end)&.then do |n| + n-1 if n > 1 + end + if COLUMNS + $column = 0 + def message(str = nil) + $stdout.print "\b \b" * $column + if str + if str.size > COLUMNS + str = "..." + str[(-COLUMNS+3)..-1] + end + $stdout.print str + end + $stdout.flush + $column = str&.size || 0 + end + else + alias message puts + end } list = ($_.chomp!('/') ? $dirs : $files) list << $_ END { status = true $\ = nil - ors = (!$dryrun and $tty) ? "\e[K\r" : "\n" $files.each do |file| - print "rm #{file}#{ors}" + message "rm #{file}" unless $dryrun file = File.join($destdir, file) if $destdir begin @@ -45,9 +63,10 @@ END { $dirs.each do |dir| unlink[dir] = true end + nonempty = {} while dir = $dirs.pop dir = File.dirname(dir) while File.basename(dir) == '.' - print "rmdir #{dir}#{ors}" + message "rmdir #{dir}" unless $dryrun realdir = $destdir ? File.join($destdir, dir) : dir begin @@ -58,16 +77,23 @@ END { raise unless File.symlink?(realdir) File.unlink(realdir) end - rescue Errno::ENOENT, Errno::ENOTEMPTY + rescue Errno::ENOTEMPTY + nonempty[dir] = true + rescue Errno::ENOENT rescue status = false puts $! else + nonempty.delete(dir) parent = File.dirname(dir) $dirs.push(parent) unless parent == dir or unlink[parent] end end end - print ors.chomp + message + unless nonempty.empty? + puts "Non empty director#{nonempty.size == 1 ? 'y' : 'ies'}:" + nonempty.each_key {|dir| print " #{dir}\n"} + end exit(status) } diff --git a/tool/rdoc-srcdir b/tool/rdoc-srcdir new file mode 100644 index 0000000000..10c63caf9e --- /dev/null +++ b/tool/rdoc-srcdir @@ -0,0 +1,20 @@ +#!ruby + +require 'rdoc/rdoc' + +# Make only the output directory relative to the invoked directory. +invoked = Dir.pwd + +# Load options and parse files from srcdir. +Dir.chdir(File.dirname(__dir__)) + +options = RDoc::Options.load_options +options.parse ARGV + +options.singleton_class.define_method(:finish) do + super() + @op_dir = File.expand_path(@op_dir, invoked) +end + +# Do not hide errors when generating documents of Ruby itself. +RDoc::RDoc.new.document options diff --git a/tool/redmine-backporter.rb b/tool/redmine-backporter.rb index df54c7eb26..44b44920d4 100755 --- a/tool/redmine-backporter.rb +++ b/tool/redmine-backporter.rb @@ -45,7 +45,7 @@ REDMINE_BASE = 'https://bugs.ruby-lang.org' @query = { 'f[]' => BACKPORT_CF_KEY, "op[#{BACKPORT_CF_KEY}]" => '~', - "v[#{BACKPORT_CF_KEY}][]" => "#{TARGET_VERSION}: REQUIRED", + "v[#{BACKPORT_CF_KEY}][]" => "\"#{TARGET_VERSION}: REQUIRED\"", 'limit' => 40, 'status_id' => STATUS_CLOSE, 'sort' => 'updated_on' @@ -422,7 +422,7 @@ eom }, "done" => proc{|args| - raise CommandSyntaxError unless /\A(\d+)?(?: by (\h+))?(?:\s*-- +(.*))?\z/ =~ args + raise CommandSyntaxError unless /\A(\d+)?(?: *by (\h+))?(?:\s*-- +(.*))?\z/ =~ args notes = $3 notes.strip! if notes rev = $2 diff --git a/tool/rjit/bindgen.rb b/tool/rjit/bindgen.rb new file mode 100755 index 0000000000..fb6653ed9c --- /dev/null +++ b/tool/rjit/bindgen.rb @@ -0,0 +1,664 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +ENV['GEM_HOME'] = File.expand_path('./.bundle', __dir__) +require 'rubygems/source' +require 'bundler/inline' +gemfile(true) do + source 'https://rubygems.org' + gem 'ffi-clang', '0.7.0', require: false +end + +# Help ffi-clang find libclang +# Hint: apt install libclang1 +ENV['LIBCLANG'] ||= Dir.glob("/usr/lib/llvm-*/lib/libclang.so.1").grep_v(/-cpp/).sort.last +require 'ffi/clang' + +require 'etc' +require 'fiddle/import' +require 'set' + +unless build_dir = ARGV.first + abort "Usage: #{$0} BUILD_DIR" +end + +class Node < Struct.new( + :kind, + :spelling, + :type, + :typedef_type, + :bitwidth, + :sizeof_type, + :offsetof, + :enum_value, + :children, + keyword_init: true, +) +end + +# Parse a C header with ffi-clang and return Node objects. +# To ease the maintenance, ffi-clang should be used only inside this class. +class HeaderParser + def initialize(header, cflags:) + @translation_unit = FFI::Clang::Index.new.parse_translation_unit(header, cflags, [], {}) + end + + def parse + parse_children(@translation_unit.cursor) + end + + private + + def parse_children(cursor) + children = [] + cursor.visit_children do |cursor, _parent| + children << parse_cursor(cursor) + next :continue + end + children + end + + def parse_cursor(cursor) + unless cursor.kind.start_with?('cursor_') + raise "unexpected cursor kind: #{cursor.kind}" + end + kind = cursor.kind.to_s.delete_prefix('cursor_').to_sym + children = parse_children(cursor) + + offsetof = {} + if kind == :struct + children.select { |c| c.kind == :field_decl }.each do |child| + offsetof[child.spelling] = cursor.type.offsetof(child.spelling) + end + end + + sizeof_type = nil + if %i[struct union].include?(kind) + sizeof_type = cursor.type.sizeof + end + + enum_value = nil + if kind == :enum_constant_decl + enum_value = cursor.enum_value + end + + Node.new( + kind: kind, + spelling: cursor.spelling, + type: cursor.type.spelling, + typedef_type: cursor.typedef_type.spelling, + bitwidth: cursor.bitwidth, + sizeof_type: sizeof_type, + offsetof: offsetof, + enum_value: enum_value, + children: children, + ) + end +end + +# Convert Node objects to a Ruby binding source. +class BindingGenerator + BINDGEN_BEG = '### RJIT bindgen begin ###' + BINDGEN_END = '### RJIT bindgen end ###' + DEFAULTS = { '_Bool' => 'CType::Bool.new' } + DEFAULTS.default_proc = proc { |_h, k| "CType::Stub.new(:#{k})" } + + attr_reader :src + + # @param src_path [String] + # @param consts [Hash{ Symbol => Array<String> }] + # @param values [Hash{ Symbol => Array<String> }] + # @param funcs [Array<String>] + # @param types [Array<String>] + # @param dynamic_types [Array<String>] #ifdef-dependent immediate types, which need Primitive.cexpr! for type detection + # @param skip_fields [Hash{ Symbol => Array<String> }] Struct fields that are skipped from bindgen + # @param ruby_fields [Hash{ Symbol => Array<String> }] Struct VALUE fields that are considered Ruby objects + def initialize(src_path:, consts:, values:, funcs:, types:, dynamic_types:, skip_fields:, ruby_fields:) + @preamble, @postamble = split_ambles(src_path) + @src = String.new + @consts = consts.transform_values(&:sort) + @values = values.transform_values(&:sort) + @funcs = funcs.sort + @types = types.sort + @dynamic_types = dynamic_types.sort + @skip_fields = skip_fields.transform_keys(&:to_s) + @ruby_fields = ruby_fields.transform_keys(&:to_s) + @references = Set.new + end + + def generate(nodes) + println @preamble + + # Define macros/enums + @consts.each do |type, values| + values.each do |value| + raise "#{value} isn't a valid constant name" unless ('A'..'Z').include?(value[0]) + println " C::#{value} = Primitive.cexpr! %q{ #{type}2NUM(#{value}) }" + end + end + println + + # Define variables + @values.each do |type, values| + values.each do |value| + println " def C.#{value} = Primitive.cexpr!(%q{ #{type}2NUM(#{value}) })" + end + end + println + + # Define function pointers + @funcs.each do |func| + println " def C.#{func}" + println " Primitive.cexpr! %q{ SIZET2NUM((size_t)#{func}) }" + println " end" + println + end + + # Build a hash table for type lookup by name + nodes_index = flatten_nodes(nodes).group_by(&:spelling).transform_values do |values| + # Try to search a declaration with definitions + node_with_children = values.find { |v| !v.children.empty? } + next node_with_children if node_with_children + + # Otherwise, assume the last one is the main declaration + values.last + end + + # Define types + @types.each do |type| + unless definition = generate_node(nodes_index[type]) + raise "Failed to find or generate type: #{type}" + end + println " def C.#{type}" + println "@#{type} ||= #{definition}".gsub(/^/, " ").chomp + println " end" + println + end + + # Define dynamic types + @dynamic_types.each do |type| + unless generate_node(nodes_index[type])&.start_with?('CType::Immediate') + raise "Non-immediate type is given to dynamic_types: #{type}" + end + # Only one Primitive.cexpr! is allowed for each line: https://github.com/ruby/ruby/pull/9612 + println " def C.#{type}" + println " @#{type} ||= CType::Immediate.find(" + println " Primitive.cexpr!(\"SIZEOF(#{type})\")," + println " Primitive.cexpr!(\"SIGNED_TYPE_P(#{type})\")," + println " )" + println " end" + println + end + + # Leave a stub for types that are referenced but not targeted + (@references - @types - @dynamic_types).each do |type| + println " def C.#{type}" + println " #{DEFAULTS[type]}" + println " end" + println + end + + print @postamble + end + + private + + # Make an array that includes all top-level and nested nodes + def flatten_nodes(nodes) + result = [] + nodes.each do |node| + unless node.children.empty? + result.concat(flatten_nodes(node.children)) + end + end + result.concat(nodes) # prioritize top-level nodes + result + end + + # Return code before BINDGEN_BEG and code after BINDGEN_END + def split_ambles(src_path) + lines = File.read(src_path).lines + + preamble_end = lines.index { |l| l.include?(BINDGEN_BEG) } + raise "`#{BINDGEN_BEG}` was not found in '#{src_path}'" if preamble_end.nil? + + postamble_beg = lines.index { |l| l.include?(BINDGEN_END) } + raise "`#{BINDGEN_END}` was not found in '#{src_path}'" if postamble_beg.nil? + raise "`#{BINDGEN_BEG}` was found after `#{BINDGEN_END}`" if preamble_end >= postamble_beg + + return lines[0..preamble_end].join, lines[postamble_beg..-1].join + end + + # Generate code from a node. Used for constructing a complex nested node. + # @param node [Node] + def generate_node(node, sizeof_type: nil) + case node&.kind + when :struct, :union + # node.spelling is often empty for union, but we'd like to give it a name when it has one. + buf = +"CType::#{node.kind.to_s.sub(/\A[a-z]/, &:upcase)}.new(\n" + buf << " \"#{node.spelling}\", Primitive.cexpr!(\"SIZEOF(#{sizeof_type || node.type})\"),\n" + bit_fields_end = node.children.index { |c| c.bitwidth == -1 } || node.children.size # first non-bit field index + node.children.each_with_index do |child, i| + skip_type = sizeof_type&.gsub(/\(\(struct ([^\)]+) \*\)NULL\)->/, '\1.') || node.spelling + next if @skip_fields.fetch(skip_type, []).include?(child.spelling) + field_builder = proc do |field, type| + if node.kind == :struct + to_ruby = @ruby_fields.fetch(node.spelling, []).include?(field) + if child.bitwidth > 0 + if bit_fields_end <= i # give up offsetof calculation for non-leading bit fields + raise "non-leading bit fields are not supported. consider including '#{field}' in skip_fields." + end + offsetof = node.offsetof.fetch(field) + else + off_type = sizeof_type || "(*((#{node.type} *)NULL))" + offsetof = "Primitive.cexpr!(\"OFFSETOF(#{off_type}, #{field})\")" + end + " #{field}: [#{type}, #{offsetof}#{', true' if to_ruby}],\n" + else + " #{field}: #{type},\n" + end + end + + case child + # BitField is struct-specific. So it must be handled here. + in Node[kind: :field_decl, spelling:, bitwidth:, children: [_grandchild, *]] if bitwidth > 0 + buf << field_builder.call(spelling, "CType::BitField.new(#{bitwidth}, #{node.offsetof.fetch(spelling) % 8})") + # "(unnamed ...)" struct and union are handled here, which are also struct-specific. + in Node[kind: :field_decl, spelling:, type:, children: [grandchild]] if type.match?(/\((unnamed|anonymous) [^)]+\)\z/) + if sizeof_type + child_type = "#{sizeof_type}.#{child.spelling}" + else + child_type = "((#{node.type} *)NULL)->#{child.spelling}" + end + buf << field_builder.call(spelling, generate_node(grandchild, sizeof_type: child_type).gsub(/^/, ' ').sub(/\A +/, '')) + # In most cases, we'd like to let generate_type handle the type unless it's "(unnamed ...)". + in Node[kind: :field_decl, spelling:, type:] if !type.empty? + buf << field_builder.call(spelling, generate_type(type)) + else # forward declarations are ignored + end + end + buf << ")" + when :typedef_decl + case node.children + in [child] + generate_node(child) + in [child, Node[kind: :integer_literal]] + generate_node(child) + in _ unless node.typedef_type.empty? + generate_type(node.typedef_type) + end + when :enum_decl + generate_type('int') + when :type_ref + generate_type(node.spelling) + end + end + + # Generate code from a type name. Used for resolving the name of a simple leaf node. + # @param type [String] + def generate_type(type) + if type.match?(/\[\d+\]\z/) + return "CType::Array.new { #{generate_type(type.sub!(/\[\d+\]\z/, ''))} }" + end + type = type.delete_suffix('const') + if type.end_with?('*') + if type == 'const void *' + # `CType::Pointer.new { CType::Immediate.parse("void") }` is never useful, + # so specially handle that case here. + return 'CType::Immediate.parse("void *")' + end + return "CType::Pointer.new { #{generate_type(type.delete_suffix('*').rstrip)} }" + end + + type = type.gsub(/((const|volatile) )+/, '').rstrip + if type.start_with?(/(struct|union|enum) /) + target = type.split(' ', 2).last + push_target(target) + "self.#{target}" + else + begin + ctype = Fiddle::Importer.parse_ctype(type) + rescue Fiddle::DLError + push_target(type) + "self.#{type}" + else + # Convert any function pointers to void* to workaround FILE* vs int* + if ctype == Fiddle::TYPE_VOIDP + "CType::Immediate.parse(\"void *\")" + else + "CType::Immediate.parse(#{type.dump})" + end + end + end + end + + def print(str) + @src << str + end + + def println(str = "") + @src << str << "\n" + end + + def chomp + @src.delete_suffix!("\n") + end + + def rstrip! + @src.rstrip! + end + + def push_target(target) + unless target.match?(/\A\w+\z/) + raise "invalid target: #{target}" + end + @references << target + end +end + +src_dir = File.expand_path('../..', __dir__) +src_path = File.join(src_dir, 'rjit_c.rb') +build_dir = File.expand_path(build_dir) +cflags = [ + src_dir, + build_dir, + File.join(src_dir, 'include'), + File.join(build_dir, ".ext/include/#{RUBY_PLATFORM}"), +].map { |dir| "-I#{dir}" } + +# Clear .cache/clangd created by the language server, which could break this bindgen +clangd_cache = File.join(src_dir, '.cache/clangd') +if Dir.exist?(clangd_cache) + system('rm', '-rf', clangd_cache, exception: true) +end + +# Parse rjit_c.h and generate rjit_c.rb +nodes = HeaderParser.new(File.join(src_dir, 'rjit_c.h'), cflags: cflags).parse +generator = BindingGenerator.new( + src_path: src_path, + consts: { + LONG: %w[ + UNLIMITED_ARGUMENTS + VM_ENV_DATA_INDEX_ME_CREF + VM_ENV_DATA_INDEX_SPECVAL + ], + SIZET: %w[ + ARRAY_REDEFINED_OP_FLAG + BOP_AND + BOP_AREF + BOP_EQ + BOP_EQQ + BOP_FREEZE + BOP_GE + BOP_GT + BOP_LE + BOP_LT + BOP_MINUS + BOP_MOD + BOP_OR + BOP_PLUS + BUILTIN_ATTR_LEAF + HASH_REDEFINED_OP_FLAG + INTEGER_REDEFINED_OP_FLAG + INVALID_SHAPE_ID + METHOD_VISI_PRIVATE + METHOD_VISI_PROTECTED + METHOD_VISI_PUBLIC + METHOD_VISI_UNDEF + OBJ_TOO_COMPLEX_SHAPE_ID + OPTIMIZED_METHOD_TYPE_BLOCK_CALL + OPTIMIZED_METHOD_TYPE_CALL + OPTIMIZED_METHOD_TYPE_SEND + OPTIMIZED_METHOD_TYPE_STRUCT_AREF + OPTIMIZED_METHOD_TYPE_STRUCT_ASET + RARRAY_EMBED_FLAG + RARRAY_EMBED_LEN_MASK + RARRAY_EMBED_LEN_SHIFT + RMODULE_IS_REFINEMENT + ROBJECT_EMBED + RSTRUCT_EMBED_LEN_MASK + RUBY_EVENT_CLASS + RUBY_EVENT_C_CALL + RUBY_EVENT_C_RETURN + RUBY_FIXNUM_FLAG + RUBY_FLONUM_FLAG + RUBY_FLONUM_MASK + RUBY_IMMEDIATE_MASK + RUBY_SPECIAL_SHIFT + RUBY_SYMBOL_FLAG + RUBY_T_ARRAY + RUBY_T_CLASS + RUBY_T_ICLASS + RUBY_T_HASH + RUBY_T_MASK + RUBY_T_MODULE + RUBY_T_STRING + RUBY_T_SYMBOL + RUBY_T_OBJECT + SHAPE_FLAG_SHIFT + SHAPE_FROZEN + SHAPE_ID_NUM_BITS + SHAPE_IVAR + SHAPE_MASK + SHAPE_ROOT + STRING_REDEFINED_OP_FLAG + T_OBJECT + VM_BLOCK_HANDLER_NONE + VM_CALL_ARGS_BLOCKARG + VM_CALL_ARGS_SPLAT + VM_CALL_FCALL + VM_CALL_KWARG + VM_CALL_KW_SPLAT + VM_CALL_KW_SPLAT_MUT + VM_CALL_KW_SPLAT_bit + VM_CALL_OPT_SEND + VM_CALL_TAILCALL + VM_CALL_TAILCALL_bit + VM_CALL_ZSUPER + VM_ENV_DATA_INDEX_FLAGS + VM_ENV_DATA_SIZE + VM_ENV_FLAG_LOCAL + VM_ENV_FLAG_WB_REQUIRED + VM_FRAME_FLAG_BMETHOD + VM_FRAME_FLAG_CFRAME + VM_FRAME_FLAG_CFRAME_KW + VM_FRAME_FLAG_LAMBDA + VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM + VM_FRAME_MAGIC_BLOCK + VM_FRAME_MAGIC_CFUNC + VM_FRAME_MAGIC_METHOD + VM_METHOD_TYPE_ALIAS + VM_METHOD_TYPE_ATTRSET + VM_METHOD_TYPE_BMETHOD + VM_METHOD_TYPE_CFUNC + VM_METHOD_TYPE_ISEQ + VM_METHOD_TYPE_IVAR + VM_METHOD_TYPE_MISSING + VM_METHOD_TYPE_NOTIMPLEMENTED + VM_METHOD_TYPE_OPTIMIZED + VM_METHOD_TYPE_REFINED + VM_METHOD_TYPE_UNDEF + VM_METHOD_TYPE_ZSUPER + VM_SPECIAL_OBJECT_VMCORE + RUBY_ENCODING_MASK + RUBY_FL_FREEZE + RHASH_PASS_AS_KEYWORDS + ], + }, + values: { + SIZET: %w[ + block_type_iseq + imemo_iseq + imemo_callinfo + rb_block_param_proxy + rb_cArray + rb_cFalseClass + rb_cFloat + rb_cInteger + rb_cNilClass + rb_cString + rb_cSymbol + rb_cTrueClass + rb_rjit_global_events + rb_mRubyVMFrozenCore + rb_vm_insns_count + idRespond_to_missing + ], + }, + funcs: %w[ + rb_ary_entry_internal + rb_ary_push + rb_ary_resurrect + rb_ary_store + rb_ec_ary_new_from_values + rb_ec_str_resurrect + rb_ensure_iv_list_size + rb_fix_aref + rb_fix_div_fix + rb_fix_mod_fix + rb_fix_mul_fix + rb_gc_writebarrier + rb_get_symbol_id + rb_hash_aref + rb_hash_aset + rb_hash_bulk_insert + rb_hash_new + rb_hash_new_with_size + rb_hash_resurrect + rb_ivar_get + rb_obj_as_string_result + rb_obj_is_kind_of + rb_str_concat_literals + rb_str_eql_internal + rb_str_getbyte + rb_vm_bh_to_procval + rb_vm_concat_array + rb_vm_defined + rb_vm_get_ev_const + rb_vm_getclassvariable + rb_vm_ic_hit_p + rb_vm_opt_newarray_min + rb_vm_opt_newarray_max + rb_vm_opt_newarray_hash + rb_vm_setinstancevariable + rb_vm_splat_array + rjit_full_cfunc_return + rjit_optimized_call + rjit_str_neq_internal + rjit_record_exit_stack + rb_ivar_defined + rb_vm_throw + rb_backref_get + rb_reg_last_match + rb_reg_match_pre + rb_reg_match_post + rb_reg_match_last + rb_reg_nth_match + rb_gvar_get + rb_range_new + rb_ary_tmp_new_from_values + rb_reg_new_ary + rb_ary_clear + rb_str_intern + rb_vm_setclassvariable + rb_str_bytesize + rjit_str_simple_append + rb_str_buf_append + rb_str_dup + rb_vm_yield_with_cfunc + rb_vm_set_ivar_id + rb_ary_dup + rjit_rb_ary_subseq_length + rb_ary_unshift_m + rjit_build_kwhash + rb_rjit_entry_stub_hit + rb_rjit_branch_stub_hit + rb_sym_to_proc + ], + types: %w[ + CALL_DATA + IC + ID + IVC + RArray + RB_BUILTIN + RBasic + RObject + RStruct + RString + attr_index_t + iseq_inline_constant_cache + iseq_inline_constant_cache_entry + iseq_inline_iv_cache_entry + iseq_inline_storage_entry + method_optimized_type + rb_block + rb_block_type + rb_builtin_function + rb_call_data + rb_callable_method_entry_struct + rb_callable_method_entry_t + rb_callcache + rb_callinfo + rb_captured_block + rb_cfunc_t + rb_control_frame_t + rb_cref_t + rb_execution_context_struct + rb_execution_context_t + rb_iseq_constant_body + rb_iseq_location_t + rb_iseq_struct + rb_iseq_t + rb_method_attr_t + rb_method_bmethod_t + rb_method_cfunc_t + rb_method_definition_struct + rb_method_entry_t + rb_method_iseq_t + rb_method_optimized_t + rb_method_type_t + rb_proc_t + rb_rjit_runtime_counters + rb_serial_t + rb_shape + rb_shape_t + rb_thread_struct + rb_jit_func_t + rb_iseq_param_keyword + rb_rjit_options + rb_callinfo_kwarg + ], + # #ifdef-dependent immediate types, which need Primitive.cexpr! for type detection + dynamic_types: %w[ + VALUE + shape_id_t + ], + skip_fields: { + 'rb_execution_context_struct.machine': %w[regs], # differs between macOS and Linux + rb_execution_context_struct: %w[method_missing_reason], # non-leading bit fields not supported + rb_iseq_constant_body: %w[jit_exception jit_exception_calls yjit_payload yjit_calls_at_interv], # conditionally defined + rb_thread_struct: %w[status has_dedicated_nt to_kill abort_on_exception report_on_exception pending_interrupt_queue_checked], + :'' => %w[is_from_method is_lambda is_isolated], # rb_proc_t + }, + ruby_fields: { + rb_iseq_constant_body: %w[ + rjit_blocks + ], + rb_iseq_location_struct: %w[ + base_label + label + pathobj + ], + rb_callable_method_entry_t: %w[ + defined_class + ], + rb_callable_method_entry_struct: %w[ + defined_class + ], + }, +) +generator.generate(nodes) + +# Write rjit_c.rb +File.write(src_path, generator.src) diff --git a/tool/ruby_vm/controllers/application_controller.rb b/tool/ruby_vm/controllers/application_controller.rb index 25c10947ed..e03e54e397 100644 --- a/tool/ruby_vm/controllers/application_controller.rb +++ b/tool/ruby_vm/controllers/application_controller.rb @@ -16,10 +16,11 @@ require_relative '../models/typemap' require_relative '../loaders/vm_opts_h' class ApplicationController - def generate i, destdir + def generate i, destdir, basedir path = Pathname.new i dst = destdir ? Pathname.new(destdir).join(i) : Pathname.new(i) - dumper = RubyVM::Dumper.new dst + base = basedir ? Pathname.new(basedir) : Pathname.pwd + dumper = RubyVM::Dumper.new dst, base.expand_path return [path, dumper] end end diff --git a/tool/ruby_vm/helpers/c_escape.rb b/tool/ruby_vm/helpers/c_escape.rb index 34fafd1e34..2a99e408da 100644 --- a/tool/ruby_vm/helpers/c_escape.rb +++ b/tool/ruby_vm/helpers/c_escape.rb @@ -17,7 +17,10 @@ module RubyVM::CEscape # generate comment, with escaps. def commentify str - return "/* #{str.b.gsub('*/', '*\\/').gsub('/*', '/\\*')} */" + unless str = str.dump[/\A"\K.*(?="\z)/] + raise Encoding::CompatibilityError, "must be ASCII-compatible (#{str.encoding})" + end + return "/* #{str.gsub('*/', '*\\/').gsub('/*', '/\\*')} */" end # Mimic gensym of CL. diff --git a/tool/ruby_vm/helpers/dumper.rb b/tool/ruby_vm/helpers/dumper.rb index 98104f4b92..8a04041da9 100644 --- a/tool/ruby_vm/helpers/dumper.rb +++ b/tool/ruby_vm/helpers/dumper.rb @@ -28,16 +28,12 @@ class RubyVM::Dumper path = Pathname.new(__FILE__) path = (path.relative_path_from(Pathname.pwd) rescue path).dirname path += '../views' - path += spec - src = path.read mode: 'rt:utf-8:utf-8' + path += Pathname.pwd.join(spec).expand_path.to_s.sub("#{@base}/", '') + src = path.expand_path.read mode: 'rt:utf-8:utf-8' rescue Errno::ENOENT raise "don't know how to generate #{path}" else - if ERB.instance_method(:initialize).parameters.assoc(:key) # Ruby 2.6+ - erb = ERB.new(src, trim_mode: '%-') - else - erb = ERB.new(src, nil, '%-') - end + erb = ERB.new(src, trim_mode: '%-') erb.filename = path.to_path return erb end @@ -85,10 +81,11 @@ class RubyVM::Dumper . join end - def initialize dst + def initialize dst, base @erb = {} @empty = new_binding @file = cstr dst.to_path + @base = base end def render partial, opts = { :locals => {} } diff --git a/tool/ruby_vm/models/attribute.rb b/tool/ruby_vm/models/attribute.rb index de35e7234a..ac4122f3ac 100644 --- a/tool/ruby_vm/models/attribute.rb +++ b/tool/ruby_vm/models/attribute.rb @@ -21,7 +21,7 @@ class RubyVM::Attribute @key = opts[:name] @expr = RubyVM::CExpr.new location: opts[:location], expr: opts[:expr] @type = opts[:type] - @ope_decls = @insn.opes.map do |operand| + @ope_decls = @insn.operands.map do |operand| decl = operand[:decl] if @key == 'comptime_sp_inc' && operand[:type] == 'CALL_DATA' decl = decl.gsub('CALL_DATA', 'CALL_INFO').gsub('cd', 'ci') diff --git a/tool/ruby_vm/models/bare_instructions.rb b/tool/ruby_vm/models/bare_instructions.rb index 6b5f1f6cf8..a810d89f3c 100755 --- a/tool/ruby_vm/models/bare_instructions.rb +++ b/tool/ruby_vm/models/bare_instructions.rb @@ -16,7 +16,7 @@ require_relative 'typemap' require_relative 'attribute' class RubyVM::BareInstructions - attr_reader :template, :name, :opes, :pops, :rets, :decls, :expr + attr_reader :template, :name, :operands, :pops, :rets, :decls, :expr def initialize opts = {} @template = opts[:template] @@ -24,7 +24,7 @@ class RubyVM::BareInstructions @loc = opts[:location] @sig = opts[:signature] @expr = RubyVM::CExpr.new opts[:expr] - @opes = typesplit @sig[:ope] + @operands = typesplit @sig[:ope] @pops = typesplit @sig[:pop].reject {|i| i == '...' } @rets = typesplit @sig[:ret].reject {|i| i == '...' } @attrs = opts[:attributes].map {|i| @@ -51,7 +51,7 @@ class RubyVM::BareInstructions def call_attribute name return sprintf 'attr_%s_%s(%s)', name, @name, \ - @opes.map {|i| i[:name] }.compact.join(', ') + @operands.map {|i| i[:name] }.compact.join(', ') end def has_attribute? k @@ -65,7 +65,7 @@ class RubyVM::BareInstructions end def width - return 1 + opes.size + return 1 + operands.size end def declarations @@ -98,7 +98,7 @@ class RubyVM::BareInstructions end def operands_info - opes.map {|o| + operands.map {|o| c, _ = RubyVM::Typemap.fetch o[:type] next c }.join @@ -137,7 +137,7 @@ class RubyVM::BareInstructions end def has_ope? var - return @opes.any? {|i| i[:name] == var[:name] } + return @operands.any? {|i| i[:name] == var[:name] } end def has_pop? var @@ -180,7 +180,7 @@ class RubyVM::BareInstructions # Beware: order matters here because some attribute depends another. generate_attribute 'const char*', 'name', "insn_name(#{bin})" generate_attribute 'enum ruby_vminsn_type', 'bin', bin - generate_attribute 'rb_num_t', 'open', opes.size + generate_attribute 'rb_num_t', 'open', operands.size generate_attribute 'rb_num_t', 'popn', pops.size generate_attribute 'rb_num_t', 'retn', rets.size generate_attribute 'rb_num_t', 'width', width @@ -191,7 +191,7 @@ class RubyVM::BareInstructions def default_definition_of_handles_sp # Insn with ISEQ should yield it; can handle sp. - return opes.any? {|o| o[:type] == 'ISEQ' } + return operands.any? {|o| o[:type] == 'ISEQ' } end def default_definition_of_leaf diff --git a/tool/ruby_vm/models/c_expr.rb b/tool/ruby_vm/models/c_expr.rb index 073112f545..4b5aec58dd 100644 --- a/tool/ruby_vm/models/c_expr.rb +++ b/tool/ruby_vm/models/c_expr.rb @@ -36,6 +36,10 @@ class RubyVM::CExpr end def inspect - sprintf "#<%s:%d %s>", @__FILE__, @__LINE__, @expr + if @__LINE__ + sprintf "#<%s:%d %s>", @__FILE__, @__LINE__, @expr + else + sprintf "#<%s %s>", @__FILE__, @expr + end end end diff --git a/tool/ruby_vm/models/operands_unifications.rb b/tool/ruby_vm/models/operands_unifications.rb index ee4e3a695d..10e5742897 100644 --- a/tool/ruby_vm/models/operands_unifications.rb +++ b/tool/ruby_vm/models/operands_unifications.rb @@ -38,8 +38,8 @@ class RubyVM::OperandsUnifications < RubyVM::BareInstructions end def operand_shift_of var - before = @original.opes.find_index var - after = @opes.find_index var + before = @original.operands.find_index var + after = @operands.find_index var raise "no #{var} for #{@name}" unless before and after return before - after end @@ -50,7 +50,7 @@ class RubyVM::OperandsUnifications < RubyVM::BareInstructions case val when '*' then next nil else - type = @original.opes[i][:type] + type = @original.operands[i][:type] expr = RubyVM::Typemap.typecast_to_VALUE type, val next "#{ptr}[#{i}] == #{expr}" end @@ -85,7 +85,7 @@ class RubyVM::OperandsUnifications < RubyVM::BareInstructions def compose location, spec, template name = namegen spec *, argv = *spec - opes = @original.opes + opes = @original.operands if opes.size != argv.size raise sprintf("operand size mismatch for %s (%s's: %d, given: %d)", name, template[:name], opes.size, argv.size) diff --git a/tool/ruby_vm/scripts/insns2vm.rb b/tool/ruby_vm/scripts/insns2vm.rb index 8325dd364f..47d8da5513 100644 --- a/tool/ruby_vm/scripts/insns2vm.rb +++ b/tool/ruby_vm/scripts/insns2vm.rb @@ -15,10 +15,10 @@ require_relative '../controllers/application_controller.rb' module RubyVM::Insns2VM def self.router argv - options = { destdir: nil } + options = { destdir: nil, basedir: nil } targets = generate_parser(options).parse argv return targets.map do |i| - next ApplicationController.new.generate i, options[:destdir] + next ApplicationController.new.generate i, options[:destdir], options[:basedir] end end @@ -84,6 +84,14 @@ module RubyVM::Insns2VM options[:destdir] = dir end + this.on "--basedir=DIR", <<-'begin' do |dir| + Change the base directory from the current working directory + to the given path. Used for searching the source template. + begin + raise "directory was not found in '#{dir}'" unless Dir.exist?(dir) + options[:basedir] = dir + end + this.on "-V", "--[no-]verbose", <<-'end' Please let us ignore this and be modest. end diff --git a/tool/ruby_vm/views/_comptime_insn_stack_increase.erb b/tool/ruby_vm/views/_comptime_insn_stack_increase.erb index b633ab4d32..cb895815ce 100644 --- a/tool/ruby_vm/views/_comptime_insn_stack_increase.erb +++ b/tool/ruby_vm/views/_comptime_insn_stack_increase.erb @@ -42,7 +42,7 @@ comptime_insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *o % end case <%= i.bin %>: return <%= attr_function %>(<%= - i.opes.map.with_index do |v, j| + i.operands.map.with_index do |v, j| if v[:type] == 'CALL_DATA' && i.has_attribute?('comptime_sp_inc') v = v.dup v[:type] = 'CALL_INFO' diff --git a/tool/ruby_vm/views/_insn_entry.erb b/tool/ruby_vm/views/_insn_entry.erb index f34afddb1f..6ec33461c4 100644 --- a/tool/ruby_vm/views/_insn_entry.erb +++ b/tool/ruby_vm/views/_insn_entry.erb @@ -20,11 +20,11 @@ INSN_ENTRY(<%= insn.name %>) <%= render_c_expr konst -%> % end % -% insn.opes.each_with_index do |ope, i| +% insn.operands.each_with_index do |ope, i| <%= ope[:decl] %> = (<%= ope[:type] %>)GET_OPERAND(<%= i + 1 %>); % end # define INSN_ATTR(x) <%= insn.call_attribute(' ## x ## ') %> - const bool leaf = INSN_ATTR(leaf); + const bool MAYBE_UNUSED(leaf) = INSN_ATTR(leaf); % insn.pops.reverse_each.with_index.reverse_each do |pop, i| <%= pop[:decl] %> = <%= insn.cast_from_VALUE pop, "TOPN(#{i})"%>; % end @@ -35,13 +35,13 @@ INSN_ENTRY(<%= insn.name %>) % end /* ### Instruction preambles. ### */ - if (! leaf) ADD_PC(INSN_ATTR(width)); + ADD_PC(INSN_ATTR(width)); % if insn.handles_sp? POPN(INSN_ATTR(popn)); % end <%= insn.handle_canary "SETUP_CANARY(leaf)" -%> COLLECT_USAGE_INSN(INSN_ATTR(bin)); -% insn.opes.each_with_index do |ope, i| +% insn.operands.each_with_index do |ope, i| COLLECT_USAGE_OPERAND(INSN_ATTR(bin), <%= i %>, <%= ope[:name] %>); % end % unless body.empty? @@ -68,7 +68,6 @@ INSN_ENTRY(<%= insn.name %>) VM_ASSERT(!RB_TYPE_P(TOPN(<%= i %>), T_MOVED)); % end % end - if (leaf) ADD_PC(INSN_ATTR(width)); # undef INSN_ATTR /* ### Leave the instruction. ### */ diff --git a/tool/ruby_vm/views/_leaf_helpers.erb b/tool/ruby_vm/views/_leaf_helpers.erb index 1735db2196..f740107a0a 100644 --- a/tool/ruby_vm/views/_leaf_helpers.erb +++ b/tool/ruby_vm/views/_leaf_helpers.erb @@ -10,7 +10,7 @@ #include "iseq.h" -// This is used to tell MJIT that this insn would be leaf if CHECK_INTS didn't exist. +// This is used to tell RJIT that this insn would be leaf if CHECK_INTS didn't exist. // It should be used only when RUBY_VM_CHECK_INTS is directly written in insns.def. static bool leafness_of_check_ints = false; diff --git a/tool/ruby_vm/views/_mjit_compile_getinlinecache.erb b/tool/ruby_vm/views/_mjit_compile_getinlinecache.erb deleted file mode 100644 index fa38af4045..0000000000 --- a/tool/ruby_vm/views/_mjit_compile_getinlinecache.erb +++ /dev/null @@ -1,31 +0,0 @@ -% # -*- C -*- -% # Copyright (c) 2020 Takashi Kokubun. All rights reserved. -% # -% # This file is a part of the programming language Ruby. Permission is hereby -% # granted, to either redistribute and/or modify this file, provided that the -% # conditions mentioned in the file COPYING are met. Consult the file for -% # details. -% -% # compiler: Declare dst and ic -% insn.opes.each_with_index do |ope, i| - <%= ope.fetch(:decl) %> = (<%= ope.fetch(:type) %>)operands[<%= i %>]; -% end - -% # compiler: Capture IC values, locking getinlinecache - struct iseq_inline_constant_cache_entry *ice = ic->entry; - if (ice != NULL && !status->compile_info->disable_const_cache) { -% # JIT: Inline everything in IC, and cancel the slow path - fprintf(f, " if (vm_inlined_ic_hit_p(0x%"PRIxVALUE", 0x%"PRIxVALUE", (const rb_cref_t *)0x%"PRIxVALUE", reg_cfp->ep)) {", ice->flags, ice->value, (VALUE)ice->ic_cref); - fprintf(f, " stack[%d] = 0x%"PRIxVALUE";\n", b->stack_size, ice->value); - fprintf(f, " goto label_%d;\n", pos + insn_len(insn) + (int)dst); - fprintf(f, " }"); - fprintf(f, " else {"); - fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); - fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); - fprintf(f, " goto const_cancel;\n"); - fprintf(f, " }"); - -% # compiler: Move JIT compiler's internal stack pointer - b->stack_size += <%= insn.call_attribute('sp_inc') %>; - break; - } diff --git a/tool/ruby_vm/views/_mjit_compile_insn.erb b/tool/ruby_vm/views/_mjit_compile_insn.erb deleted file mode 100644 index f54d1b0e0e..0000000000 --- a/tool/ruby_vm/views/_mjit_compile_insn.erb +++ /dev/null @@ -1,92 +0,0 @@ -% # -*- C -*- -% # Copyright (c) 2018 Takashi Kokubun. All rights reserved. -% # -% # This file is a part of the programming language Ruby. Permission is hereby -% # granted, to either redistribute and/or modify this file, provided that the -% # conditions mentioned in the file COPYING are met. Consult the file for -% # details. - fprintf(f, "{\n"); - { -% # compiler: Prepare operands which may be used by `insn.call_attribute` -% insn.opes.each_with_index do |ope, i| - MAYBE_UNUSED(<%= ope.fetch(:decl) %>) = (<%= ope.fetch(:type) %>)operands[<%= i %>]; -% end -% -% # JIT: Declare stack_size to be used in some macro of _mjit_compile_insn_body.erb - if (status->local_stack_p) { - fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); - } -% -% # JIT: Declare variables for operands, popped values and return values -% insn.declarations.each do |decl| - fprintf(f, " <%= decl %>;\n"); -% end - -% # JIT: Set const expressions for `RubyVM::OperandsUnifications` insn -% insn.preamble.each do |amble| - fprintf(f, "<%= amble.expr.sub(/const \S+\s+/, '') %>\n"); -% end -% -% # JIT: Initialize operands -% insn.opes.each_with_index do |ope, i| - fprintf(f, " <%= ope.fetch(:name) %> = (<%= ope.fetch(:type) %>)0x%"PRIxVALUE";", operands[<%= i %>]); -% case ope.fetch(:type) -% when 'ID' - comment_id(f, (ID)operands[<%= i %>]); -% when 'CALL_DATA' - comment_id(f, vm_ci_mid(((CALL_DATA)operands[<%= i %>])->ci)); -% when 'VALUE' - if (SYMBOL_P((VALUE)operands[<%= i %>])) comment_id(f, SYM2ID((VALUE)operands[<%= i %>])); -% end - fprintf(f, "\n"); -% end -% -% # JIT: Initialize popped values -% insn.pops.reverse_each.with_index.reverse_each do |pop, i| - fprintf(f, " <%= pop.fetch(:name) %> = stack[%d];\n", b->stack_size - <%= i + 1 %>); -% end -% -% # JIT: move sp and pc if necessary -<%= render 'mjit_compile_pc_and_sp', locals: { insn: insn } -%> -% -% # JIT: Print insn body in insns.def -<%= render 'mjit_compile_insn_body', locals: { insn: insn } -%> -% -% # JIT: Set return values -% insn.rets.reverse_each.with_index do |ret, i| -% # TOPN(n) = ... - fprintf(f, " stack[%d] = <%= ret.fetch(:name) %>;\n", b->stack_size + (int)<%= insn.call_attribute('sp_inc') %> - <%= i + 1 %>); -% end -% -% # JIT: We should evaluate ISeq modified for TracePoint if it's enabled. Note: This is slow. -% # leaf insn may not cancel JIT. leaf_without_check_ints is covered in RUBY_VM_CHECK_INTS of _mjit_compile_insn_body.erb. -% unless insn.always_leaf? || insn.leaf_without_check_ints? - fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); - fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)<%= insn.call_attribute('sp_inc') %>); - if (!pc_moved_p) { - fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); - } - fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); - fprintf(f, " goto cancel;\n"); - fprintf(f, " }\n"); -% end -% -% # compiler: Move JIT compiler's internal stack pointer - b->stack_size += <%= insn.call_attribute('sp_inc') %>; - } - fprintf(f, "}\n"); -% -% # compiler: If insn has conditional JUMP, the code should go to the branch not targeted by JUMP next. -% if insn.expr.expr =~ /if\s+\([^{}]+\)\s+\{[^{}]+JUMP\([^)]+\);[^{}]+\}/ - if (ALREADY_COMPILED_P(status, pos + insn_len(insn))) { - fprintf(f, "goto label_%d;\n", pos + insn_len(insn)); - } - else { - compile_insns(f, body, b->stack_size, pos + insn_len(insn), status); - } -% end -% -% # compiler: If insn returns (leave) or does longjmp (throw), the branch should no longer be compiled. TODO: create attr for it? -% if insn.expr.expr =~ /\sTHROW_EXCEPTION\([^)]+\);/ || insn.expr.expr =~ /\bvm_pop_frame\(/ - b->finish_p = TRUE; -% end diff --git a/tool/ruby_vm/views/_mjit_compile_insn_body.erb b/tool/ruby_vm/views/_mjit_compile_insn_body.erb deleted file mode 100644 index 187e043837..0000000000 --- a/tool/ruby_vm/views/_mjit_compile_insn_body.erb +++ /dev/null @@ -1,129 +0,0 @@ -% # -*- C -*- -% # Copyright (c) 2018 Takashi Kokubun. All rights reserved. -% # -% # This file is a part of the programming language Ruby. Permission is hereby -% # granted, to either redistribute and/or modify this file, provided that the -% # conditions mentioned in the file COPYING are met. Consult the file for -% # details. -% -% to_cstr = lambda do |line| -% normalized = line.gsub(/\t/, ' ' * 8) -% indented = normalized.sub(/\A(?!#)/, ' ') # avoid indenting preprocessor -% rstring2cstr(indented.rstrip).sub(/"\z/, '\\n"') -% end -% -% # -% # Expand simple macro, which doesn't require dynamic C code. -% # -% expand_simple_macros = lambda do |arg_expr| -% arg_expr.dup.tap do |expr| -% # For `leave`. We can't proceed next ISeq in the same JIT function. -% expr.gsub!(/^(?<indent>\s*)RESTORE_REGS\(\);\n/) do -% indent = Regexp.last_match[:indent] -% <<-end.gsub(/^ +/, '') -% #if OPT_CALL_THREADED_CODE -% #{indent}rb_ec_thread_ptr(ec)->retval = val; -% #{indent}return 0; -% #else -% #{indent}return val; -% #endif -% end -% end -% expr.gsub!(/^(?<indent>\s*)NEXT_INSN\(\);\n/) do -% indent = Regexp.last_match[:indent] -% <<-end.gsub(/^ +/, '') -% #{indent}UNREACHABLE_RETURN(Qundef); -% end -% end -% end -% end -% -% # -% # Print a body of insn, but with macro expansion. -% # -% expand_simple_macros.call(insn.expr.expr).each_line do |line| -% # -% # Expand dynamic macro here (only JUMP for now) -% # -% # TODO: support combination of following macros in the same line -% case line -% when /\A\s+RUBY_VM_CHECK_INTS\(ec\);\s+\z/ -% if insn.leaf_without_check_ints? # lazily move PC and optionalize mjit_call_p here - fprintf(f, " if (UNLIKELY(RUBY_VM_INTERRUPTED_ANY(ec))) {\n"); - fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ - fprintf(f, " rb_threadptr_execute_interrupts(rb_ec_thread_ptr(ec), 0);\n"); - fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); - fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); - fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); - fprintf(f, " goto cancel;\n"); - fprintf(f, " }\n"); - fprintf(f, " }\n"); -% else - fprintf(f, <%= to_cstr.call(line) %>); -% end -% when /\A\s+JUMP\((?<dest>[^)]+)\);\s+\z/ -% dest = Regexp.last_match[:dest] -% -% if insn.name == 'opt_case_dispatch' # special case... TODO: use another macro to avoid checking name - { - struct case_dispatch_var arg; - arg.f = f; - arg.base_pos = pos + insn_len(insn); - arg.last_value = Qundef; - - fprintf(f, " switch (<%= dest %>) {\n"); - st_foreach(RHASH_TBL_RAW(hash), compile_case_dispatch_each, (VALUE)&arg); - fprintf(f, " case %lu:\n", else_offset); - fprintf(f, " goto label_%lu;\n", arg.base_pos + else_offset); - fprintf(f, " }\n"); - } -% else -% # Before we `goto` next insn, we need to set return values, especially for getinlinecache -% insn.rets.reverse_each.with_index do |ret, i| -% # TOPN(n) = ... - fprintf(f, " stack[%d] = <%= ret.fetch(:name) %>;\n", b->stack_size + (int)<%= insn.call_attribute('sp_inc') %> - <%= i + 1 %>); -% end -% - next_pos = pos + insn_len(insn) + (unsigned int)<%= dest %>; - fprintf(f, " goto label_%d;\n", next_pos); -% end -% when /\A\s+CALL_SIMPLE_METHOD\(\);\s+\z/ -% # For `opt_xxx`'s fallbacks. - if (status->local_stack_p) { - fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); - } - fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); - fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); - fprintf(f, " goto cancel;\n"); -% when /\A(?<prefix>.+\b)INSN_LABEL\((?<name>[^)]+)\)(?<suffix>.+)\z/m -% prefix, name, suffix = Regexp.last_match[:prefix], Regexp.last_match[:name], Regexp.last_match[:suffix] - fprintf(f, " <%= prefix.gsub(/\t/, ' ' * 8) %>INSN_LABEL(<%= name %>_%d)<%= suffix.sub(/\n/, '\n') %>", pos); -% else -% if insn.handles_sp? -% # If insn.handles_sp? is true, cfp->sp might be changed inside insns (like vm_caller_setup_arg_block) -% # and thus we need to use cfp->sp, even when local_stack_p is TRUE. When insn.handles_sp? is true, -% # cfp->sp should be available too because _mjit_compile_pc_and_sp.erb sets it. - fprintf(f, <%= to_cstr.call(line) %>); -% else -% # If local_stack_p is TRUE and insn.handles_sp? is false, stack values are only available in local variables -% # for stack. So we need to replace those macros if local_stack_p is TRUE here. -% case line -% when /\bGET_SP\(\)/ -% # reg_cfp->sp - fprintf(f, <%= to_cstr.call(line.sub(/\bGET_SP\(\)/, '%s')) %>, (status->local_stack_p ? "(stack + stack_size)" : "GET_SP()")); -% when /\bSTACK_ADDR_FROM_TOP\((?<num>[^)]+)\)/ -% # #define STACK_ADDR_FROM_TOP(n) (GET_SP()-(n)) -% num = Regexp.last_match[:num] - fprintf(f, <%= to_cstr.call(line.sub(/\bSTACK_ADDR_FROM_TOP\(([^)]+)\)/, '%s')) %>, - (status->local_stack_p ? "(stack + (stack_size - (<%= num %>)))" : "STACK_ADDR_FROM_TOP(<%= num %>)")); -% when /\bTOPN\((?<num>[^)]+)\)/ -% # #define TOPN(n) (*(GET_SP()-(n)-1)) -% num = Regexp.last_match[:num] - fprintf(f, <%= to_cstr.call(line.sub(/\bTOPN\(([^)]+)\)/, '%s')) %>, - (status->local_stack_p ? "*(stack + (stack_size - (<%= num %>) - 1))" : "TOPN(<%= num %>)")); -% else - fprintf(f, <%= to_cstr.call(line) %>); -% end -% end -% end -% end diff --git a/tool/ruby_vm/views/_mjit_compile_invokebuiltin.erb b/tool/ruby_vm/views/_mjit_compile_invokebuiltin.erb deleted file mode 100644 index a3796ffc5e..0000000000 --- a/tool/ruby_vm/views/_mjit_compile_invokebuiltin.erb +++ /dev/null @@ -1,29 +0,0 @@ -% # -*- C -*- -% # Copyright (c) 2020 Urabe, Shyouhei. All rights reserved. -% # -% # This file is a part of the programming language Ruby. Permission is hereby -% # granted, to either redistribute and/or modify this file, provided that the -% # conditions mentioned in the file COPYING are met. Consult the file for -% # details. -% -% insn.opes.each_with_index do |ope, i| - <%= ope.fetch(:decl) %> = (<%= ope.fetch(:type) %>)operands[<%= i %>]; -% end - rb_snum_t sp_inc = <%= insn.call_attribute('sp_inc') %>; - unsigned sp = b->stack_size + (unsigned)sp_inc; - VM_ASSERT(b->stack_size > -sp_inc); - VM_ASSERT(sp_inc < UINT_MAX - b->stack_size); - - if (bf->compiler) { - fprintf(f, "{\n"); - fprintf(f, " VALUE val;\n"); - bf->compiler(f, <%= - insn.name == 'invokebuiltin' ? '-1' : '(rb_num_t)operands[1]' - %>, b->stack_size, body->builtin_inline_p); - fprintf(f, " stack[%u] = val;\n", sp - 1); - fprintf(f, "}\n"); -% if insn.name != 'opt_invokebuiltin_delegate_leave' - b->stack_size = sp; - break; -% end - } diff --git a/tool/ruby_vm/views/_mjit_compile_ivar.erb b/tool/ruby_vm/views/_mjit_compile_ivar.erb deleted file mode 100644 index 1425b3b055..0000000000 --- a/tool/ruby_vm/views/_mjit_compile_ivar.erb +++ /dev/null @@ -1,110 +0,0 @@ -% # -*- C -*- -% # Copyright (c) 2018 Takashi Kokubun. All rights reserved. -% # -% # This file is a part of the programming language Ruby. Permission is hereby -% # granted, to either redistribute and/or modify this file, provided that the -% # conditions mentioned in the file COPYING are met. Consult the file for -% # details. -% -% # Optimized case of get_instancevariable instruction. -#if OPT_IC_FOR_IVAR -{ -% # compiler: Prepare operands which may be used by `insn.call_attribute` -% insn.opes.each_with_index do |ope, i| - MAYBE_UNUSED(<%= ope.fetch(:decl) %>) = (<%= ope.fetch(:type) %>)operands[<%= i %>]; -% end -% # compiler: Use copied IVC to avoid race condition - IVC ic_copy = &(status->is_entries + ((union iseq_inline_storage_entry *)ic - body->is_entries))->iv_cache; -% - if (!status->compile_info->disable_ivar_cache && ic_copy->entry) { // Only ic_copy is enabled. -% # JIT: optimize away motion of sp and pc. This path does not call rb_warning() and so it's always leaf and not `handles_sp`. -% # <%= render 'mjit_compile_pc_and_sp', locals: { insn: insn } -%> -% -% # JIT: prepare vm_getivar/vm_setivar arguments and variables - fprintf(f, "{\n"); - fprintf(f, " VALUE obj = GET_SELF();\n"); - fprintf(f, " const uint32_t index = %u;\n", (ic_copy->entry->index)); - if (status->merge_ivar_guards_p) { -% # JIT: Access ivar without checking these VM_ASSERTed prerequisites as we checked them in the beginning of `mjit_compile_body` - fprintf(f, " VM_ASSERT(RB_TYPE_P(obj, T_OBJECT));\n"); - fprintf(f, " VM_ASSERT((rb_serial_t)%"PRI_SERIALT_PREFIX"u == RCLASS_SERIAL(RBASIC(obj)->klass));\n", ic_copy->entry->class_serial); - fprintf(f, " VM_ASSERT(index < ROBJECT_NUMIV(obj));\n"); -% if insn.name == 'setinstancevariable' -#if USE_RVARGC - fprintf(f, " if (LIKELY(!RB_OBJ_FROZEN_RAW(obj) && index < ROBJECT_NUMIV(obj))) {\n"); - fprintf(f, " RB_OBJ_WRITE(obj, &ROBJECT_IVPTR(obj)[index], stack[%d]);\n", b->stack_size - 1); -#else - fprintf(f, " if (LIKELY(!RB_OBJ_FROZEN_RAW(obj) && %s)) {\n", status->max_ivar_index >= ROBJECT_EMBED_LEN_MAX ? "true" : "RB_FL_ANY_RAW(obj, ROBJECT_EMBED)"); - fprintf(f, " RB_OBJ_WRITE(obj, &ROBJECT(obj)->as.%s, stack[%d]);\n", - status->max_ivar_index >= ROBJECT_EMBED_LEN_MAX ? "heap.ivptr[index]" : "ary[index]", b->stack_size - 1); -#endif - fprintf(f, " }\n"); -% else - fprintf(f, " VALUE val;\n"); -#if USE_RVARGC - fprintf(f, " if (LIKELY(index < ROBJECT_NUMIV(obj) && (val = ROBJECT_IVPTR(obj)[index]) != Qundef)) {\n"); -#else - fprintf(f, " if (LIKELY(%s && (val = ROBJECT(obj)->as.%s) != Qundef)) {\n", - status->max_ivar_index >= ROBJECT_EMBED_LEN_MAX ? "true" : "RB_FL_ANY_RAW(obj, ROBJECT_EMBED)", - status->max_ivar_index >= ROBJECT_EMBED_LEN_MAX ? "heap.ivptr[index]" : "ary[index]"); -#endif - fprintf(f, " stack[%d] = val;\n", b->stack_size); - fprintf(f, " }\n"); -%end - } - else { - fprintf(f, " const rb_serial_t ic_serial = (rb_serial_t)%"PRI_SERIALT_PREFIX"u;\n", ic_copy->entry->class_serial); -% # JIT: cache hit path of vm_getivar/vm_setivar, or cancel JIT (recompile it with exivar) -% if insn.name == 'setinstancevariable' - fprintf(f, " if (LIKELY(RB_TYPE_P(obj, T_OBJECT) && ic_serial == RCLASS_SERIAL(RBASIC(obj)->klass) && index < ROBJECT_NUMIV(obj) && !RB_OBJ_FROZEN_RAW(obj))) {\n"); - fprintf(f, " VALUE *ptr = ROBJECT_IVPTR(obj);\n"); - fprintf(f, " RB_OBJ_WRITE(obj, &ptr[index], stack[%d]);\n", b->stack_size - 1); - fprintf(f, " }\n"); -% else - fprintf(f, " VALUE val;\n"); - fprintf(f, " if (LIKELY(RB_TYPE_P(obj, T_OBJECT) && ic_serial == RCLASS_SERIAL(RBASIC(obj)->klass) && index < ROBJECT_NUMIV(obj) && (val = ROBJECT_IVPTR(obj)[index]) != Qundef)) {\n"); - fprintf(f, " stack[%d] = val;\n", b->stack_size); - fprintf(f, " }\n"); -% end - } - fprintf(f, " else {\n"); - fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); - fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); - fprintf(f, " goto ivar_cancel;\n"); - fprintf(f, " }\n"); - -% # compiler: Move JIT compiler's internal stack pointer - b->stack_size += <%= insn.call_attribute('sp_inc') %>; - fprintf(f, "}\n"); - break; - } -% if insn.name == 'getinstancevariable' - else if (!status->compile_info->disable_exivar_cache && ic_copy->entry) { -% # JIT: optimize away motion of sp and pc. This path does not call rb_warning() and so it's always leaf and not `handles_sp`. -% # <%= render 'mjit_compile_pc_and_sp', locals: { insn: insn } -%> -% -% # JIT: prepare vm_getivar's arguments and variables - fprintf(f, "{\n"); - fprintf(f, " VALUE obj = GET_SELF();\n"); - fprintf(f, " const rb_serial_t ic_serial = (rb_serial_t)%"PRI_SERIALT_PREFIX"u;\n", ic_copy->entry->class_serial); - fprintf(f, " const uint32_t index = %u;\n", ic_copy->entry->index); -% # JIT: cache hit path of vm_getivar, or cancel JIT (recompile it without any ivar optimization) - fprintf(f, " struct gen_ivtbl *ivtbl;\n"); - fprintf(f, " VALUE val;\n"); - fprintf(f, " if (LIKELY(FL_TEST_RAW(obj, FL_EXIVAR) && ic_serial == RCLASS_SERIAL(RBASIC(obj)->klass) && rb_ivar_generic_ivtbl_lookup(obj, &ivtbl) && index < ivtbl->numiv && (val = ivtbl->ivptr[index]) != Qundef)) {\n"); - fprintf(f, " stack[%d] = val;\n", b->stack_size); - fprintf(f, " }\n"); - fprintf(f, " else {\n"); - fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); - fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); - fprintf(f, " goto exivar_cancel;\n"); - fprintf(f, " }\n"); - -% # compiler: Move JIT compiler's internal stack pointer - b->stack_size += <%= insn.call_attribute('sp_inc') %>; - fprintf(f, "}\n"); - break; - } -% end -} -#endif // OPT_IC_FOR_IVAR diff --git a/tool/ruby_vm/views/_mjit_compile_pc_and_sp.erb b/tool/ruby_vm/views/_mjit_compile_pc_and_sp.erb deleted file mode 100644 index 390b3ce525..0000000000 --- a/tool/ruby_vm/views/_mjit_compile_pc_and_sp.erb +++ /dev/null @@ -1,38 +0,0 @@ -% # Copyright (c) 2018 Takashi Kokubun. All rights reserved. -% # -% # This file is a part of the programming language Ruby. Permission is hereby -% # granted, to either redistribute and/or modify this file, provided that the -% # conditions mentioned in the file COPYING are met. Consult the file for -% # details. -% -% # JIT: When an insn is leaf, we don't need to Move pc for a catch table on catch_except_p, #caller_locations, -% # and rb_profile_frames. For check_ints, we lazily move PC when we have interruptions. - MAYBE_UNUSED(bool pc_moved_p) = false; - if (<%= !(insn.always_leaf? || insn.leaf_without_check_ints?) %>) { - fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ - pc_moved_p = true; - } -% -% # JIT: move sp to use or preserve stack variables - if (status->local_stack_p) { -% # sp motion is optimized away for `handles_sp? #=> false` case. -% # Thus sp should be set properly before `goto cancel`. -% if insn.handles_sp? -% # JIT-only behavior (pushing JIT's local variables to VM's stack): - { - rb_snum_t i, push_size; - push_size = -<%= insn.call_attribute('sp_inc') %> + <%= insn.rets.size %> - <%= insn.pops.size %>; - fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ - for (i = 0; i < push_size; i++) { - fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); - } - } -% end - } - else { -% if insn.handles_sp? - fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - <%= insn.pops.size %>); /* POPN(INSN_ATTR(popn)); */ -% else - fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); -% end - } diff --git a/tool/ruby_vm/views/_mjit_compile_send.erb b/tool/ruby_vm/views/_mjit_compile_send.erb deleted file mode 100644 index 8900ee6425..0000000000 --- a/tool/ruby_vm/views/_mjit_compile_send.erb +++ /dev/null @@ -1,119 +0,0 @@ -% # -*- C -*- -% # Copyright (c) 2018 Takashi Kokubun. All rights reserved. -% # -% # This file is a part of the programming language Ruby. Permission is hereby -% # granted, to either redistribute and/or modify this file, provided that the -% # conditions mentioned in the file COPYING are met. Consult the file for -% # details. -% -% # Optimized case of send / opt_send_without_block instructions. -{ -% # compiler: Prepare operands which may be used by `insn.call_attribute` -% insn.opes.each_with_index do |ope, i| - MAYBE_UNUSED(<%= ope.fetch(:decl) %>) = (<%= ope.fetch(:type) %>)operands[<%= i %>]; -% end -% # compiler: Use captured cc to avoid race condition - size_t cd_index = call_data_index(cd, body); - const struct rb_callcache **cc_entries = captured_cc_entries(status); - const struct rb_callcache *captured_cc = cc_entries[cd_index]; -% -% # compiler: Inline send insn where some supported fastpath is used. - const rb_iseq_t *iseq = NULL; - const CALL_INFO ci = cd->ci; - int kw_splat = IS_ARGS_KW_SPLAT(ci) > 0; - extern bool rb_splat_or_kwargs_p(const struct rb_callinfo *restrict ci); - if (!status->compile_info->disable_send_cache && has_valid_method_type(captured_cc) && ( -% # `CC_SET_FASTPATH(cd->cc, vm_call_cfunc_with_frame, ...)` in `vm_call_cfunc` - (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC - && !rb_splat_or_kwargs_p(ci) && !kw_splat) -% # `CC_SET_FASTPATH(cc, vm_call_iseq_setup_func(...), vm_call_iseq_optimizable_p(...))` in `vm_callee_setup_arg`, -% # and support only non-VM_CALL_TAILCALL path inside it - || (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_ISEQ - && fastpath_applied_iseq_p(ci, captured_cc, iseq = def_iseq_ptr(vm_cc_cme(captured_cc)->def)) - && !(vm_ci_flag(ci) & VM_CALL_TAILCALL)) - )) { - const bool cfunc_debug = false; // Set true when you want to see inlined cfunc - if (cfunc_debug && vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) - fprintf(stderr, " * %s\n", rb_id2name(vm_ci_mid(ci))); - - int sp_inc = (int)sp_inc_of_sendish(ci); - fprintf(f, "{\n"); - -% # JIT: Invalidate call cache if it requires vm_search_method. This allows to inline some of following things. - bool opt_class_of = !maybe_special_const_class_p(captured_cc->klass); // If true, use RBASIC_CLASS instead of CLASS_OF to reduce code size - fprintf(f, " const struct rb_callcache *cc = (const struct rb_callcache *)0x%"PRIxVALUE";\n", (VALUE)captured_cc); - fprintf(f, " const rb_callable_method_entry_t *cc_cme = (const rb_callable_method_entry_t *)0x%"PRIxVALUE";\n", (VALUE)vm_cc_cme(captured_cc)); - fprintf(f, " const VALUE recv = stack[%d];\n", b->stack_size + sp_inc - 1); - fprintf(f, " if (UNLIKELY(%s || !vm_cc_valid_p(cc, cc_cme, %s(recv)))) {\n", opt_class_of ? "RB_SPECIAL_CONST_P(recv)" : "false", opt_class_of ? "RBASIC_CLASS" : "CLASS_OF"); - fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); - fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); - fprintf(f, " goto send_cancel;\n"); - fprintf(f, " }\n"); - -% # JIT: move sp and pc if necessary -<%= render 'mjit_compile_pc_and_sp', locals: { insn: insn } -%> - -% # JIT: If ISeq is inlinable, call the inlined method without pushing a frame. - if (iseq && status->inlined_iseqs != NULL && ISEQ_BODY(iseq) == status->inlined_iseqs[pos]) { - fprintf(f, " {\n"); - fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); - fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size + sp_inc - 1); - fprintf(f, " stack[%d] = _mjit%d_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size + sp_inc - 1, status->compiled_id, pos); - fprintf(f, " reg_cfp->self = orig_self;\n"); - fprintf(f, " }\n"); - } - else { -% # JIT: Forked `vm_sendish` (except method_explorer = vm_search_method_wrap) to inline various things - fprintf(f, " {\n"); - fprintf(f, " VALUE val;\n"); - fprintf(f, " struct rb_calling_info calling;\n"); -% if insn.name == 'send' - fprintf(f, " calling.block_handler = vm_caller_setup_arg_block(ec, reg_cfp, (const struct rb_callinfo *)0x%"PRIxVALUE", (rb_iseq_t *)0x%"PRIxVALUE", FALSE);\n", (VALUE)ci, (VALUE)blockiseq); -% else - fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); -% end - fprintf(f, " calling.kw_splat = %d;\n", kw_splat); - fprintf(f, " calling.recv = stack[%d];\n", b->stack_size + sp_inc - 1); - fprintf(f, " calling.argc = %d;\n", vm_ci_argc(ci)); - - if (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) { -% # TODO: optimize this more - fprintf(f, " calling.ci = (CALL_INFO)0x%"PRIxVALUE";\n", (VALUE)ci); // creating local cd here because operand's cd->cc may not be the same as inlined cc. - fprintf(f, " calling.cc = cc;"); - fprintf(f, " val = vm_call_cfunc_with_frame(ec, reg_cfp, &calling);\n"); - } - else { // VM_METHOD_TYPE_ISEQ -% # fastpath_applied_iseq_p checks rb_simple_iseq_p, which ensures has_opt == FALSE - fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, cc_cme, 0, %d, %d);\n", ISEQ_BODY(iseq)->param.size, ISEQ_BODY(iseq)->local_table_size); - if (ISEQ_BODY(iseq)->catch_except_p) { - fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); - fprintf(f, " val = vm_exec(ec, true);\n"); - } - else { - fprintf(f, " if ((val = mjit_exec(ec)) == Qundef) {\n"); - fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup - fprintf(f, " val = vm_exec(ec, false);\n"); - fprintf(f, " }\n"); - } - } - fprintf(f, " stack[%d] = val;\n", b->stack_size + sp_inc - 1); - fprintf(f, " }\n"); - -% # JIT: We should evaluate ISeq modified for TracePoint if it's enabled. Note: This is slow. - fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); - fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)<%= insn.call_attribute('sp_inc') %>); - if (!pc_moved_p) { - fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); - } - fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); - fprintf(f, " goto cancel;\n"); - fprintf(f, " }\n"); - } - -% # compiler: Move JIT compiler's internal stack pointer - b->stack_size += <%= insn.call_attribute('sp_inc') %>; - - fprintf(f, "}\n"); - break; - } -} diff --git a/tool/ruby_vm/views/lib/ruby_vm/rjit/instruction.rb.erb b/tool/ruby_vm/views/lib/ruby_vm/rjit/instruction.rb.erb new file mode 100644 index 0000000000..4f08524a77 --- /dev/null +++ b/tool/ruby_vm/views/lib/ruby_vm/rjit/instruction.rb.erb @@ -0,0 +1,14 @@ +module RubyVM::RJIT # :nodoc: all + Instruction = Data.define(:name, :bin, :len, :operands) + + INSNS = { +% RubyVM::Instructions.each_with_index do |insn, i| + <%= i %> => Instruction.new( + name: :<%= insn.name %>, + bin: <%= i %>, # BIN(<%= insn.name %>) + len: <%= insn.width %>, # insn_len + operands: <%= (insn.operands unless insn.name.start_with?('trace_')).inspect %>, + ), +% end + } +end diff --git a/tool/ruby_vm/views/mjit_compile.inc.erb b/tool/ruby_vm/views/mjit_compile.inc.erb deleted file mode 100644 index 5820f81770..0000000000 --- a/tool/ruby_vm/views/mjit_compile.inc.erb +++ /dev/null @@ -1,110 +0,0 @@ -/* -*- C -*- */ - -% # Copyright (c) 2018 Takashi Kokubun. All rights reserved. -% # -% # This file is a part of the programming language Ruby. Permission is hereby -% # granted, to either redistribute and/or modify this file, provided that the -% # conditions mentioned in the file COPYING are met. Consult the file for -% # details. -<%= render 'copyright' %> -% -% # This is an ERB template that generates Ruby code that generates C code that -% # generates JIT-ed C code. -<%= render 'notice', locals: { - this_file: 'is the main part of compile_insn() in mjit_compile.c', - edit: __FILE__, -} -%> -% -% unsupported_insns = [ -% 'defineclass', # low priority -% ] -% -% opt_send_without_block = RubyVM::Instructions.find { |i| i.name == 'opt_send_without_block' } -% if opt_send_without_block.nil? -% raise 'opt_send_without_block not found' -% end -% -% send_compatible_opt_insns = RubyVM::BareInstructions.to_a.select do |insn| -% insn.name.start_with?('opt_') && opt_send_without_block.opes == insn.opes && -% insn.expr.expr.lines.any? { |l| l.match(/\A\s+CALL_SIMPLE_METHOD\(\);\s+\z/) } -% end.map(&:name) -% -% # Available variables and macros in JIT-ed function: -% # ec: the first argument of _mjitXXX -% # reg_cfp: the second argument of _mjitXXX -% # GET_CFP(): refers to `reg_cfp` -% # GET_EP(): refers to `reg_cfp->ep` -% # GET_SP(): refers to `reg_cfp->sp`, or `(stack + stack_size)` if local_stack_p -% # GET_SELF(): refers to `cfp_self` -% # GET_LEP(): refers to `VM_EP_LEP(reg_cfp->ep)` -% # EXEC_EC_CFP(): refers to `val = vm_exec(ec, true)` with frame setup -% # CALL_METHOD(): using `GET_CFP()` and `EXEC_EC_CFP()` -% # TOPN(): refers to `reg_cfp->sp`, or `*(stack + (stack_size - num - 1))` if local_stack_p -% # STACK_ADDR_FROM_TOP(): refers to `reg_cfp->sp`, or `stack + (stack_size - num)` if local_stack_p -% # DISPATCH_ORIGINAL_INSN(): expanded in _mjit_compile_insn.erb -% # THROW_EXCEPTION(): specially defined for JIT -% # RESTORE_REGS(): specially defined for `leave` - -switch (insn) { -% (RubyVM::BareInstructions.to_a + RubyVM::OperandsUnifications.to_a).each do |insn| -% next if unsupported_insns.include?(insn.name) - case BIN(<%= insn.name %>): { -% # Instruction-specific behavior in JIT -% case insn.name -% when 'opt_send_without_block', 'send' -<%= render 'mjit_compile_send', locals: { insn: insn } -%> -% when *send_compatible_opt_insns -% # To avoid cancel, just emit `opt_send_without_block` instead of `opt_*` insn if call cache is populated. -% cd_index = insn.opes.index { |o| o.fetch(:type) == 'CALL_DATA' } - if (has_cache_for_send(captured_cc_entries(status)[call_data_index((CALL_DATA)operands[<%= cd_index %>], body)], BIN(<%= insn.name %>))) { -<%= render 'mjit_compile_send', locals: { insn: opt_send_without_block } -%> -<%= render 'mjit_compile_insn', locals: { insn: opt_send_without_block } -%> - break; - } -% when 'getinstancevariable', 'setinstancevariable' -<%= render 'mjit_compile_ivar', locals: { insn: insn } -%> -% when 'invokebuiltin', 'opt_invokebuiltin_delegate' -<%= render 'mjit_compile_invokebuiltin', locals: { insn: insn } -%> -% when 'opt_getinlinecache' -<%= render 'mjit_compile_getinlinecache', locals: { insn: insn } -%> -% when 'leave', 'opt_invokebuiltin_delegate_leave' -% # opt_invokebuiltin_delegate_leave also implements leave insn. We need to handle it here for inlining. -% if insn.name == 'opt_invokebuiltin_delegate_leave' -<%= render 'mjit_compile_invokebuiltin', locals: { insn: insn } -%> -% else - if (b->stack_size != 1) { - if (mjit_opts.warnings || mjit_opts.verbose) - fprintf(stderr, "MJIT warning: Unexpected JIT stack_size on leave: %d\n", b->stack_size); - status->success = false; - } -% end -% # Skip vm_pop_frame for inlined call - if (status->inlined_iseqs != NULL) { // the current ISeq is NOT being inlined -% # Cancel on interrupts to make leave insn leaf - fprintf(f, " if (UNLIKELY(RUBY_VM_INTERRUPTED_ANY(ec))) {\n"); - fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); - fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); - fprintf(f, " rb_threadptr_execute_interrupts(rb_ec_thread_ptr(ec), 0);\n"); - fprintf(f, " }\n"); - fprintf(f, " ec->cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(reg_cfp);\n"); // vm_pop_frame - } - fprintf(f, " return stack[0];\n"); - b->stack_size += <%= insn.call_attribute('sp_inc') %>; - b->finish_p = TRUE; - break; -% end -% -% # Main insn implementation generated by insns.def -<%= render 'mjit_compile_insn', locals: { insn: insn } -%> - break; - } -% end -% -% # We don't support InstructionsUnifications yet because it's not used for now. -% # We don't support TraceInstructions yet. There is no blocker for it but it's just not implemented. - default: - if (mjit_opts.warnings || mjit_opts.verbose) - fprintf(stderr, "MJIT warning: Skipped to compile unsupported instruction: %s\n", insn_name(insn)); - status->success = false; - break; -} diff --git a/tool/ruby_vm/views/opt_sc.inc.erb b/tool/ruby_vm/views/opt_sc.inc.erb deleted file mode 100644 index e58c81989f..0000000000 --- a/tool/ruby_vm/views/opt_sc.inc.erb +++ /dev/null @@ -1,40 +0,0 @@ -/* -*- C -*- */ - -%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. -%# -%# This file is a part of the programming language Ruby. Permission is hereby -%# granted, to either redistribute and/or modify this file, provided that the -%# conditions mentioned in the file COPYING are met. Consult the file for -%# details. -% raise ':FIXME:TBW' if RubyVM::VmOptsH['STACK_CACHING'] -<%= render 'copyright' %> -<%= render 'notice', locals: { - this_file: 'is for threaded code', - edit: __FILE__, -} -%> - -#define SC_STATE_SIZE 6 - -#define SCS_XX 1 -#define SCS_AX 2 -#define SCS_BX 3 -#define SCS_AB 4 -#define SCS_BA 5 - -#define SC_ERROR 0xffffffff - -static const VALUE sc_insn_info[][SC_STATE_SIZE] = { -#define NO_SC { SC_ERROR, SC_ERROR, SC_ERROR, SC_ERROR, SC_ERROR, SC_ERROR } -% RubyVM::Instructions.each_slice 8 do |a| - <%= a.map{|i| 'NO_SC' }.join(', ') %>, -% end -#undef NO_SC -}; - -static const VALUE sc_insn_next[] = { -% RubyVM::Instructions.each_slice 8 do |a| - <%= a.map{|i| 'SCS_XX' }.join(', ') %>, -% end -}; - -ASSERT_VM_INSTRUCTION_SIZE(sc_insn_next); diff --git a/tool/ruby_vm/views/optinsn.inc.erb b/tool/ruby_vm/views/optinsn.inc.erb index 0190f9e07a..de7bb210ea 100644 --- a/tool/ruby_vm/views/optinsn.inc.erb +++ b/tool/ruby_vm/views/optinsn.inc.erb @@ -29,14 +29,14 @@ insn_operands_unification(INSN *iobj) /* <%= insn.pretty_name %> */ if ( <%= insn.condition('op') %> ) { -% insn.opes.each_with_index do |o, x| +% insn.operands.each_with_index do |o, x| % n = insn.operand_shift_of(o) % if n != 0 then op[<%= x %>] = op[<%= x + n %>]; % end % end iobj->insn_id = <%= insn.bin %>; - iobj->operand_size = <%= insn.opes.size %>; + iobj->operand_size = <%= insn.operands.size %>; break; } % end diff --git a/tool/runruby.rb b/tool/runruby.rb index d9a4c855d3..ec63d1008a 100755 --- a/tool/runruby.rb +++ b/tool/runruby.rb @@ -11,6 +11,8 @@ when ENV['RUNRUBY_USE_GDB'] == 'true' debugger = :gdb when ENV['RUNRUBY_USE_LLDB'] == 'true' debugger = :lldb +when ENV['RUNRUBY_USE_RR'] == 'true' + debugger = :rr when ENV['RUNRUBY_YJIT_STATS'] use_yjit_stat = true end @@ -133,22 +135,16 @@ if File.file?(libruby_so) if e = config['LIBPATHENV'] and !e.empty? env[e] = [abs_archdir, ENV[e]].compact.join(File::PATH_SEPARATOR) end - unless runner - if e = config['PRELOADENV'] - e = nil if e.empty? - e ||= "LD_PRELOAD" if /linux/ =~ RUBY_PLATFORM - end - if e - env[e] = [libruby_so, ENV[e]].compact.join(File::PATH_SEPARATOR) - end - end end +# Work around a bug in FreeBSD 13.2 which can cause fork(2) to hang +# See: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=271490 +env['LD_BIND_NOW'] = 'yes' if /freebsd/ =~ RUBY_PLATFORM ENV.update env if debugger case debugger - when :gdb, nil + when :gdb debugger = %W'gdb -x #{srcdir}/.gdbinit' if File.exist?(gdb = 'run.gdb') or File.exist?(gdb = File.join(abs_archdir, 'run.gdb')) @@ -162,6 +158,8 @@ if debugger debugger.push('-s', lldb) end debugger << '--' + when :rr + debugger = ['rr', 'record'] end if idx = precommand.index(:debugger) diff --git a/tool/sync_default_gems.rb b/tool/sync_default_gems.rb index 78620e1508..0307028eb7 100755 --- a/tool/sync_default_gems.rb +++ b/tool/sync_default_gems.rb @@ -1,606 +1,809 @@ #!/usr/bin/env ruby -# sync upstream github repositories to ruby repository +# Sync upstream github repositories to ruby repository. +# See `tool/sync_default_gems.rb --help` for how to use this. require 'fileutils' -include FileUtils - -REPOSITORIES = { - rubygems: 'rubygems/rubygems', - rdoc: 'ruby/rdoc', - reline: 'ruby/reline', - json: 'flori/json', - psych: 'ruby/psych', - fileutils: 'ruby/fileutils', - fiddle: 'ruby/fiddle', - stringio: 'ruby/stringio', - "io-console": 'ruby/io-console', - "io-nonblock": 'ruby/io-nonblock', - "io-wait": 'ruby/io-wait', - csv: 'ruby/csv', - etc: 'ruby/etc', - date: 'ruby/date', - zlib: 'ruby/zlib', - fcntl: 'ruby/fcntl', - strscan: 'ruby/strscan', - ipaddr: 'ruby/ipaddr', - logger: 'ruby/logger', - ostruct: 'ruby/ostruct', - irb: 'ruby/irb', - forwardable: "ruby/forwardable", - mutex_m: "ruby/mutex_m", - racc: "ruby/racc", - singleton: "ruby/singleton", - open3: "ruby/open3", - getoptlong: "ruby/getoptlong", - pstore: "ruby/pstore", - delegate: "ruby/delegate", - benchmark: "ruby/benchmark", - cgi: "ruby/cgi", - readline: "ruby/readline", - "readline-ext": "ruby/readline-ext", - observer: "ruby/observer", - timeout: "ruby/timeout", - yaml: "ruby/yaml", - uri: "ruby/uri", - openssl: "ruby/openssl", - did_you_mean: "ruby/did_you_mean", - weakref: "ruby/weakref", - tempfile: "ruby/tempfile", - tmpdir: "ruby/tmpdir", - English: "ruby/English", - "net-protocol": "ruby/net-protocol", - "net-http": "ruby/net-http", - bigdecimal: "ruby/bigdecimal", - optparse: "ruby/optparse", - set: "ruby/set", - find: "ruby/find", - rinda: "ruby/rinda", - erb: "ruby/erb", - nkf: "ruby/nkf", - tsort: "ruby/tsort", - abbrev: "ruby/abbrev", - shellwords: "ruby/shellwords", - base64: "ruby/base64", - syslog: "ruby/syslog", - "open-uri": "ruby/open-uri", - securerandom: "ruby/securerandom", - resolv: "ruby/resolv", - "resolv-replace": "ruby/resolv-replace", - time: "ruby/time", - pp: "ruby/pp", - prettyprint: "ruby/prettyprint", - drb: "ruby/drb", - pathname: "ruby/pathname", - digest: "ruby/digest", - error_highlight: "ruby/error_highlight", - un: "ruby/un", - win32ole: "ruby/win32ole", -} - -def pipe_readlines(args, rs: "\0", chomp: true) - IO.popen(args) do |f| - f.readlines(rs, chomp: chomp) +require "rbconfig" + +module SyncDefaultGems + include FileUtils + extend FileUtils + + module_function + + REPOSITORIES = { + "io-console": 'ruby/io-console', + "io-nonblock": 'ruby/io-nonblock', + "io-wait": 'ruby/io-wait', + "net-http": "ruby/net-http", + "net-protocol": "ruby/net-protocol", + "open-uri": "ruby/open-uri", + English: "ruby/English", + benchmark: "ruby/benchmark", + cgi: "ruby/cgi", + date: 'ruby/date', + delegate: "ruby/delegate", + did_you_mean: "ruby/did_you_mean", + digest: "ruby/digest", + erb: "ruby/erb", + error_highlight: "ruby/error_highlight", + etc: 'ruby/etc', + fcntl: 'ruby/fcntl', + fiddle: 'ruby/fiddle', + fileutils: 'ruby/fileutils', + find: "ruby/find", + forwardable: "ruby/forwardable", + ipaddr: 'ruby/ipaddr', + irb: 'ruby/irb', + json: 'flori/json', + logger: 'ruby/logger', + open3: "ruby/open3", + openssl: "ruby/openssl", + optparse: "ruby/optparse", + ostruct: 'ruby/ostruct', + pathname: "ruby/pathname", + pp: "ruby/pp", + prettyprint: "ruby/prettyprint", + prism: ["ruby/prism", "main"], + pstore: "ruby/pstore", + psych: 'ruby/psych', + rdoc: 'ruby/rdoc', + readline: "ruby/readline", + reline: 'ruby/reline', + resolv: "ruby/resolv", + rubygems: 'rubygems/rubygems', + securerandom: "ruby/securerandom", + set: "ruby/set", + shellwords: "ruby/shellwords", + singleton: "ruby/singleton", + stringio: 'ruby/stringio', + strscan: 'ruby/strscan', + syntax_suggest: ["ruby/syntax_suggest", "main"], + tempfile: "ruby/tempfile", + time: "ruby/time", + timeout: "ruby/timeout", + tmpdir: "ruby/tmpdir", + tsort: "ruby/tsort", + un: "ruby/un", + uri: "ruby/uri", + weakref: "ruby/weakref", + win32ole: "ruby/win32ole", + yaml: "ruby/yaml", + zlib: 'ruby/zlib', + }.transform_keys(&:to_s) + + CLASSICAL_DEFAULT_BRANCH = "master" + + class << REPOSITORIES + def [](gem) + repo, branch = super(gem) + return repo, branch || CLASSICAL_DEFAULT_BRANCH + end + + def each_pair + super do |gem, (repo, branch)| + yield gem, [repo, branch || CLASSICAL_DEFAULT_BRANCH] + end + end end -end -# We usually don't use this. Please consider using #sync_default_gems_with_commits instead. -def sync_default_gems(gem) - repo = REPOSITORIES[gem.to_sym] - puts "Sync #{repo}" - - upstream = File.join("..", "..", repo) - - case gem - when "rubygems" - rm_rf(%w[lib/rubygems lib/rubygems.rb test/rubygems]) - cp_r(Dir.glob("#{upstream}/lib/rubygems*"), "lib") - cp_r("#{upstream}/test/rubygems", "test") - rm_rf(%w[lib/bundler lib/bundler.rb libexec/bundler libexec/bundle spec/bundler tool/bundler/*]) - cp_r(Dir.glob("#{upstream}/bundler/lib/bundler*"), "lib") - cp_r(Dir.glob("#{upstream}/bundler/exe/bundle*"), "libexec") - - gemspec_content = File.readlines("#{upstream}/bundler/bundler.gemspec").map do |line| - next if line =~ /LICENSE\.md/ - - line.gsub("bundler.gemspec", "lib/bundler/bundler.gemspec").gsub('"exe"', '"libexec"') - end.compact.join - File.write("lib/bundler/bundler.gemspec", gemspec_content) - - cp_r("#{upstream}/bundler/spec", "spec/bundler") - cp_r(Dir.glob("#{upstream}/bundler/tool/bundler/dev_gems*"), "tool/bundler") - cp_r(Dir.glob("#{upstream}/bundler/tool/bundler/test_gems*"), "tool/bundler") - cp_r(Dir.glob("#{upstream}/bundler/tool/bundler/rubocop_gems*"), "tool/bundler") - cp_r(Dir.glob("#{upstream}/bundler/tool/bundler/standard_gems*"), "tool/bundler") - rm_rf(%w[spec/bundler/support/artifice/vcr_cassettes]) - license_files = %w[ - lib/bundler/vendor/thor/LICENSE.md - lib/rubygems/resolver/molinillo/LICENSE - lib/bundler/vendor/molinillo/LICENSE - lib/bundler/vendor/connection_pool/LICENSE - lib/bundler/vendor/net-http-persistent/README.rdoc - lib/bundler/vendor/fileutils/LICENSE.txt - lib/bundler/vendor/tsort/LICENSE.txt - lib/bundler/vendor/uri/LICENSE.txt - lib/rubygems/optparse/COPYING - lib/rubygems/tsort/LICENSE.txt - ] - rm_rf license_files - when "rdoc" - rm_rf(%w[lib/rdoc lib/rdoc.rb test/rdoc libexec/rdoc libexec/ri]) - cp_r(Dir.glob("#{upstream}/lib/rdoc*"), "lib") - cp_r("#{upstream}/doc/rdoc", "doc") - cp_r("#{upstream}/test/rdoc", "test") - cp_r("#{upstream}/rdoc.gemspec", "lib/rdoc") - cp_r("#{upstream}/Gemfile", "lib/rdoc") - cp_r("#{upstream}/Rakefile", "lib/rdoc") - cp_r("#{upstream}/exe/rdoc", "libexec") - cp_r("#{upstream}/exe/ri", "libexec") - parser_files = { - 'lib/rdoc/markdown.kpeg' => 'lib/rdoc/markdown.rb', - 'lib/rdoc/markdown/literals.kpeg' => 'lib/rdoc/markdown/literals.rb', - 'lib/rdoc/rd/block_parser.ry' => 'lib/rdoc/rd/block_parser.rb', - 'lib/rdoc/rd/inline_parser.ry' => 'lib/rdoc/rd/inline_parser.rb' - } - Dir.chdir(upstream) do - `bundle install` - parser_files.each_value do |dst| - `bundle exec rake #{dst}` + def pipe_readlines(args, rs: "\0", chomp: true) + IO.popen(args) do |f| + f.readlines(rs, chomp: chomp) + end + end + + def replace_rdoc_ref(file) + src = File.binread(file) + changed = false + changed |= src.gsub!(%r[\[\Khttps://docs\.ruby-lang\.org/en/master(?:/doc)?/(([A-Z]\w+(?:/[A-Z]\w+)*)|\w+_rdoc)\.html(\#\S+)?(?=\])]) do + name, mod, label = $1, $2, $3 + mod &&= mod.gsub('/', '::') + if label && (m = label.match(/\A\#(?:method-([ci])|(?:(?:class|module)-#{mod}-)?label)-([-+\w]+)\z/)) + scope, label = m[1], m[2] + scope = scope ? scope.tr('ci', '.#') : '@' end + "rdoc-ref:#{mod || name.chomp("_rdoc") + ".rdoc"}#{scope}#{label}" end - parser_files.each_pair do |src, dst| - rm_rf(src) - cp_r("#{upstream}/#{dst}", dst) - end - `git checkout lib/rdoc/.document` - rm_rf(%w[lib/rdoc/Gemfile lib/rdoc/Rakefile]) - when "reline" - rm_rf(%w[lib/reline lib/reline.rb test/reline]) - cp_r(Dir.glob("#{upstream}/lib/reline*"), "lib") - cp_r("#{upstream}/test/reline", "test") - cp_r("#{upstream}/reline.gemspec", "lib/reline") - when "irb" - rm_rf(%w[lib/irb lib/irb.rb test/irb]) - cp_r(Dir.glob("#{upstream}/lib/irb*"), "lib") - cp_r("#{upstream}/test/irb", "test") - cp_r("#{upstream}/irb.gemspec", "lib/irb") - cp_r("#{upstream}/man/irb.1", "man/irb.1") - cp_r("#{upstream}/doc/irb", "doc") - when "json" - rm_rf(%w[ext/json test/json]) - cp_r("#{upstream}/ext/json/ext", "ext/json") - cp_r("#{upstream}/tests", "test/json") - rm_rf("test/json/lib") - cp_r("#{upstream}/lib", "ext/json") - cp_r("#{upstream}/json.gemspec", "ext/json") - cp_r("#{upstream}/VERSION", "ext/json") - rm_rf(%w[ext/json/lib/json/ext ext/json/lib/json/pure.rb ext/json/lib/json/pure]) - `git checkout ext/json/extconf.rb ext/json/parser/prereq.mk ext/json/generator/depend ext/json/parser/depend ext/json/depend` - when "psych" - rm_rf(%w[ext/psych test/psych]) - cp_r("#{upstream}/ext/psych", "ext") - cp_r("#{upstream}/lib", "ext/psych") - cp_r("#{upstream}/test/psych", "test") - rm_rf(%w[ext/psych/lib/org ext/psych/lib/psych.jar ext/psych/lib/psych_jars.rb]) - rm_rf(%w[ext/psych/lib/psych.{bundle,so} ext/psych/lib/2.*]) - rm_rf(["ext/psych/yaml/LICENSE"]) - cp_r("#{upstream}/psych.gemspec", "ext/psych") - `git checkout ext/psych/depend ext/psych/.gitignore` - when "fiddle" - rm_rf(%w[ext/fiddle test/fiddle]) - cp_r("#{upstream}/ext/fiddle", "ext") - cp_r("#{upstream}/lib", "ext/fiddle") - cp_r("#{upstream}/test/fiddle", "test") - cp_r("#{upstream}/fiddle.gemspec", "ext/fiddle") - `git checkout ext/fiddle/depend` - rm_rf(%w[ext/fiddle/lib/fiddle.{bundle,so}]) - when "stringio" - rm_rf(%w[ext/stringio test/stringio]) - cp_r("#{upstream}/ext/stringio", "ext") - cp_r("#{upstream}/test/stringio", "test") - cp_r("#{upstream}/stringio.gemspec", "ext/stringio") - `git checkout ext/stringio/depend ext/stringio/README.md` - when "io-console" - rm_rf(%w[ext/io/console test/io/console]) - cp_r("#{upstream}/ext/io/console", "ext/io") - cp_r("#{upstream}/test/io/console", "test/io") - mkdir_p("ext/io/console/lib") - cp_r("#{upstream}/lib/io/console", "ext/io/console/lib") - rm_rf("ext/io/console/lib/console/ffi") - cp_r("#{upstream}/io-console.gemspec", "ext/io/console") - `git checkout ext/io/console/depend` - when "io-nonblock" - rm_rf(%w[ext/io/nonblock test/io/nonblock]) - cp_r("#{upstream}/ext/io/nonblock", "ext/io") - cp_r("#{upstream}/test/io/nonblock", "test/io") - cp_r("#{upstream}/io-nonblock.gemspec", "ext/io/nonblock") - `git checkout ext/io/nonblock/depend` - when "io-wait" - rm_rf(%w[ext/io/wait test/io/wait]) - cp_r("#{upstream}/ext/io/wait", "ext/io") - cp_r("#{upstream}/test/io/wait", "test/io") - cp_r("#{upstream}/io-wait.gemspec", "ext/io/wait") - `git checkout ext/io/wait/depend` - when "etc" - rm_rf(%w[ext/etc test/etc]) - cp_r("#{upstream}/ext/etc", "ext") - cp_r("#{upstream}/test/etc", "test") - cp_r("#{upstream}/etc.gemspec", "ext/etc") - `git checkout ext/etc/depend` - when "date" - rm_rf(%w[ext/date test/date]) - cp_r("#{upstream}/doc/date", "doc") - cp_r("#{upstream}/ext/date", "ext") - cp_r("#{upstream}/lib", "ext/date") - cp_r("#{upstream}/test/date", "test") - cp_r("#{upstream}/date.gemspec", "ext/date") - `git checkout ext/date/depend` - rm_rf(["ext/date/lib/date_core.bundle"]) - when "zlib" - rm_rf(%w[ext/zlib test/zlib]) - cp_r("#{upstream}/ext/zlib", "ext") - cp_r("#{upstream}/test/zlib", "test") - cp_r("#{upstream}/zlib.gemspec", "ext/zlib") - `git checkout ext/zlib/depend` - when "fcntl" - rm_rf(%w[ext/fcntl]) - cp_r("#{upstream}/ext/fcntl", "ext") - cp_r("#{upstream}/fcntl.gemspec", "ext/fcntl") - `git checkout ext/fcntl/depend` - when "strscan" - rm_rf(%w[ext/strscan test/strscan]) - cp_r("#{upstream}/ext/strscan", "ext") - cp_r("#{upstream}/test/strscan", "test") - cp_r("#{upstream}/strscan.gemspec", "ext/strscan") - rm_rf(%w["ext/strscan/regenc.h ext/strscan/regint.h"]) - `git checkout ext/strscan/depend` - when "racc" - rm_rf(%w[lib/racc lib/racc.rb ext/racc test/racc]) - cp_r(Dir.glob("#{upstream}/lib/racc*"), "lib") - mkdir_p("ext/racc/cparse") - cp_r(Dir.glob("#{upstream}/ext/racc/cparse/*"), "ext/racc/cparse") - cp_r("#{upstream}/test", "test/racc") - cp_r("#{upstream}/racc.gemspec", "lib/racc") - rm_rf("test/racc/lib") - rm_rf("lib/racc/cparse-jruby.jar") - `git checkout ext/racc/cparse/README ext/racc/cparse/depend` - when "cgi" - rm_rf(%w[lib/cgi.rb lib/cgi ext/cgi test/cgi]) - cp_r("#{upstream}/ext/cgi", "ext") - cp_r("#{upstream}/lib", ".") - cp_r("#{upstream}/test/cgi", "test") - cp_r("#{upstream}/cgi.gemspec", "lib/cgi") - `git checkout ext/cgi/escape/depend` - when "openssl" - rm_rf(%w[ext/openssl test/openssl]) - cp_r("#{upstream}/ext/openssl", "ext") - cp_r("#{upstream}/lib", "ext/openssl") - cp_r("#{upstream}/test/openssl", "test") - rm_rf("test/openssl/envutil.rb") - cp_r("#{upstream}/openssl.gemspec", "ext/openssl") - cp_r("#{upstream}/History.md", "ext/openssl") - `git checkout ext/openssl/depend` - when "net-protocol" - rm_rf(%w[lib/net/protocol.rb lib/net/net-protocol.gemspec test/net/protocol]) - cp_r("#{upstream}/lib/net/protocol.rb", "lib/net") - cp_r("#{upstream}/test/net/protocol", "test/net") - cp_r("#{upstream}/net-protocol.gemspec", "lib/net") - when "net-http" - rm_rf(%w[lib/net/http.rb lib/net/http test/net/http]) - cp_r("#{upstream}/lib/net/http.rb", "lib/net") - cp_r("#{upstream}/lib/net/http", "lib/net") - cp_r("#{upstream}/test/net/http", "test/net") - cp_r("#{upstream}/net-http.gemspec", "lib/net/http") - when "readline-ext" - rm_rf(%w[ext/readline test/readline]) - cp_r("#{upstream}/ext/readline", "ext") - cp_r("#{upstream}/test/readline", "test") - cp_r("#{upstream}/readline-ext.gemspec", "ext/readline") - `git checkout ext/readline/depend` - when "did_you_mean" - rm_rf(%w[lib/did_you_mean lib/did_you_mean.rb test/did_you_mean]) - cp_r(Dir.glob("#{upstream}/lib/did_you_mean*"), "lib") - cp_r("#{upstream}/did_you_mean.gemspec", "lib/did_you_mean") - cp_r("#{upstream}/test", "test/did_you_mean") - rm_rf(%w[test/did_you_mean/tree_spell/test_explore.rb]) - when "erb" - rm_rf(%w[lib/erb* test/erb libexec/erb]) - cp_r("#{upstream}/lib/erb.rb", "lib") - cp_r("#{upstream}/test/erb", "test") - cp_r("#{upstream}/erb.gemspec", "lib") - cp_r("#{upstream}/libexec/erb", "libexec") - when "nkf" - rm_rf(%w[ext/nkf test/nkf]) - cp_r("#{upstream}/ext/nkf", "ext") - cp_r("#{upstream}/lib", "ext/nkf") - cp_r("#{upstream}/test/nkf", "test") - cp_r("#{upstream}/nkf.gemspec", "ext/nkf") - `git checkout ext/nkf/depend` - when "syslog" - rm_rf(%w[ext/syslog test/syslog test/test_syslog.rb]) - cp_r("#{upstream}/ext/syslog", "ext") - cp_r("#{upstream}/lib", "ext/syslog") - cp_r("#{upstream}/test/syslog", "test") - cp_r("#{upstream}/test/test_syslog.rb", "test") - cp_r("#{upstream}/syslog.gemspec", "ext/syslog") - `git checkout ext/syslog/depend` - when "bigdecimal" - rm_rf(%w[ext/bigdecimal test/bigdecimal]) - cp_r("#{upstream}/ext/bigdecimal", "ext") - cp_r("#{upstream}/sample", "ext/bigdecimal") - cp_r("#{upstream}/lib", "ext/bigdecimal") - cp_r("#{upstream}/test/bigdecimal", "test") - cp_r("#{upstream}/bigdecimal.gemspec", "ext/bigdecimal") - `git checkout ext/bigdecimal/depend` - when "pathname" - rm_rf(%w[ext/pathname test/pathname]) - cp_r("#{upstream}/ext/pathname", "ext") - cp_r("#{upstream}/test/pathname", "test") - cp_r("#{upstream}/lib", "ext/pathname") - cp_r("#{upstream}/pathname.gemspec", "ext/pathname") - `git checkout ext/pathname/depend` - when "digest" - rm_rf(%w[ext/digest test/digest]) - cp_r("#{upstream}/ext/digest", "ext") - mkdir_p("ext/digest/lib/digest") - cp_r("#{upstream}/lib/digest.rb", "ext/digest/lib/") - cp_r("#{upstream}/lib/digest/version.rb", "ext/digest/lib/digest/") - mkdir_p("ext/digest/sha2/lib") - cp_r("#{upstream}/lib/digest/sha2.rb", "ext/digest/sha2/lib") - move("ext/digest/lib/digest/sha2", "ext/digest/sha2/lib") - cp_r("#{upstream}/test/digest", "test") - cp_r("#{upstream}/digest.gemspec", "ext/digest") - `git checkout ext/digest/depend ext/digest/*/depend` - when "set" - sync_lib gem, upstream - cp_r("#{upstream}/test", ".") - when "optparse" - sync_lib gem, upstream - rm_rf(%w[doc/optparse]) - mkdir_p("doc/optparse") - cp_r("#{upstream}/doc/optparse", "doc") - when "error_highlight" - rm_rf(%w[lib/error_highlight lib/error_highlight.rb test/error_highlight]) - cp_r(Dir.glob("#{upstream}/lib/error_highlight*"), "lib") - cp_r("#{upstream}/error_highlight.gemspec", "lib/error_highlight") - cp_r("#{upstream}/test", "test/error_highlight") - when "win32ole" - sync_lib gem, upstream - rm_rf(%w[ext/win32ole/lib]) - Dir.mkdir(*%w[ext/win32ole/lib]) - move("lib/win32ole/win32ole.gemspec", "ext/win32ole") - move(Dir.glob("lib/win32ole*"), "ext/win32ole/lib") - when "open3" - sync_lib gem, upstream - rm_rf("lib/open3/jruby_windows.rb") - else - sync_lib gem, upstream + changed or return false + File.binwrite(file, src) + return true end -end -IGNORE_FILE_PATTERN = - /\A(?:[A-Z]\w*\.(?:md|txt) - |[^\/]+\.yml - |\.git.* - |[A-Z]\w+file - |COPYING - |rakelib\/.* - )\z/mx - -def message_filter(repo, sha) - log = STDIN.read - log.delete!("\r") - url = "https://github.com/#{repo}" - print "[#{repo}] ", log.gsub(/\b(?i:fix) +\K#(?=\d+\b)|\(\K#(?=\d+\))|\bGH-(?=\d+\b)/) { - "#{url}/pull/" - }.gsub(%r{(?<![-\[\](){}\w@/])(?:(\w+(?:-\w+)*/\w+(?:-\w+)*)@)?(\h{10,40})\b}) {|c| - "https://github.com/#{$1 || repo}/commit/#{$2[0,12]}" - }.sub(/\s*(?=(?i:\nCo-authored-by:.*)*\Z)/) { - "\n\n" "#{url}/commit/#{sha[0,10]}\n" - } -end + def replace_rdoc_ref_all + result = pipe_readlines(%W"git status --porcelain -z -- *.c *.rb *.rdoc") + result.map! {|line| line[/\A.M (.*)/, 1]} + result.compact! + return if result.empty? + result = pipe_readlines(%W"git grep -z -l -F [https://docs.ruby-lang.org/en/master/ --" + result) + result.inject(false) {|changed, file| changed | replace_rdoc_ref(file)} + end + + # We usually don't use this. Please consider using #sync_default_gems_with_commits instead. + def sync_default_gems(gem) + repo, = REPOSITORIES[gem] + puts "Sync #{repo}" + + upstream = File.join("..", "..", repo) -# NOTE: This method is also used by ruby-commit-hook/bin/update-default-gem.sh -# @param gem [String] A gem name, also used as a git remote name. REPOSITORIES converts it to the appropriate GitHub repository. -# @param ranges [Array<String>] "before..after". Note that it will NOT sync "before" (but commits after that). -# @param edit [TrueClass] Set true if you want to resolve conflicts. Obviously, update-default-gem.sh doesn't use this. -def sync_default_gems_with_commits(gem, ranges, edit: nil) - repo = REPOSITORIES[gem.to_sym] - puts "Sync #{repo} with commit history." + case gem + when "rubygems" + rm_rf(%w[lib/rubygems lib/rubygems.rb test/rubygems]) + cp_r(Dir.glob("#{upstream}/lib/rubygems*"), "lib") + cp_r("#{upstream}/test/rubygems", "test") + rm_rf(%w[lib/bundler lib/bundler.rb libexec/bundler libexec/bundle spec/bundler tool/bundler/*]) + cp_r(Dir.glob("#{upstream}/bundler/lib/bundler*"), "lib") + cp_r(Dir.glob("#{upstream}/bundler/exe/bundle*"), "libexec") - IO.popen(%W"git remote") do |f| - unless f.read.split.include?(gem) - `git remote add #{gem} git@github.com:#{repo}.git` + gemspec_content = File.readlines("#{upstream}/bundler/bundler.gemspec").map do |line| + next if line =~ /LICENSE\.md/ + + line.gsub("bundler.gemspec", "lib/bundler/bundler.gemspec") + end.compact.join + File.write("lib/bundler/bundler.gemspec", gemspec_content) + + cp_r("#{upstream}/bundler/spec", "spec/bundler") + %w[dev_gems test_gems rubocop_gems standard_gems].each do |gemfile| + cp_r("#{upstream}/tool/bundler/#{gemfile}.rb", "tool/bundler") + end + rm_rf Dir.glob("spec/bundler/support/artifice/{vcr_cassettes,used_cassettes.txt}") + rm_rf Dir.glob("lib/{bundler,rubygems}/**/{COPYING,LICENSE,README}{,.{md,txt,rdoc}}") + when "rdoc" + rm_rf(%w[lib/rdoc lib/rdoc.rb test/rdoc libexec/rdoc libexec/ri]) + cp_r(Dir.glob("#{upstream}/lib/rdoc*"), "lib") + cp_r("#{upstream}/doc/rdoc", "doc") + cp_r("#{upstream}/test/rdoc", "test") + cp_r("#{upstream}/rdoc.gemspec", "lib/rdoc") + cp_r("#{upstream}/Gemfile", "lib/rdoc") + cp_r("#{upstream}/Rakefile", "lib/rdoc") + cp_r("#{upstream}/exe/rdoc", "libexec") + cp_r("#{upstream}/exe/ri", "libexec") + parser_files = { + 'lib/rdoc/markdown.kpeg' => 'lib/rdoc/markdown.rb', + 'lib/rdoc/markdown/literals.kpeg' => 'lib/rdoc/markdown/literals.rb', + 'lib/rdoc/rd/block_parser.ry' => 'lib/rdoc/rd/block_parser.rb', + 'lib/rdoc/rd/inline_parser.ry' => 'lib/rdoc/rd/inline_parser.rb' + } + Dir.chdir(upstream) do + `bundle install` + parser_files.each_value do |dst| + `bundle exec rake #{dst}` + end + end + parser_files.each_pair do |src, dst| + rm_rf(src) + cp_r("#{upstream}/#{dst}", dst) + end + `git checkout lib/rdoc/.document` + rm_rf(%w[lib/rdoc/Gemfile lib/rdoc/Rakefile]) + when "reline" + rm_rf(%w[lib/reline lib/reline.rb test/reline]) + cp_r(Dir.glob("#{upstream}/lib/reline*"), "lib") + cp_r("#{upstream}/test/reline", "test") + cp_r("#{upstream}/reline.gemspec", "lib/reline") + when "irb" + rm_rf(%w[lib/irb lib/irb.rb test/irb]) + cp_r(Dir.glob("#{upstream}/lib/irb*"), "lib") + cp_r("#{upstream}/test/irb", "test") + cp_r("#{upstream}/irb.gemspec", "lib/irb") + cp_r("#{upstream}/man/irb.1", "man/irb.1") + cp_r("#{upstream}/doc/irb", "doc") + when "json" + rm_rf(%w[ext/json test/json]) + cp_r("#{upstream}/ext/json/ext", "ext/json") + cp_r("#{upstream}/tests", "test/json") + rm_rf("test/json/lib") + cp_r("#{upstream}/lib", "ext/json") + cp_r("#{upstream}/json.gemspec", "ext/json") + rm_rf(%w[ext/json/lib/json/ext ext/json/lib/json/pure.rb ext/json/lib/json/pure]) + `git checkout ext/json/extconf.rb ext/json/parser/prereq.mk ext/json/generator/depend ext/json/parser/depend ext/json/depend` + when "psych" + rm_rf(%w[ext/psych test/psych]) + cp_r("#{upstream}/ext/psych", "ext") + cp_r("#{upstream}/lib", "ext/psych") + cp_r("#{upstream}/test/psych", "test") + rm_rf(%w[ext/psych/lib/org ext/psych/lib/psych.jar ext/psych/lib/psych_jars.rb]) + rm_rf(%w[ext/psych/lib/psych.{bundle,so} ext/psych/lib/2.*]) + rm_rf(["ext/psych/yaml/LICENSE"]) + cp_r("#{upstream}/psych.gemspec", "ext/psych") + `git checkout ext/psych/depend ext/psych/.gitignore` + when "fiddle" + rm_rf(%w[ext/fiddle test/fiddle]) + cp_r("#{upstream}/ext/fiddle", "ext") + cp_r("#{upstream}/lib", "ext/fiddle") + cp_r("#{upstream}/test/fiddle", "test") + cp_r("#{upstream}/fiddle.gemspec", "ext/fiddle") + `git checkout ext/fiddle/depend` + rm_rf(%w[ext/fiddle/lib/fiddle.{bundle,so}]) + when "stringio" + rm_rf(%w[ext/stringio test/stringio]) + cp_r("#{upstream}/ext/stringio", "ext") + cp_r("#{upstream}/test/stringio", "test") + cp_r("#{upstream}/stringio.gemspec", "ext/stringio") + `git checkout ext/stringio/depend ext/stringio/README.md` + when "io-console" + rm_rf(%w[ext/io/console test/io/console]) + cp_r("#{upstream}/ext/io/console", "ext/io") + cp_r("#{upstream}/test/io/console", "test/io") + mkdir_p("ext/io/console/lib") + cp_r("#{upstream}/lib/io/console", "ext/io/console/lib") + rm_rf("ext/io/console/lib/console/ffi") + cp_r("#{upstream}/io-console.gemspec", "ext/io/console") + `git checkout ext/io/console/depend` + when "io-nonblock" + rm_rf(%w[ext/io/nonblock test/io/nonblock]) + cp_r("#{upstream}/ext/io/nonblock", "ext/io") + cp_r("#{upstream}/test/io/nonblock", "test/io") + cp_r("#{upstream}/io-nonblock.gemspec", "ext/io/nonblock") + `git checkout ext/io/nonblock/depend` + when "io-wait" + rm_rf(%w[ext/io/wait test/io/wait]) + cp_r("#{upstream}/ext/io/wait", "ext/io") + cp_r("#{upstream}/test/io/wait", "test/io") + cp_r("#{upstream}/io-wait.gemspec", "ext/io/wait") + `git checkout ext/io/wait/depend` + when "etc" + rm_rf(%w[ext/etc test/etc]) + cp_r("#{upstream}/ext/etc", "ext") + cp_r("#{upstream}/test/etc", "test") + cp_r("#{upstream}/etc.gemspec", "ext/etc") + `git checkout ext/etc/depend` + when "date" + rm_rf(%w[ext/date test/date]) + cp_r("#{upstream}/doc/date", "doc") + cp_r("#{upstream}/ext/date", "ext") + cp_r("#{upstream}/lib", "ext/date") + cp_r("#{upstream}/test/date", "test") + cp_r("#{upstream}/date.gemspec", "ext/date") + `git checkout ext/date/depend` + rm_rf(["ext/date/lib/date_core.bundle"]) + when "zlib" + rm_rf(%w[ext/zlib test/zlib]) + cp_r("#{upstream}/ext/zlib", "ext") + cp_r("#{upstream}/test/zlib", "test") + cp_r("#{upstream}/zlib.gemspec", "ext/zlib") + `git checkout ext/zlib/depend` + when "fcntl" + rm_rf(%w[ext/fcntl]) + cp_r("#{upstream}/ext/fcntl", "ext") + cp_r("#{upstream}/fcntl.gemspec", "ext/fcntl") + `git checkout ext/fcntl/depend` + when "strscan" + rm_rf(%w[ext/strscan test/strscan]) + cp_r("#{upstream}/ext/strscan", "ext") + cp_r("#{upstream}/test/strscan", "test") + cp_r("#{upstream}/strscan.gemspec", "ext/strscan") + rm_rf(%w["ext/strscan/regenc.h ext/strscan/regint.h"]) + `git checkout ext/strscan/depend` + when "cgi" + rm_rf(%w[lib/cgi.rb lib/cgi ext/cgi test/cgi]) + cp_r("#{upstream}/ext/cgi", "ext") + cp_r("#{upstream}/lib", ".") + rm_rf("lib/cgi/escape.jar") + cp_r("#{upstream}/test/cgi", "test") + cp_r("#{upstream}/cgi.gemspec", "lib/cgi") + `git checkout ext/cgi/escape/depend` + when "openssl" + rm_rf(%w[ext/openssl test/openssl]) + cp_r("#{upstream}/ext/openssl", "ext") + cp_r("#{upstream}/lib", "ext/openssl") + cp_r("#{upstream}/test/openssl", "test") + rm_rf("test/openssl/envutil.rb") + cp_r("#{upstream}/openssl.gemspec", "ext/openssl") + cp_r("#{upstream}/History.md", "ext/openssl") + `git checkout ext/openssl/depend` + when "net-protocol" + rm_rf(%w[lib/net/protocol.rb lib/net/net-protocol.gemspec test/net/protocol]) + cp_r("#{upstream}/lib/net/protocol.rb", "lib/net") + cp_r("#{upstream}/test/net/protocol", "test/net") + cp_r("#{upstream}/net-protocol.gemspec", "lib/net") + when "net-http" + rm_rf(%w[lib/net/http.rb lib/net/http test/net/http]) + cp_r("#{upstream}/lib/net/http.rb", "lib/net") + cp_r("#{upstream}/lib/net/http", "lib/net") + cp_r("#{upstream}/test/net/http", "test/net") + cp_r("#{upstream}/net-http.gemspec", "lib/net/http") + when "did_you_mean" + rm_rf(%w[lib/did_you_mean lib/did_you_mean.rb test/did_you_mean]) + cp_r(Dir.glob("#{upstream}/lib/did_you_mean*"), "lib") + cp_r("#{upstream}/did_you_mean.gemspec", "lib/did_you_mean") + cp_r("#{upstream}/test", "test/did_you_mean") + rm_rf("test/did_you_mean/lib") + rm_rf(%w[test/did_you_mean/tree_spell/test_explore.rb]) + when "erb" + rm_rf(%w[lib/erb* test/erb libexec/erb]) + cp_r("#{upstream}/lib/erb.rb", "lib") + cp_r("#{upstream}/test/erb", "test") + cp_r("#{upstream}/erb.gemspec", "lib") + cp_r("#{upstream}/libexec/erb", "libexec") + when "pathname" + rm_rf(%w[ext/pathname test/pathname]) + cp_r("#{upstream}/ext/pathname", "ext") + cp_r("#{upstream}/test/pathname", "test") + cp_r("#{upstream}/lib", "ext/pathname") + cp_r("#{upstream}/pathname.gemspec", "ext/pathname") + `git checkout ext/pathname/depend` + when "digest" + rm_rf(%w[ext/digest test/digest]) + cp_r("#{upstream}/ext/digest", "ext") + mkdir_p("ext/digest/lib/digest") + cp_r("#{upstream}/lib/digest.rb", "ext/digest/lib/") + cp_r("#{upstream}/lib/digest/version.rb", "ext/digest/lib/digest/") + mkdir_p("ext/digest/sha2/lib") + cp_r("#{upstream}/lib/digest/sha2.rb", "ext/digest/sha2/lib") + move("ext/digest/lib/digest/sha2", "ext/digest/sha2/lib") + cp_r("#{upstream}/test/digest", "test") + cp_r("#{upstream}/digest.gemspec", "ext/digest") + `git checkout ext/digest/depend ext/digest/*/depend` + when "set" + sync_lib gem, upstream + cp_r(Dir.glob("#{upstream}/test/*"), "test/set") + when "optparse" + sync_lib gem, upstream + rm_rf(%w[doc/optparse]) + mkdir_p("doc/optparse") + cp_r("#{upstream}/doc/optparse", "doc") + when "error_highlight" + rm_rf(%w[lib/error_highlight lib/error_highlight.rb test/error_highlight]) + cp_r(Dir.glob("#{upstream}/lib/error_highlight*"), "lib") + cp_r("#{upstream}/error_highlight.gemspec", "lib/error_highlight") + cp_r("#{upstream}/test", "test/error_highlight") + when "win32ole" + sync_lib gem, upstream + rm_rf(%w[ext/win32ole/lib]) + Dir.mkdir(*%w[ext/win32ole/lib]) + move("lib/win32ole/win32ole.gemspec", "ext/win32ole") + move(Dir.glob("lib/win32ole*"), "ext/win32ole/lib") + when "open3" + sync_lib gem, upstream + rm_rf("lib/open3/jruby_windows.rb") + when "syntax_suggest" + sync_lib gem, upstream + rm_rf(%w[spec/syntax_suggest libexec/syntax_suggest]) + cp_r("#{upstream}/spec", "spec/syntax_suggest") + cp_r("#{upstream}/exe/syntax_suggest", "libexec/syntax_suggest") + when "prism" + rm_rf(%w[test/prism prism]) + + cp_r("#{upstream}/ext/prism", "prism") + cp_r("#{upstream}/lib/.", "lib") + cp_r("#{upstream}/test/prism", "test") + cp_r("#{upstream}/src/.", "prism") + + cp_r("#{upstream}/prism.gemspec", "lib/prism") + cp_r("#{upstream}/include/prism/.", "prism") + cp_r("#{upstream}/include/prism.h", "prism") + + cp_r("#{upstream}/config.yml", "prism/") + cp_r("#{upstream}/templates", "prism/") + rm_rf("prism/templates/javascript") + rm_rf("prism/templates/java") + rm_rf("prism/templates/rbi") + rm_rf("prism/templates/sig") + + rm("prism/extconf.rb") + else + sync_lib gem, upstream end - end - system(*%W"git fetch --no-tags #{gem}") - if ranges == true - pattern = "https://github\.com/#{Regexp.quote(repo)}/commit/([0-9a-f]+)$" - log = IO.popen(%W"git log -E --grep=#{pattern} -n1 --format=%B", &:read) - ranges = ["#{log[%r[#{pattern}\n\s*(?i:co-authored-by:.*)*\s*\Z], 1]}..#{gem}/master"] + # Architecture-dependent files must not pollute libdir. + rm_rf(Dir["lib/**/*.#{RbConfig::CONFIG['DLEXT']}"]) + replace_rdoc_ref_all end - commits = ranges.flat_map do |range| - unless range.include?("..") - range = "#{range}~1..#{range}" + def ignore_file_pattern_for(gem) + patterns = [] + + # Common patterns + patterns << %r[\A(?: + [^/]+ # top-level entries + |\.git.* + |bin/.* + |ext/.*\.java + |rakelib/.* + |test/(?:lib|fixtures)/.* + |tool/(?!bundler/).* + )\z]mx + + # Gem-specific patterns + case gem + when nil + end&.tap do |pattern| + patterns << pattern end - IO.popen(%W"git log --format=%H,%s #{range} --") do |f| - f.read.split("\n").reverse.map{|commit| commit.split(',', 2)} + Regexp.union(*patterns) + end + + def message_filter(repo, sha, input: ARGF) + log = input.read + log.delete!("\r") + log << "\n" if !log.end_with?("\n") + repo_url = "https://github.com/#{repo}" + + # Split the subject from the log message according to git conventions. + # SPECIAL TREAT: when the first line ends with a dot `.` (which is not + # obeying the conventions too), takes only that line. + subject, log = log.split(/\A.+\.\K\n(?=\S)|\n(?:[ \t]*(?:\n|\z))/, 2) + conv = proc do |s| + mod = true if s.gsub!(/\b(?:(?i:fix(?:e[sd])?|close[sd]?|resolve[sd]?) +)\K#(?=\d+\b)|\bGH-#?(?=\d+\b)|\(\K#(?=\d+\))/) { + "#{repo_url}/pull/" + } + mod |= true if s.gsub!(%r{(?<![-\[\](){}\w@/])(?:(\w+(?:-\w+)*/\w+(?:-\w+)*)@)?(\h{10,40})\b}) {|c| + "https://github.com/#{$1 || repo}/commit/#{$2[0,12]}" + } + mod + end + subject = "[#{repo}] #{subject}" + subject.gsub!(/\s*\n\s*/, " ") + if conv[subject] + if subject.size > 68 + subject.gsub!(/\G.{,67}[^\s.,][.,]*\K\s+/, "\n") + end end + commit_url = "#{repo_url}/commit/#{sha[0,10]}\n" + if log and !log.empty? + log.sub!(/(?<=\n)\n+\z/, '') # drop empty lines at the last + conv[log] + log.sub!(/(?:(\A\s*)|\s*\n)(?=((?i:^Co-authored-by:.*\n?)+)?\Z)/) { + ($~.begin(1) ? "" : "\n\n") + commit_url + ($~.begin(2) ? "\n" : "") + } + else + log = commit_url + end + puts subject, "\n", log end - # Ignore Merge commit and insufficiency commit for ruby core repository. - commits.delete_if do |sha, subject| - files = pipe_readlines(%W"git diff-tree -z --no-commit-id --name-only -r #{sha}") - subject.start_with?("Merge", "Auto Merge") or files.all?(IGNORE_FILE_PATTERN) + # Returns commit list as array of [commit_hash, subject]. + def commits_in_ranges(gem, repo, default_branch, ranges) + # If -a is given, discover all commits since the last picked commit + if ranges == true + # \r? needed in the regex in case the commit has windows-style line endings (because e.g. we're running + # tests on Windows) + pattern = "https://github\.com/#{Regexp.quote(repo)}/commit/([0-9a-f]+)\r?$" + log = IO.popen(%W"git log -E --grep=#{pattern} -n1 --format=%B", "rb", &:read) + ranges = ["#{log[%r[#{pattern}\n\s*(?i:co-authored-by:.*)*\s*\Z], 1]}..#{gem}/#{default_branch}"] + end + + # Parse a given range with git log + ranges.flat_map do |range| + unless range.include?("..") + range = "#{range}~1..#{range}" + end + + IO.popen(%W"git log --format=%H,%s #{range} --", "rb") do |f| + f.read.split("\n").reverse.map{|commit| commit.split(',', 2)} + end + end end - if commits.empty? - puts "No commits to pick" + #-- + # Following methods used by sync_default_gems_with_commits return + # true: success + # false: skipped + # nil: failed + #++ + + def resolve_conflicts(gem, sha, edit) + # Skip this commit if everything has been removed as `ignored_paths`. + changes = pipe_readlines(%W"git status --porcelain -z") + if changes.empty? + puts "Skip empty commit #{sha}" + return false + end + + # We want to skip DD: deleted by both. + deleted = changes.grep(/^DD /) {$'} + system(*%W"git rm -f --", *deleted) unless deleted.empty? + + # Import UA: added by them + added = changes.grep(/^UA /) {$'} + system(*%W"git add --", *added) unless added.empty? + + # Discover unmerged files + # AU: unmerged, added by us + # DU: unmerged, deleted by us + # UU: unmerged, both modified + # AA: unmerged, both added + conflict = changes.grep(/\A(?:.U|AA) /) {$'} + # If -e option is given, open each conflicted file with an editor + unless conflict.empty? + if edit + case + when (editor = ENV["GIT_EDITOR"] and !editor.empty?) + when (editor = `git config core.editor` and (editor.chomp!; !editor.empty?)) + end + if editor + system([editor, conflict].join(' ')) + conflict.delete_if {|f| !File.exist?(f)} + return true if conflict.empty? + return system(*%w"git add --", *conflict) + end + end + return false + end + return true end - puts "Try to pick these commits:" - puts commits.map{|commit| commit.join(": ")} - puts "----" + def preexisting?(base, file) + system(*%w"git cat-file -e", "#{base}:#{file}", err: File::NULL) + end + + def filter_pickup_files(changed, ignore_file_pattern, base) + toplevels = {} + remove = [] + ignore = [] + changed = changed.reject do |f| + case + when toplevels.fetch(top = f[%r[\A[^/]+(?=/|\z)]m]) { + remove << top if toplevels[top] = !preexisting?(base, top) + } + # Remove any new top-level directories. + true + when ignore_file_pattern.match?(f) + # Forcibly reset any changes matching ignore_file_pattern. + (preexisting?(base, f) ? ignore : remove) << f + end + end + return changed, remove, ignore + end + + def pickup_files(gem, changed, picked) + # Forcibly remove any files that we don't want to copy to this + # repository. - failed_commits = [] + ignore_file_pattern = ignore_file_pattern_for(gem) - ENV["FILTER_BRANCH_SQUELCH_WARNING"] = "1" + base = picked ? "HEAD~" : "HEAD" + changed, remove, ignore = filter_pickup_files(changed, ignore_file_pattern, base) - require 'shellwords' - filter = [ - ENV.fetch('RUBY', 'ruby').shellescape, - File.realpath(__FILE__).shellescape, - "--message-filter", - ] - commits.each do |sha, subject| - puts "Pick #{sha} from #{repo}." + unless remove.empty? + puts "Remove added files: #{remove.join(', ')}" + system(*%w"git rm -fr --", *remove) + if picked + system(*%w"git commit --amend --no-edit --", *remove, %i[out err] => File::NULL) + end + end + + unless ignore.empty? + puts "Reset ignored files: #{ignore.join(', ')}" + system(*%W"git rm -r --", *ignore) + ignore.each {|f| system(*%W"git checkout -f", base, "--", f)} + end - skipped = false - result = IO.popen(%W"git cherry-pick #{sha}", &:read) + if changed.empty? + return nil + end + + return changed + end + + def pickup_commit(gem, sha, edit) + # Attempt to cherry-pick a commit + result = IO.popen(%W"git cherry-pick #{sha}", "rb", &:read) + picked = $?.success? if result =~ /nothing\ to\ commit/ `git reset` - skipped = true puts "Skip empty commit #{sha}" + return false end - next if skipped + # Skip empty commits if result.empty? - skipped = true - elsif /^CONFLICT/ =~ result - result = pipe_readlines(%W"git status --porcelain -z") - result.map! {|line| line[/^.U (.*)/, 1]} - result.compact! - ignore, conflict = result.partition {|name| IGNORE_FILE_PATTERN =~ name} - unless ignore.empty? - system(*%W"git reset HEAD --", *ignore) - File.unlink(*ignore) - ignore = pipe_readlines(%W"git status --porcelain -z" + ignore).map! {|line| line[/^.. (.*)/, 1]} - system(*%W"git checkout HEAD --", *ignore) unless ignore.empty? - end - unless conflict.empty? - if edit - case - when (editor = ENV["GIT_EDITOR"] and !editor.empty?) - when (editor = `git config core.editor` and (editor.chomp!; !editor.empty?)) - end - if editor - system([editor, conflict].join(' ')) - end - end + return false + end + + if picked + changed = pipe_readlines(%w"git diff-tree --name-only -r -z HEAD~..HEAD --") + else + changed = pipe_readlines(%w"git diff --name-only -r -z HEAD --") + end + + # Pick up files to merge. + unless changed = pickup_files(gem, changed, picked) + puts "Skip commit #{sha} only for tools or toplevel" + if picked + `git reset --hard HEAD~` + else + `git cherry-pick --abort` end - skipped = !system({"GIT_EDITOR"=>"true"}, *%W"git cherry-pick --no-edit --continue") + return false end - if skipped - failed_commits << sha + # If the cherry-pick attempt failed, try to resolve conflicts. + # Skip the commit, if it contains unresolved conflicts or no files to pick up. + unless picked or resolve_conflicts(gem, sha, edit) `git reset` && `git checkout .` && `git clean -fd` - puts "Failed to pick #{sha}" - next + return picked || nil # Fail unless cherry-picked end - puts "Update commit message: #{sha}" - - IO.popen(%W[git filter-branch -f --msg-filter #{[filter, repo, sha].join(' ')} -- HEAD~1..HEAD], &:read) - unless $?.success? - puts "Failed to modify commit message of #{sha}" - break + # Commit cherry-picked commit + if picked + system(*%w"git commit --amend --no-edit") + else + system(*%w"git cherry-pick --continue --no-edit") + end or return nil + + # Amend the commit if RDoc references need to be replaced + head = `git log --format=%H -1 HEAD`.chomp + system(*%w"git reset --quiet HEAD~ --") + amend = replace_rdoc_ref_all + system(*%W"git reset --quiet #{head} --") + if amend + `git commit --amend --no-edit --all` end - end - unless failed_commits.empty? - puts "---- failed commits ----" - puts failed_commits - return false + return true end - return true -end -def sync_lib(repo, upstream = nil) - unless upstream and File.directory?(upstream) or File.directory?(upstream = "../#{repo}") - abort %[Expected '#{upstream}' \(#{File.expand_path("#{upstream}")}\) to be a directory, but it wasn't.] - end - rm_rf(["lib/#{repo}.rb", "lib/#{repo}/*", "test/test_#{repo}.rb"]) - cp_r(Dir.glob("#{upstream}/lib/*"), "lib") - tests = if File.directory?("test/#{repo}") - "test/#{repo}" - else - "test/test_#{repo}.rb" - end - cp_r("#{upstream}/#{tests}", "test") if File.exist?("#{upstream}/#{tests}") - gemspec = if File.directory?("lib/#{repo}") - "lib/#{repo}/#{repo}.gemspec" - else - "lib/#{repo}.gemspec" - end - cp_r("#{upstream}/#{repo}.gemspec", "#{gemspec}") -end + # NOTE: This method is also used by GitHub ruby/git.ruby-lang.org's bin/update-default-gem.sh + # @param gem [String] A gem name, also used as a git remote name. REPOSITORIES converts it to the appropriate GitHub repository. + # @param ranges [Array<String>] "before..after". Note that it will NOT sync "before" (but commits after that). + # @param edit [TrueClass] Set true if you want to resolve conflicts. Obviously, update-default-gem.sh doesn't use this. + def sync_default_gems_with_commits(gem, ranges, edit: nil) + repo, default_branch = REPOSITORIES[gem] + puts "Sync #{repo} with commit history." + + # Fetch the repository to be synchronized + IO.popen(%W"git remote") do |f| + unless f.read.split.include?(gem) + `git remote add #{gem} https://github.com/#{repo}.git` + end + end + system(*%W"git fetch --no-tags #{gem}") -def update_default_gems(gem, release: false) + commits = commits_in_ranges(gem, repo, default_branch, ranges) - author, repository = REPOSITORIES[gem.to_sym].split('/') + # Ignore Merge commits and already-merged commits. + commits.delete_if do |sha, subject| + subject.start_with?("Merge", "Auto Merge") + end - puts "Update #{author}/#{repository}" + if commits.empty? + puts "No commits to pick" + return true + end - unless File.exist?("../../#{author}/#{repository}") - mkdir_p("../../#{author}") - `git clone git@github.com:#{author}/#{repository}.git ../../#{author}/#{repository}` - end + puts "Try to pick these commits:" + puts commits.map{|commit| commit.join(": ")} + puts "----" - Dir.chdir("../../#{author}/#{repository}") do - unless `git remote`.match(/ruby\-core/) - `git remote add ruby-core git@github.com:ruby/ruby.git` - end - `git fetch ruby-core master --no-tags` - unless `git branch`.match(/ruby\-core/) - `git checkout ruby-core/master` - `git branch ruby-core` + failed_commits = [] + + require 'shellwords' + filter = [ + ENV.fetch('RUBY', 'ruby').shellescape, + File.realpath(__FILE__).shellescape, + "--message-filter", + ] + commits.each do |sha, subject| + puts "Pick #{sha} from #{repo}." + case pickup_commit(gem, sha, edit) + when false + next + when nil + failed_commits << sha + next + end + + puts "Update commit message: #{sha}" + + # Run this script itself (tool/sync_default_gems.rb --message-filter) as a message filter + IO.popen({"FILTER_BRANCH_SQUELCH_WARNING" => "1"}, + %W[git filter-branch -f --msg-filter #{[filter, repo, sha].join(' ')} -- HEAD~1..HEAD], + &:read) + unless $?.success? + puts "Failed to modify commit message of #{sha}" + break + end end - `git checkout ruby-core` - `git rebase ruby-core/master` - `git fetch origin --tags` - if release - last_release = `git tag`.chomp.split.delete_if{|v| v =~ /pre|beta/ }.last - `git checkout #{last_release}` - else - `git checkout master` - `git rebase origin/master` + unless failed_commits.empty? + puts "---- failed commits ----" + puts failed_commits + return false end + return true end -end -case ARGV[0] -when "up" - if ARGV[1] - update_default_gems(ARGV[1]) - else - REPOSITORIES.keys.each{|gem| update_default_gems(gem.to_s)} - end -when "all" - if ARGV[1] == "release" - REPOSITORIES.keys.each do |gem| - update_default_gems(gem.to_s, release: true) - sync_default_gems(gem.to_s) + def sync_lib(repo, upstream = nil) + unless upstream and File.directory?(upstream) or File.directory?(upstream = "../#{repo}") + abort %[Expected '#{upstream}' \(#{File.expand_path("#{upstream}")}\) to be a directory, but it wasn't.] end - else - REPOSITORIES.keys.each{|gem| sync_default_gems(gem.to_s)} + rm_rf(["lib/#{repo}.rb", "lib/#{repo}/*", "test/test_#{repo}.rb"]) + cp_r(Dir.glob("#{upstream}/lib/*"), "lib") + tests = if File.directory?("test/#{repo}") + "test/#{repo}" + else + "test/test_#{repo}.rb" + end + cp_r("#{upstream}/#{tests}", "test") if File.exist?("#{upstream}/#{tests}") + gemspec = if File.directory?("lib/#{repo}") + "lib/#{repo}/#{repo}.gemspec" + else + "lib/#{repo}.gemspec" + end + cp_r("#{upstream}/#{repo}.gemspec", "#{gemspec}") end -when "list" - ARGV.shift - pattern = Regexp.new(ARGV.join('|')) - REPOSITORIES.each_pair do |name, gem| - next unless pattern =~ name or pattern =~ gem - printf "%-15s https://github.com/%s\n", name, gem + + def update_default_gems(gem, release: false) + + repository, default_branch = REPOSITORIES[gem] + author, repository = repository.split('/') + + puts "Update #{author}/#{repository}" + + unless File.exist?("../../#{author}/#{repository}") + mkdir_p("../../#{author}") + `git clone git@github.com:#{author}/#{repository}.git ../../#{author}/#{repository}` + end + + Dir.chdir("../../#{author}/#{repository}") do + unless `git remote`.match(/ruby\-core/) + `git remote add ruby-core git@github.com:ruby/ruby.git` + end + `git fetch ruby-core master --no-tags` + unless `git branch`.match(/ruby\-core/) + `git checkout ruby-core/master` + `git branch ruby-core` + end + `git checkout ruby-core` + `git rebase ruby-core/master` + `git fetch origin --tags` + + if release + last_release = `git tag | sort -V`.chomp.split.delete_if{|v| v =~ /pre|beta/ }.last + `git checkout #{last_release}` + else + `git checkout #{default_branch}` + `git rebase origin/#{default_branch}` + end + end end -when "--message-filter" - ARGV.shift - abort unless ARGV.size == 2 - message_filter(*ARGV) - exit -when nil, "-h", "--help" + + case ARGV[0] + when "up" + if ARGV[1] + update_default_gems(ARGV[1]) + else + REPOSITORIES.each_key {|gem| update_default_gems(gem)} + end + when "all" + if ARGV[1] == "release" + REPOSITORIES.each_key do |gem| + update_default_gems(gem, release: true) + sync_default_gems(gem) + end + else + REPOSITORIES.each_key {|gem| sync_default_gems(gem)} + end + when "list" + ARGV.shift + pattern = Regexp.new(ARGV.join('|')) + REPOSITORIES.each_pair do |name, (gem)| + next unless pattern =~ name or pattern =~ gem + printf "%-15s https://github.com/%s\n", name, gem + end + when "--message-filter" + ARGV.shift + if ARGV.size < 2 + abort "usage: #{$0} --message-filter repository commit-hash [input...]" + end + message_filter(*ARGV.shift(2)) + exit + when "rdoc-ref" + ARGV.shift + pattern = ARGV.empty? ? %w[*.c *.rb *.rdoc] : ARGV + result = pipe_readlines(%W"git grep -z -l -F [https://docs.ruby-lang.org/en/master/ --" + pattern) + result.inject(false) do |changed, file| + if replace_rdoc_ref(file) + puts "replaced rdoc-ref in #{file}" + changed = true + end + changed + end + when nil, "-h", "--help" puts <<-HELP \e[1mSync with upstream code of default libraries\e[0m @@ -613,6 +816,9 @@ when nil, "-h", "--help" \e[1mPick a commit range from the upstream repository\e[0m ruby #$0 rubygems 97e9768612..9e53702832 +\e[1mPick all commits since the last picked commit\e[0m + ruby #$0 -a rubygems + \e[1mList known libraries\e[0m ruby #$0 list @@ -620,27 +826,28 @@ when nil, "-h", "--help" ruby #$0 list read HELP - exit -else - while /\A-/ =~ ARGV[0] - case ARGV[0] - when "-e" - edit = true - ARGV.shift - when "-a" - auto = true - ARGV.shift + exit + else + while /\A-/ =~ ARGV[0] + case ARGV[0] + when "-e" + edit = true + ARGV.shift + when "-a" + auto = true + ARGV.shift + else + $stderr.puts "Unknown command line option: #{ARGV[0]}" + exit 1 + end + end + gem = ARGV.shift + if ARGV[0] + exit sync_default_gems_with_commits(gem, ARGV, edit: edit) + elsif auto + exit sync_default_gems_with_commits(gem, true, edit: edit) else - $stderr.puts "Unknown command line option: #{ARGV[0]}" - exit 1 + sync_default_gems(gem) end - end - gem = ARGV.shift - if ARGV[0] - exit sync_default_gems_with_commits(gem, ARGV, edit: edit) - elsif auto - exit sync_default_gems_with_commits(gem, true, edit: edit) - else - sync_default_gems(gem) - end + end if $0 == __FILE__ end diff --git a/tool/test-bundled-gems.rb b/tool/test-bundled-gems.rb index 12358b69cc..eeec1d7aed 100644 --- a/tool/test-bundled-gems.rb +++ b/tool/test-bundled-gems.rb @@ -1,6 +1,7 @@ require 'rbconfig' require 'timeout' require 'fileutils' +require_relative 'lib/colorize' ENV.delete("GNUMAKEFLAGS") @@ -11,10 +12,10 @@ allowed_failures = allowed_failures.split(',').reject(&:empty?) ENV["GEM_PATH"] = [File.realpath('.bundle'), File.realpath('../.bundle', __dir__)].join(File::PATH_SEPARATOR) +colorize = Colorize.new rake = File.realpath("../../.bundle/bin/rake", __FILE__) gem_dir = File.realpath('../../gems', __FILE__) -dummy_rake_compiler_dir = File.realpath('../dummy-rake-compiler', __FILE__) -rubylib = [File.expand_path(dummy_rake_compiler_dir), ENV["RUBYLIB"]].compact.join(File::PATH_SEPARATOR) +rubylib = [gem_dir+'/lib', ENV["RUBYLIB"]].compact.join(File::PATH_SEPARATOR) exit_code = 0 ruby = ENV['RUBY'] || RbConfig.ruby failed = [] @@ -22,30 +23,41 @@ File.foreach("#{gem_dir}/bundled_gems") do |line| next if /^\s*(?:#|$)/ =~ line gem = line.split.first next if ARGV.any? {|pat| !File.fnmatch?(pat, gem)} - puts "#{github_actions ? "##[group]" : "\n"}Testing the #{gem} gem" + # 93(bright yellow) is copied from .github/workflows/mingw.yml + puts "#{github_actions ? "::group::\e\[93m" : "\n"}Testing the #{gem} gem#{github_actions ? "\e\[m" : ""}" test_command = "#{ruby} -C #{gem_dir}/src/#{gem} #{rake} test" first_timeout = 600 # 10min toplib = gem case gem + when "resolv-replace" + # Skip test suite + next when "typeprof" when "rbs" - test_command << " stdlib_test validate" - first_timeout *= 3 + # TODO: We should skip test file instead of test class/methods + skip_test_files = %w[ + ] - when "minitest" - # Tentatively exclude some tests that conflict with error_highlight - # https://github.com/seattlerb/minitest/pull/880 - test_command << " 'TESTOPTS=-e /test_stub_value_block_args_5__break_if_not_passed|test_no_method_error_on_unexpected_methods/'" + skip_test_files.each do |file| + path = "#{gem_dir}/src/#{gem}/#{file}" + File.unlink(path) if File.exist?(path) + end + + test_command << " stdlib_test validate RBS_SKIP_TESTS=#{__dir__}/rbs_skip_tests SKIP_RBS_VALIDATION=true" + first_timeout *= 3 when "debug" # Since debug gem requires debug.so in child processes without - # acitvating the gem, we preset necessary paths in RUBYLIB + # activating the gem, we preset necessary paths in RUBYLIB # environment variable. load_path = true + when "test-unit" + test_command = "#{ruby} -C #{gem_dir}/src/#{gem} test/run-test.rb" + when /\Anet-/ toplib = gem.tr("-", "/") @@ -79,19 +91,21 @@ File.foreach("#{gem_dir}/bundled_gems") do |line| break end + print "::endgroup::\n" if github_actions unless $?.success? - puts "Tests failed " + - ($?.signaled? ? "by SIG#{Signal.signame($?.termsig)}" : - "with exit code #{$?.exitstatus}") + mesg = "Tests failed " + + ($?.signaled? ? "by SIG#{Signal.signame($?.termsig)}" : + "with exit code #{$?.exitstatus}") + puts colorize.decorate(mesg, "fail") if allowed_failures.include?(gem) - puts "Ignoring test failures for #{gem} due to \$TEST_BUNDLED_GEMS_ALLOW_FAILURES" + mesg = "Ignoring test failures for #{gem} due to \$TEST_BUNDLED_GEMS_ALLOW_FAILURES" + puts colorize.decorate(mesg, "skip") else failed << gem exit_code = $?.exitstatus if $?.exitstatus end end - print "##[endgroup]\n" if github_actions end puts "Failed gems: #{failed.join(', ')}" unless failed.empty? diff --git a/tool/test-coverage.rb b/tool/test-coverage.rb index 4950bc65de..055577feea 100644 --- a/tool/test-coverage.rb +++ b/tool/test-coverage.rb @@ -4,6 +4,16 @@ Coverage.start(lines: true, branches: true, methods: true) TEST_COVERAGE_DATA_FILE = "test-coverage.dat" +FILTER_PATHS = %w[ + lib/bundler/vendor + lib/rubygems/resolver/molinillo + lib/rubygems/tsort + lib/rubygems/optparse + tool + test + spec +] + def merge_coverage_data(res1, res2) res1.each do |path, cov1| cov2 = res2[path] @@ -62,8 +72,11 @@ def save_coverage_data(res1) end def invoke_simplecov_formatter - %w[doclie simplecov-html simplecov].each do |f| - $LOAD_PATH.unshift "#{__dir__}/../coverage/#{f}/lib" + # XXX docile-x.y.z and simplecov-x.y.z, simplecov-html-x.y.z, simplecov_json_formatter-x.y.z + %w[simplecov simplecov-html simplecov_json_formatter docile].each do |f| + Dir.glob("#{__dir__}/../.bundle/gems/#{f}-*/lib").each do |d| + $LOAD_PATH.unshift d + end end require "simplecov" @@ -74,8 +87,8 @@ def invoke_simplecov_formatter res.each do |path, cov| next unless path.start_with?(base_dir) || path.start_with?(cur_dir) - next if path.start_with?(File.join(base_dir, "test")) - simplecov_result[path] = cov[:lines] + next if FILTER_PATHS.any? {|dir| path.start_with?(File.join(base_dir, dir))} + simplecov_result[path] = cov end a, b = base_dir, cur_dir diff --git a/tool/test/init.rb b/tool/test/init.rb new file mode 100644 index 0000000000..3a1143d01d --- /dev/null +++ b/tool/test/init.rb @@ -0,0 +1,18 @@ +# This file includes the settings for "make test-all". +# Note that this file is loaded not only by test/runner.rb but also by tool/lib/test/unit/parallel.rb. + +ENV["GEM_SKIP"] = ENV["GEM_HOME"] = ENV["GEM_PATH"] = "".freeze +ENV.delete("RUBY_CODESIGN") + +Warning[:experimental] = false + +$LOAD_PATH.unshift File.expand_path("../lib", __dir__) + +require 'test/unit' + +require "profile_test_all" if ENV.key?('RUBY_TEST_ALL_PROFILE') +require "tracepointchecker" +require "zombie_hunter" +require "iseq_loader_checker" +require "gc_checker" +require_relative "../test-coverage.rb" if ENV.key?('COVERAGE') diff --git a/tool/test/runner.rb b/tool/test/runner.rb index c629943090..9001fc2d06 100644 --- a/tool/test/runner.rb +++ b/tool/test/runner.rb @@ -1,16 +1,7 @@ # frozen_string_literal: true require 'rbconfig' -$LOAD_PATH.unshift File.expand_path("../lib", __dir__) - -require 'test/unit' - -require "profile_test_all" if ENV.key?('RUBY_TEST_ALL_PROFILE') -require "tracepointchecker" -require "zombie_hunter" -require "iseq_loader_checker" -require "gc_checker" -require_relative "../test-coverage.rb" if ENV.key?('COVERAGE') +require_relative "init" case $0 when __FILE__ @@ -18,6 +9,6 @@ when __FILE__ when "-e" # No default directory else - dir = File.expand_path("..", $0) + dir = File.realdirpath("..", $0) end exit Test::Unit::AutoRunner.run(true, dir) diff --git a/tool/test/test_sync_default_gems.rb b/tool/test/test_sync_default_gems.rb new file mode 100755 index 0000000000..e64c6c6fda --- /dev/null +++ b/tool/test/test_sync_default_gems.rb @@ -0,0 +1,297 @@ +#!/usr/bin/ruby +require 'test/unit' +require 'stringio' +require 'tmpdir' +require_relative '../sync_default_gems' + +module Test_SyncDefaultGems + class TestMessageFilter < Test::Unit::TestCase + def assert_message_filter(expected, trailers, input, repo = "ruby/test", sha = "0123456789") + subject, *expected = expected + expected = [ + "[#{repo}] #{subject}\n", + *expected.map {_1+"\n"}, + "\n", + "https://github.com/#{repo}/commit/#{sha[0, 10]}\n", + ] + if trailers + expected << "\n" + expected.concat(trailers.map {_1+"\n"}) + end + + out, err = capture_output do + SyncDefaultGems.message_filter(repo, sha, input: StringIO.new(input, "r")) + end + + all_assertions do |a| + a.for("error") {assert_empty err} + a.for("result") {assert_pattern_list(expected, out)} + end + end + + def test_subject_only + expected = [ + "initial commit", + ] + assert_message_filter(expected, nil, "initial commit") + end + + def test_link_in_parenthesis + expected = [ + "fix (https://github.com/ruby/test/pull/1)", + ] + assert_message_filter(expected, nil, "fix (#1)") + end + + def test_co_authored_by + expected = [ + "commit something", + ] + trailers = [ + "Co-Authored-By: git <git@ruby-lang.org>", + ] + assert_message_filter(expected, trailers, [expected, "", trailers, ""].join("\n")) + end + + def test_multiple_co_authored_by + expected = [ + "many commits", + ] + trailers = [ + "Co-authored-by: git <git@ruby-lang.org>", + "Co-authored-by: svn <svn@ruby-lang.org>", + ] + assert_message_filter(expected, trailers, [expected, "", trailers, ""].join("\n")) + end + + def test_co_authored_by_no_newline + expected = [ + "commit something", + ] + trailers = [ + "Co-Authored-By: git <git@ruby-lang.org>", + ] + assert_message_filter(expected, trailers, [expected, "", trailers].join("\n")) + end + + def test_dot_ending_subject + expected = [ + "subject with a dot.", + "", + "- next body line", + ] + assert_message_filter(expected, nil, [expected[0], expected[2], ""].join("\n")) + end + end + + class TestSyncWithCommits < Test::Unit::TestCase + def setup + super + @target = nil + pend "No git" unless system("git --version", out: IO::NULL) + @testdir = Dir.mktmpdir("sync") + @git_config = %W"HOME GIT_CONFIG_GLOBAL".each_with_object({}) {|k, c| c[k] = ENV[k]} + ENV["HOME"] = @testdir + ENV["GIT_CONFIG_GLOBAL"] = @testdir + "/gitconfig" + git(*%W"config --global user.email test@ruby-lang.org") + git(*%W"config --global user.name", "Ruby") + git(*%W"config --global init.defaultBranch default") + @target = "sync-test" + SyncDefaultGems::REPOSITORIES[@target] = ["ruby/#{@target}", "default"] + @sha = {} + @origdir = Dir.pwd + Dir.chdir(@testdir) + ["src", @target].each do |dir| + git(*%W"init -q #{dir}") + File.write("#{dir}/.gitignore", "*~\n") + Dir.mkdir("#{dir}/lib") + File.write("#{dir}/lib/common.rb", ":ok\n") + Dir.mkdir("#{dir}/.github") + Dir.mkdir("#{dir}/.github/workflows") + File.write("#{dir}/.github/workflows/default.yml", "default:\n") + git(*%W"add .gitignore lib/common.rb .github", chdir: dir) + git(*%W"commit -q -m", "Initialize", chdir: dir) + if dir == "src" + File.write("#{dir}/lib/fine.rb", "return\n") + Dir.mkdir("#{dir}/test") + File.write("#{dir}/test/test_fine.rb", "return\n") + git(*%W"add lib/fine.rb test/test_fine.rb", chdir: dir) + git(*%W"commit -q -m", "Looks fine", chdir: dir) + end + Dir.mkdir("#{dir}/tool") + File.write("#{dir}/tool/ok", "#!/bin/sh\n""echo ok\n") + git(*%W"add tool/ok", chdir: dir) + git(*%W"commit -q -m", "Add tool #{dir}", chdir: dir) + @sha[dir] = top_commit(dir) + end + git(*%W"remote add #{@target} ../#{@target}", chdir: "src") + end + + def teardown + if @target + Dir.chdir(@origdir) + SyncDefaultGems::REPOSITORIES.delete(@target) + ENV.update(@git_config) + FileUtils.rm_rf(@testdir) + end + super + end + + def capture_process_output_to(outputs) + return yield unless outputs&.empty? == false + IO.pipe do |r, w| + orig = outputs.map {|out| out.dup} + outputs.each {|out| out.reopen(w)} + w.close + reader = Thread.start {r.read} + yield + ensure + outputs.each {|out| o = orig.shift; out.reopen(o); o.close} + return reader.value + end + end + + def capture_process_outputs + out = err = nil + synchronize do + out = capture_process_output_to(STDOUT) do + err = capture_process_output_to(STDERR) do + yield + end + end + end + return out, err + end + + def git(*commands, **opts) + system("git", *commands, exception: true, **opts) + end + + def top_commit(dir, format: "%H") + IO.popen(%W[git log --format=#{format} -1], chdir: dir, &:read)&.chomp + end + + def assert_sync(commits = true, success: true, editor: nil) + result = nil + out = capture_process_output_to([STDOUT, STDERR]) do + Dir.chdir("src") do + orig_editor = ENV["GIT_EDITOR"] + ENV["GIT_EDITOR"] = editor || 'false' + edit = true if editor + + result = SyncDefaultGems.sync_default_gems_with_commits(@target, commits, edit: edit) + ensure + ENV["GIT_EDITOR"] = orig_editor + end + end + assert_equal(success, result, out) + out + end + + def test_sync + File.write("#@target/lib/common.rb", "# OK!\n") + git(*%W"commit -q -m", "OK", "lib/common.rb", chdir: @target) + out = assert_sync() + assert_not_equal(@sha["src"], top_commit("src"), out) + assert_equal("# OK!\n", File.read("src/lib/common.rb")) + log = top_commit("src", format: "%B").lines + assert_equal("[ruby/#@target] OK\n", log.first, out) + assert_match(%r[/ruby/#{@target}/commit/\h+$], log.last, out) + assert_operator(top_commit(@target), :start_with?, log.last[/\h+$/], out) + end + + def test_skip_tool + git(*%W"rm -q tool/ok", chdir: @target) + git(*%W"commit -q -m", "Remove tool", chdir: @target) + out = assert_sync() + assert_equal(@sha["src"], top_commit("src"), out) + end + + def test_skip_test_fixtures + Dir.mkdir("#@target/test") + Dir.mkdir("#@target/test/fixtures") + File.write("#@target/test/fixtures/fixme.rb", "") + git(*%W"add test/fixtures/fixme.rb", chdir: @target) + git(*%W"commit -q -m", "Add fixtures", chdir: @target) + out = assert_sync(["#{@sha[@target]}..#{@target}/default"]) + assert_equal(@sha["src"], top_commit("src"), out) + end + + def test_skip_toplevel + Dir.mkdir("#@target/docs") + File.write("#@target/docs/NEWS.md", "= NEWS!!!\n") + git(*%W"add --", "docs/NEWS.md", chdir: @target) + File.write("#@target/docs/hello.md", "Hello\n") + git(*%W"add --", "docs/hello.md", chdir: @target) + git(*%W"commit -q -m", "It's a news", chdir: @target) + out = assert_sync() + assert_equal(@sha["src"], top_commit("src"), out) + end + + def test_adding_toplevel + Dir.mkdir("#@target/docs") + File.write("#@target/docs/NEWS.md", "= New library\n") + File.write("#@target/lib/news.rb", "return\n") + git(*%W"add --", "docs/NEWS.md", "lib/news.rb", chdir: @target) + git(*%W"commit -q -m", "New lib", chdir: @target) + out = assert_sync() + assert_not_equal(@sha["src"], top_commit("src"), out) + assert_equal "return\n", File.read("src/lib/news.rb") + assert_include top_commit("src", format: "oneline"), "[ruby/#{@target}] New lib" + assert_not_operator File, :exist?, "src/docs" + end + + def test_gitignore + File.write("#@target/.gitignore", "*.bak\n", mode: "a") + File.write("#@target/lib/common.rb", "Should.be_merged\n", mode: "a") + File.write("#@target/.github/workflows/main.yml", "# Should not merge\n", mode: "a") + git(*%W"add .github", chdir: @target) + git(*%W"commit -q -m", "Should be common.rb only", + *%W".gitignore lib/common.rb .github", chdir: @target) + out = assert_sync() + assert_not_equal(@sha["src"], top_commit("src"), out) + assert_equal("*~\n", File.read("src/.gitignore"), out) + assert_equal("#!/bin/sh\n""echo ok\n", File.read("src/tool/ok"), out) + assert_equal(":ok\n""Should.be_merged\n", File.read("src/lib/common.rb"), out) + assert_not_operator(File, :exist?, "src/.github/workflows/main.yml", out) + end + + def test_gitignore_after_conflict + File.write("src/Gemfile", "# main\n") + git(*%W"add Gemfile", chdir: "src") + git(*%W"commit -q -m", "Add Gemfile", chdir: "src") + File.write("#@target/Gemfile", "# conflict\n", mode: "a") + File.write("#@target/lib/common.rb", "Should.be_merged\n", mode: "a") + File.write("#@target/.github/workflows/main.yml", "# Should not merge\n", mode: "a") + git(*%W"add Gemfile .github lib/common.rb", chdir: @target) + git(*%W"commit -q -m", "Should be common.rb only", chdir: @target) + out = assert_sync() + assert_not_equal(@sha["src"], top_commit("src"), out) + assert_equal("# main\n", File.read("src/Gemfile"), out) + assert_equal(":ok\n""Should.be_merged\n", File.read("src/lib/common.rb"), out) + assert_not_operator(File, :exist?, "src/.github/workflows/main.yml", out) + end + + def test_delete_after_conflict + File.write("#@target/lib/bad.rb", "raise\n") + git(*%W"add lib/bad.rb", chdir: @target) + git(*%W"commit -q -m", "Add bad.rb", chdir: @target) + out = assert_sync + assert_equal("raise\n", File.read("src/lib/bad.rb")) + + git(*%W"rm lib/bad.rb", chdir: "src", out: IO::NULL) + git(*%W"commit -q -m", "Remove bad.rb", chdir: "src") + + File.write("#@target/lib/bad.rb", "raise 'bar'\n") + File.write("#@target/lib/common.rb", "Should.be_merged\n", mode: "a") + git(*%W"add lib/bad.rb lib/common.rb", chdir: @target) + git(*%W"commit -q -m", "Add conflict", chdir: @target) + + head = top_commit("src") + out = assert_sync(editor: "git rm -f lib/bad.rb") + assert_not_equal(head, top_commit("src")) + assert_equal(":ok\n""Should.be_merged\n", File.read("src/lib/common.rb"), out) + assert_not_operator(File, :exist?, "src/lib/bad.rb", out) + end + end +end diff --git a/tool/test/testunit/test4test_load_failure.rb b/tool/test/testunit/test4test_load_failure.rb new file mode 100644 index 0000000000..e1570c2542 --- /dev/null +++ b/tool/test/testunit/test4test_load_failure.rb @@ -0,0 +1 @@ +raise LoadError, "no-such-library" diff --git a/tool/test/testunit/test4test_timeout.rb b/tool/test/testunit/test4test_timeout.rb new file mode 100644 index 0000000000..3225f66398 --- /dev/null +++ b/tool/test/testunit/test4test_timeout.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true +$LOAD_PATH.unshift "#{File.dirname(__FILE__)}/../../lib" + +require 'test/unit' +require 'timeout' + +class TestForTestTimeout < Test::Unit::TestCase + 10.times do |i| + define_method("test_timeout_#{i}") do + Timeout.timeout(0.001) do + sleep + end + end + end +end diff --git a/tool/test/testunit/test_assertion.rb b/tool/test/testunit/test_assertion.rb index 8c83b447a7..709b495572 100644 --- a/tool/test/testunit/test_assertion.rb +++ b/tool/test/testunit/test_assertion.rb @@ -26,4 +26,28 @@ class TestAssertion < Test::Unit::TestCase return_in_assert_raise end end + + def test_assert_pattern_list + assert_pattern_list([/foo?/], "foo") + assert_not_pattern_list([/foo?/], "afoo") + assert_not_pattern_list([/foo?/], "foo?") + assert_pattern_list([:*, /foo?/, :*], "foo") + assert_pattern_list([:*, /foo?/], "afoo") + assert_not_pattern_list([:*, /foo?/], "afoo?") + assert_pattern_list([/foo?/, :*], "foo?") + + assert_not_pattern_list(["foo?"], "foo") + assert_not_pattern_list(["foo?"], "afoo") + assert_pattern_list(["foo?"], "foo?") + assert_not_pattern_list([:*, "foo?", :*], "foo") + assert_not_pattern_list([:*, "foo?"], "afoo") + assert_pattern_list([:*, "foo?"], "afoo?") + assert_pattern_list(["foo?", :*], "foo?") + end + + def assert_not_pattern_list(pattern_list, actual, message=nil) + assert_raise(Test::Unit::AssertionFailedError) do + assert_pattern_list(pattern_list, actual, message) + end + end end diff --git a/tool/test/testunit/test_hideskip.rb b/tool/test/testunit/test_hideskip.rb index e15947fe53..0c4c9b40f2 100644 --- a/tool/test/testunit/test_hideskip.rb +++ b/tool/test/testunit/test_hideskip.rb @@ -6,14 +6,14 @@ class TestHideSkip < Test::Unit::TestCase assert_not_match(/^ *1\) Skipped/, hideskip) assert_match(/^ *1\) Skipped.*^ *2\) Skipped/m, hideskip("--show-skip")) output = hideskip("--hide-skip") - output.gsub!(/Successful MJIT finish\n/, '') if defined?(RubyVM::MJIT) && RubyVM::MJIT.enabled? + output.gsub!(/Successful RJIT finish\n/, '') if defined?(RubyVM::RJIT) && RubyVM::RJIT.enabled? assert_match(/assertions\/s.\n+2 tests, 0 assertions, 0 failures, 0 errors, 2 skips/, output) end private def hideskip(*args) - IO.popen([*@options[:ruby], "#{File.dirname(__FILE__)}/test4test_hideskip.rb", + IO.popen([*@__runner_options__[:ruby], "#{File.dirname(__FILE__)}/test4test_hideskip.rb", "--verbose", *args], err: [:child, :out]) {|f| f.read } diff --git a/tool/test/testunit/test_launchable.rb b/tool/test/testunit/test_launchable.rb new file mode 100644 index 0000000000..70c371e212 --- /dev/null +++ b/tool/test/testunit/test_launchable.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: false +require 'test/unit' +require 'tempfile' +require 'json' + +class TestLaunchable < Test::Unit::TestCase + def test_json_stream_writer + Tempfile.create(['launchable-test-', '.json']) do |f| + json_stream_writer = Test::Unit::LaunchableOption::JsonStreamWriter.new(f.path) + json_stream_writer.write_array('testCases') + json_stream_writer.write_object( + { + testPath: "file=test/test_a.rb#class=class1#testcase=testcase899", + duration: 42, + status: "TEST_FAILED", + stdout: nil, + stderr: nil, + createdAt: "2021-10-05T12:34:00", + data: { + lineNumber: 1 + } + } + ) + json_stream_writer.write_object( + { + testPath: "file=test/test_a.rb#class=class1#testcase=testcase899", + duration: 45, + status: "TEST_PASSED", + stdout: "This is stdout", + stderr: "This is stderr", + createdAt: "2021-10-05T12:36:00", + data: { + lineNumber: 10 + } + } + ) + json_stream_writer.close() + expected = <<JSON +{ + "testCases": [ + { + "testPath": "file=test/test_a.rb#class=class1#testcase=testcase899", + "duration": 42, + "status": "TEST_FAILED", + "stdout": null, + "stderr": null, + "createdAt": "2021-10-05T12:34:00", + "data": { + "lineNumber": 1 + } + }, + { + "testPath": "file=test/test_a.rb#class=class1#testcase=testcase899", + "duration": 45, + "status": "TEST_PASSED", + "stdout": "This is stdout", + "stderr": "This is stderr", + "createdAt": "2021-10-05T12:36:00", + "data": { + "lineNumber": 10 + } + } + ] +} +JSON + assert_equal(expected, f.read) + end + end +end diff --git a/tool/test/testunit/test_load_failure.rb b/tool/test/testunit/test_load_failure.rb new file mode 100644 index 0000000000..8defa9e39a --- /dev/null +++ b/tool/test/testunit/test_load_failure.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true +require 'test/unit' + +class TestLoadFailure < Test::Unit::TestCase + def test_load_failure + assert_not_predicate(load_failure, :success?) + end + + def test_load_failure_parallel + assert_not_predicate(load_failure("-j2"), :success?) + end + + private + + def load_failure(*args) + IO.popen([*@__runner_options__[:ruby], "#{__dir__}/../runner.rb", + "#{__dir__}/test4test_load_failure.rb", + "--verbose", *args], err: [:child, :out]) {|f| + assert_include(f.read, "test4test_load_failure.rb") + } + $? + end +end diff --git a/tool/test/testunit/test_parallel.rb b/tool/test/testunit/test_parallel.rb index 006354aee2..6882fd6c5f 100644 --- a/tool/test/testunit/test_parallel.rb +++ b/tool/test/testunit/test_parallel.rb @@ -6,14 +6,14 @@ module TestParallel PARALLEL_RB = "#{__dir__}/../../lib/test/unit/parallel.rb" TESTS = "#{__dir__}/tests_for_parallel" # use large timeout for --jit-wait - TIMEOUT = EnvUtil.apply_timeout_scale(defined?(RubyVM::MJIT) && RubyVM::MJIT.enabled? ? 100 : 30) + TIMEOUT = EnvUtil.apply_timeout_scale(defined?(RubyVM::RJIT) && RubyVM::RJIT.enabled? ? 100 : 30) class TestParallelWorker < Test::Unit::TestCase def setup i, @worker_in = IO.pipe @worker_out, o = IO.pipe - @worker_pid = spawn(*@options[:ruby], PARALLEL_RB, - "--ruby", @options[:ruby].join(" "), + @worker_pid = spawn(*@__runner_options__[:ruby], PARALLEL_RB, + "--ruby", @__runner_options__[:ruby].join(" "), "-j", "t1", "-v", out: o, in: i) [i,o].each(&:close) end @@ -119,7 +119,7 @@ module TestParallel result = Marshal.load($1.chomp.unpack1("m")) assert_equal(5, result[0]) - pend "TODO: result[1] returns 17. We should investigate it" do + pend "TODO: result[1] returns 17. We should investigate it" do # TODO: misusage of pend (pend doens't use given block) assert_equal(12, result[1]) end assert_kind_of(Array,result[2]) @@ -143,11 +143,11 @@ module TestParallel end class TestParallel < Test::Unit::TestCase - def spawn_runner(*opt_args) + def spawn_runner(*opt_args, jobs: "t1") @test_out, o = IO.pipe - @test_pid = spawn(*@options[:ruby], TESTS+"/runner.rb", - "--ruby", @options[:ruby].join(" "), - "-j","t1",*opt_args, out: o, err: o) + @test_pid = spawn(*@__runner_options__[:ruby], TESTS+"/runner.rb", + "--ruby", @__runner_options__[:ruby].join(" "), + "-j", jobs, *opt_args, out: o, err: o) o.close end @@ -166,11 +166,7 @@ module TestParallel end def test_ignore_jzero - @test_out, o = IO.pipe - @test_pid = spawn(*@options[:ruby], TESTS+"/runner.rb", - "--ruby", @options[:ruby].join(" "), - "-j","0", out: File::NULL, err: o) - o.close + spawn_runner(jobs: "0") Timeout.timeout(TIMEOUT) { assert_match(/Error: parameter of -j option should be greater than 0/,@test_out.read) } @@ -215,5 +211,12 @@ module TestParallel assert_match(/^Retrying hung up testcases\.+$/, buf) assert_match(/^2 tests,.* 0 failures,/, buf) end + + def test_retry_workers + spawn_runner "--worker-timeout=1", "test4test_slow_0.rb", "test4test_slow_1.rb", jobs: "2" + buf = Timeout.timeout(TIMEOUT) {@test_out.read} + assert_match(/^Retrying hung up testcases\.+$/, buf) + assert_match(/^2 tests,.* 0 failures,/, buf) + end end end diff --git a/tool/test/testunit/test_sorting.rb b/tool/test/testunit/test_sorting.rb index 7678249ec2..3e5d7bfdcc 100644 --- a/tool/test/testunit/test_sorting.rb +++ b/tool/test/testunit/test_sorting.rb @@ -10,7 +10,7 @@ class TestTestUnitSorting < Test::Unit::TestCase end def sorting(*args) - IO.popen([*@options[:ruby], "#{File.dirname(__FILE__)}/test4test_sorting.rb", + IO.popen([*@__runner_options__[:ruby], "#{File.dirname(__FILE__)}/test4test_sorting.rb", "--verbose", *args], err: [:child, :out]) {|f| f.read } diff --git a/tool/test/testunit/test_timeout.rb b/tool/test/testunit/test_timeout.rb new file mode 100644 index 0000000000..452f5e1a7e --- /dev/null +++ b/tool/test/testunit/test_timeout.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: false +require 'test/unit' + +class TestTiemout < Test::Unit::TestCase + def test_timeout + cmd = [*@__runner_options__[:ruby], "#{File.dirname(__FILE__)}/test4test_timeout.rb"] + result = IO.popen(cmd, err: [:child, :out], &:read) + assert_not_match(/^T{10}$/, result) + end +end diff --git a/tool/test/testunit/tests_for_parallel/slow_helper.rb b/tool/test/testunit/tests_for_parallel/slow_helper.rb new file mode 100644 index 0000000000..38067c1f47 --- /dev/null +++ b/tool/test/testunit/tests_for_parallel/slow_helper.rb @@ -0,0 +1,8 @@ +require 'test/unit' + +module TestSlowTimeout + def test_slow + sleep_for = EnvUtil.apply_timeout_scale((ENV['sec'] || 3).to_i) + sleep sleep_for if on_parallel_worker? + end +end diff --git a/tool/test/testunit/tests_for_parallel/test4test_slow_0.rb b/tool/test/testunit/tests_for_parallel/test4test_slow_0.rb new file mode 100644 index 0000000000..a749b0e1d3 --- /dev/null +++ b/tool/test/testunit/tests_for_parallel/test4test_slow_0.rb @@ -0,0 +1,5 @@ +require_relative 'slow_helper' + +class TestSlowV0 < Test::Unit::TestCase + include TestSlowTimeout +end diff --git a/tool/test/testunit/tests_for_parallel/test4test_slow_1.rb b/tool/test/testunit/tests_for_parallel/test4test_slow_1.rb new file mode 100644 index 0000000000..924a3b11fa --- /dev/null +++ b/tool/test/testunit/tests_for_parallel/test4test_slow_1.rb @@ -0,0 +1,5 @@ +require_relative 'slow_helper' + +class TestSlowV1 < Test::Unit::TestCase + include TestSlowTimeout +end diff --git a/tool/test/webrick/test_cgi.rb b/tool/test/webrick/test_cgi.rb index 7a75cf565e..a9be8f353d 100644 --- a/tool/test/webrick/test_cgi.rb +++ b/tool/test/webrick/test_cgi.rb @@ -12,30 +12,8 @@ class TestWEBrickCGI < Test::Unit::TestCase super end - def start_cgi_server(log_tester=TestWEBrick::DefaultLogTester, &block) - config = { - :CGIInterpreter => TestWEBrick::RubyBin, - :DocumentRoot => File.dirname(__FILE__), - :DirectoryIndex => ["webrick.cgi"], - :RequestCallback => Proc.new{|req, res| - def req.meta_vars - meta = super - meta["RUBYLIB"] = $:.join(File::PATH_SEPARATOR) - meta[RbConfig::CONFIG['LIBPATHENV']] = ENV[RbConfig::CONFIG['LIBPATHENV']] if RbConfig::CONFIG['LIBPATHENV'] - return meta - end - }, - } - if RUBY_PLATFORM =~ /mswin|mingw|cygwin|bccwin32/ - config[:CGIPathEnv] = ENV['PATH'] # runtime dll may not be in system dir. - end - TestWEBrick.start_httpserver(config, log_tester){|server, addr, port, log| - block.call(server, addr, port, log) - } - end - def test_cgi - start_cgi_server{|server, addr, port, log| + TestWEBrick.start_cgi_server{|server, addr, port, log| http = Net::HTTP.new(addr, port) req = Net::HTTP::Get.new("/webrick.cgi") http.request(req){|res| assert_equal("/webrick.cgi", res.body, log.call)} @@ -98,7 +76,7 @@ class TestWEBrickCGI < Test::Unit::TestCase log_tester = lambda {|log, access_log| assert_match(/BadRequest/, log.join) } - start_cgi_server(log_tester) {|server, addr, port, log| + TestWEBrick.start_cgi_server({}, log_tester) {|server, addr, port, log| sock = TCPSocket.new(addr, port) begin sock << "POST /webrick.cgi HTTP/1.0" << CRLF @@ -115,7 +93,7 @@ class TestWEBrickCGI < Test::Unit::TestCase end def test_cgi_env - start_cgi_server do |server, addr, port, log| + TestWEBrick.start_cgi_server do |server, addr, port, log| http = Net::HTTP.new(addr, port) req = Net::HTTP::Get.new("/webrick.cgi/dumpenv") req['proxy'] = 'http://example.com/' @@ -137,7 +115,7 @@ class TestWEBrickCGI < Test::Unit::TestCase assert_equal(1, log.length) assert_match(/ERROR bad URI/, log[0]) } - start_cgi_server(log_tester) {|server, addr, port, log| + TestWEBrick.start_cgi_server({}, log_tester) {|server, addr, port, log| res = TCPSocket.open(addr, port) {|sock| sock << "GET /#{CtrlSeq}#{CRLF}#{CRLF}" sock.close_write @@ -155,7 +133,7 @@ class TestWEBrickCGI < Test::Unit::TestCase assert_equal(1, log.length) assert_match(/ERROR bad header/, log[0]) } - start_cgi_server(log_tester) {|server, addr, port, log| + TestWEBrick.start_cgi_server({}, log_tester) {|server, addr, port, log| res = TCPSocket.open(addr, port) {|sock| sock << "GET / HTTP/1.0#{CRLF}#{CtrlSeq}#{CRLF}#{CRLF}" sock.close_write diff --git a/tool/test/webrick/test_filehandler.rb b/tool/test/webrick/test_filehandler.rb index 146d8ce792..452667d4f4 100644 --- a/tool/test/webrick/test_filehandler.rb +++ b/tool/test/webrick/test_filehandler.rb @@ -85,12 +85,12 @@ class WEBrick::TestFileHandler < Test::Unit::TestCase "Content-Type: text/plain\r\n" \ "Content-Range: bytes 0-0/#{filesize}\r\n" \ "\r\n" \ - "#{IO.read(__FILE__, 1)}\r\n" \ + "#{File.read(__FILE__, 1)}\r\n" \ "--#{boundary}\r\n" \ "Content-Type: text/plain\r\n" \ "Content-Range: bytes #{off}-#{last}/#{filesize}\r\n" \ "\r\n" \ - "#{IO.read(__FILE__, 2, off)}\r\n" \ + "#{File.read(__FILE__, 2, off)}\r\n" \ "--#{boundary}--\r\n" assert_equal exp, body end @@ -247,22 +247,16 @@ class WEBrick::TestFileHandler < Test::Unit::TestCase def test_short_filename return if File.executable?(__FILE__) # skip on strange file system - return if /mswin/ =~ RUBY_PLATFORM && ENV.key?('GITHUB_ACTIONS') # not working from the beginning - config = { - :CGIInterpreter => TestWEBrick::RubyBin, - :DocumentRoot => File.dirname(__FILE__), - :CGIPathEnv => ENV['PATH'], - } log_tester = lambda {|log, access_log| log = log.reject {|s| /ERROR `.*\' not found\./ =~ s } log = log.reject {|s| /WARN the request refers nondisclosure name/ =~ s } assert_equal([], log) } - TestWEBrick.start_httpserver(config, log_tester) do |server, addr, port, log| + TestWEBrick.start_cgi_server({}, log_tester) do |server, addr, port, log| http = Net::HTTP.new(addr, port) if windows? - root = config[:DocumentRoot].tr("/", "\\") + root = File.dirname(__FILE__).tr("/", "\\") fname = IO.popen(%W[dir /x #{root}\\webrick_long_filename.cgi], encoding: "binary", &:read) fname.sub!(/\A.*$^$.*$^$/m, '') if fname diff --git a/tool/test/webrick/test_httprequest.rb b/tool/test/webrick/test_httprequest.rb index 759ccbdada..3c0ea937d9 100644 --- a/tool/test/webrick/test_httprequest.rb +++ b/tool/test/webrick/test_httprequest.rb @@ -245,7 +245,7 @@ GET / _end_of_message_ msg.gsub!(/^ {6}/, "") - open(__FILE__){|io| + File.open(__FILE__){|io| while chunk = io.read(100) msg << chunk.size.to_s(16) << crlf msg << chunk << crlf diff --git a/tool/test/webrick/test_httpserver.rb b/tool/test/webrick/test_httpserver.rb index 4133be85ad..f6b53e142b 100644 --- a/tool/test/webrick/test_httpserver.rb +++ b/tool/test/webrick/test_httpserver.rb @@ -253,7 +253,7 @@ class TestWEBrickHTTPServer < Test::Unit::TestCase server.virtual_host(WEBrick::HTTPServer.new(vhost_config)) Thread.pass while server.status != :Running - sleep 1 if defined?(RubyVM::MJIT) && RubyVM::MJIT.enabled? # server.status behaves unexpectedly with --jit-wait + sleep 1 if defined?(RubyVM::RJIT) && RubyVM::RJIT.enabled? # server.status behaves unexpectedly with --jit-wait assert_equal(1, started, log.call) assert_equal(0, stopped, log.call) assert_equal(0, accepted, log.call) diff --git a/tool/test/webrick/test_server.rb b/tool/test/webrick/test_server.rb index 815cc3ce39..3bd8115c61 100644 --- a/tool/test/webrick/test_server.rb +++ b/tool/test/webrick/test_server.rb @@ -65,7 +65,7 @@ class TestWEBrickServer < Test::Unit::TestCase } TestWEBrick.start_server(Echo, config){|server, addr, port, log| true while server.status != :Running - sleep 1 if defined?(RubyVM::MJIT) && RubyVM::MJIT.enabled? # server.status behaves unexpectedly with --jit-wait + sleep 1 if defined?(RubyVM::RJIT) && RubyVM::RJIT.enabled? # server.status behaves unexpectedly with --jit-wait assert_equal(1, started, log.call) assert_equal(0, stopped, log.call) assert_equal(0, accepted, log.call) diff --git a/tool/test/webrick/utils.rb b/tool/test/webrick/utils.rb index a8568d0a43..c8e84c37f1 100644 --- a/tool/test/webrick/utils.rb +++ b/tool/test/webrick/utils.rb @@ -81,4 +81,24 @@ module TestWEBrick def start_httpproxy(config={}, log_tester=DefaultLogTester, &block) start_server(WEBrick::HTTPProxyServer, config, log_tester, &block) end + + def start_cgi_server(config={}, log_tester=TestWEBrick::DefaultLogTester, &block) + config = { + :CGIInterpreter => TestWEBrick::RubyBin, + :DocumentRoot => File.dirname(__FILE__), + :DirectoryIndex => ["webrick.cgi"], + :RequestCallback => Proc.new{|req, res| + def req.meta_vars + meta = super + meta["RUBYLIB"] = $:.join(File::PATH_SEPARATOR) + meta[RbConfig::CONFIG['LIBPATHENV']] = ENV[RbConfig::CONFIG['LIBPATHENV']] if RbConfig::CONFIG['LIBPATHENV'] + return meta + end + }, + }.merge(config) + if RUBY_PLATFORM =~ /mswin|mingw|cygwin|bccwin32/ + config[:CGIPathEnv] = ENV['PATH'] # runtime dll may not be in system dir. + end + start_server(WEBrick::HTTPServer, config, log_tester, &block) + end end diff --git a/tool/test/webrick/webrick.cgi b/tool/test/webrick/webrick.cgi index a294fa72f9..45594b7a7b 100644..100755 --- a/tool/test/webrick/webrick.cgi +++ b/tool/test/webrick/webrick.cgi @@ -15,11 +15,11 @@ class TestApp < WEBrick::CGI }.join(", ") }.join(", ") elsif %r{/$} =~ req.request_uri.to_s - res.body = "" + res.body = +"" res.body << req.request_uri.to_s << "\n" res.body << req.script_name elsif !req.cookies.empty? - res.body = req.cookies.inject(""){|result, cookie| + res.body = req.cookies.inject(+""){|result, cookie| result << "%s=%s\n" % [cookie.name, cookie.value] } res.cookies << WEBrick::Cookie.new("Customer", "WILE_E_COYOTE") diff --git a/tool/test_for_warn_bundled_gems/.gitignore b/tool/test_for_warn_bundled_gems/.gitignore new file mode 100644 index 0000000000..a9a5aecf42 --- /dev/null +++ b/tool/test_for_warn_bundled_gems/.gitignore @@ -0,0 +1 @@ +tmp diff --git a/tool/test_for_warn_bundled_gems/Gemfile b/tool/test_for_warn_bundled_gems/Gemfile new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/tool/test_for_warn_bundled_gems/Gemfile diff --git a/tool/test_for_warn_bundled_gems/Gemfile.lock b/tool/test_for_warn_bundled_gems/Gemfile.lock new file mode 100644 index 0000000000..003cb81444 --- /dev/null +++ b/tool/test_for_warn_bundled_gems/Gemfile.lock @@ -0,0 +1,11 @@ +GEM + specs: + +PLATFORMS + arm64-darwin-22 + ruby + +DEPENDENCIES + +BUNDLED WITH + 2.5.0.dev diff --git a/tool/test_for_warn_bundled_gems/README.md b/tool/test_for_warn_bundled_gems/README.md new file mode 100644 index 0000000000..dc2d2a6cb9 --- /dev/null +++ b/tool/test_for_warn_bundled_gems/README.md @@ -0,0 +1,3 @@ +This directory contains tests for the bundled gems warning under the Bundler. + +see [test.sh](./test.sh) for details. diff --git a/tool/test_for_warn_bundled_gems/test.sh b/tool/test_for_warn_bundled_gems/test.sh new file mode 100755 index 0000000000..2404571daf --- /dev/null +++ b/tool/test_for_warn_bundled_gems/test.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +echo "* Show warning require and LoadError" +ruby test_warn_bundled_gems.rb +echo + +echo "* Show warning when bundled gems called as dependency" +ruby test_warn_dependency.rb +echo + +echo "* Show warning sub-feature like bigdecimal/util" +ruby test_warn_sub_feature.rb +echo + +echo "* Show warning dash gem like net/smtp" +ruby test_warn_dash_gem.rb +echo + +echo "* Show warning when bundle exec with ruby and script" +bundle exec ruby test_warn_bundle_exec.rb +echo + +echo "* Show warning when bundle exec with shebang's script" +bundle exec ./test_warn_bundle_exec_shebang.rb +echo + +echo "* Show warning when bundle exec with -r option" +bundle exec ruby -rostruct -e '' +echo + +echo "* Show warning with bootsnap" +ruby test_warn_bootsnap.rb +echo + +echo "* Show warning with zeitwerk" +ruby test_warn_zeitwerk.rb +echo + +echo "* Don't show warning bundled gems on Gemfile" +ruby test_no_warn_dependency.rb +echo + +echo "* Don't show warning with net/smtp when net-smtp on Gemfile" +ruby test_no_warn_dash_gem.rb +echo + +echo "* Don't show warning bigdecimal/util when bigdecimal on Gemfile" +ruby test_no_warn_sub_feature.rb +echo diff --git a/tool/test_for_warn_bundled_gems/test_no_warn_dash_gem.rb b/tool/test_for_warn_bundled_gems/test_no_warn_dash_gem.rb new file mode 100644 index 0000000000..72ae23b040 --- /dev/null +++ b/tool/test_for_warn_bundled_gems/test_no_warn_dash_gem.rb @@ -0,0 +1,8 @@ +require "bundler/inline" + +gemfile do + source "https://rubygems.org" + gem "net-smtp" +end + +require "net/smtp" diff --git a/tool/test_for_warn_bundled_gems/test_no_warn_dependency.rb b/tool/test_for_warn_bundled_gems/test_no_warn_dependency.rb new file mode 100644 index 0000000000..94a32a9108 --- /dev/null +++ b/tool/test_for_warn_bundled_gems/test_no_warn_dependency.rb @@ -0,0 +1,10 @@ +require "bundler/inline" + +gemfile do + source "https://rubygems.org" + gem "activesupport", "7.0.7.2" + gem "bigdecimal" + gem "mutex_m" +end + +require "active_support/all" diff --git a/tool/test_for_warn_bundled_gems/test_no_warn_sub_feature.rb b/tool/test_for_warn_bundled_gems/test_no_warn_sub_feature.rb new file mode 100644 index 0000000000..7d62a2f9d0 --- /dev/null +++ b/tool/test_for_warn_bundled_gems/test_no_warn_sub_feature.rb @@ -0,0 +1,8 @@ +require "bundler/inline" + +gemfile do + source "https://rubygems.org" + gem "bigdecimal" +end + +require "bigdecimal/util" diff --git a/tool/test_for_warn_bundled_gems/test_warn_bootsnap.rb b/tool/test_for_warn_bundled_gems/test_warn_bootsnap.rb new file mode 100644 index 0000000000..eac58de974 --- /dev/null +++ b/tool/test_for_warn_bundled_gems/test_warn_bootsnap.rb @@ -0,0 +1,11 @@ +require "bundler/inline" + +gemfile do + source "https://rubygems.org" + gem "bootsnap", require: false +end + +require 'bootsnap' +Bootsnap.setup(cache_dir: 'tmp/cache') + +require 'csv' diff --git a/tool/test_for_warn_bundled_gems/test_warn_bundle_exec.rb b/tool/test_for_warn_bundled_gems/test_warn_bundle_exec.rb new file mode 100644 index 0000000000..30db47ce61 --- /dev/null +++ b/tool/test_for_warn_bundled_gems/test_warn_bundle_exec.rb @@ -0,0 +1 @@ +require "base64" diff --git a/tool/test_for_warn_bundled_gems/test_warn_bundle_exec_shebang.rb b/tool/test_for_warn_bundled_gems/test_warn_bundle_exec_shebang.rb new file mode 100755 index 0000000000..0338928e1e --- /dev/null +++ b/tool/test_for_warn_bundled_gems/test_warn_bundle_exec_shebang.rb @@ -0,0 +1,3 @@ +#!/usr/bin/env ruby + +require "base64" diff --git a/tool/test_for_warn_bundled_gems/test_warn_bundled_gems.rb b/tool/test_for_warn_bundled_gems/test_warn_bundled_gems.rb new file mode 100644 index 0000000000..13168292e3 --- /dev/null +++ b/tool/test_for_warn_bundled_gems/test_warn_bundled_gems.rb @@ -0,0 +1,8 @@ +require "bundler/inline" + +gemfile do + source "https://rubygems.org" +end + +require "mutex_m" +require "rss" diff --git a/tool/test_for_warn_bundled_gems/test_warn_dash_gem.rb b/tool/test_for_warn_bundled_gems/test_warn_dash_gem.rb new file mode 100644 index 0000000000..04ef2a52c0 --- /dev/null +++ b/tool/test_for_warn_bundled_gems/test_warn_dash_gem.rb @@ -0,0 +1,7 @@ +require "bundler/inline" + +gemfile do + source "https://rubygems.org" +end + +require "net/smtp" diff --git a/tool/test_for_warn_bundled_gems/test_warn_dependency.rb b/tool/test_for_warn_bundled_gems/test_warn_dependency.rb new file mode 100644 index 0000000000..9be3a2f6d9 --- /dev/null +++ b/tool/test_for_warn_bundled_gems/test_warn_dependency.rb @@ -0,0 +1,8 @@ +require "bundler/inline" + +gemfile do + source "https://rubygems.org" + gem "activesupport", "7.0.7.2" +end + +require "active_support/all" diff --git a/tool/test_for_warn_bundled_gems/test_warn_sub_feature.rb b/tool/test_for_warn_bundled_gems/test_warn_sub_feature.rb new file mode 100644 index 0000000000..bf7eb3572d --- /dev/null +++ b/tool/test_for_warn_bundled_gems/test_warn_sub_feature.rb @@ -0,0 +1,7 @@ +require "bundler/inline" + +gemfile do + source "https://rubygems.org" +end + +require "bigdecimal/util" diff --git a/tool/test_for_warn_bundled_gems/test_warn_zeitwerk.rb b/tool/test_for_warn_bundled_gems/test_warn_zeitwerk.rb new file mode 100644 index 0000000000..d554a0e675 --- /dev/null +++ b/tool/test_for_warn_bundled_gems/test_warn_zeitwerk.rb @@ -0,0 +1,12 @@ +require "bundler/inline" + +gemfile do + source "https://rubygems.org" + gem "zeitwerk", require: false +end + +require "zeitwerk" +loader = Zeitwerk::Loader.for_gem(warn_on_extra_files: false) +loader.setup + +require 'csv' diff --git a/tool/transcode-tblgen.rb b/tool/transcode-tblgen.rb index dba6f33ff9..1257a92d38 100644 --- a/tool/transcode-tblgen.rb +++ b/tool/transcode-tblgen.rb @@ -725,7 +725,7 @@ def citrus_decode_mapsrc(ces, csid, mapsrcs) path << ".src" path[path.rindex('/')] = '%' STDOUT.puts 'load mapsrc %s' % path if VERBOSE_MODE > 1 - open(path, 'rb') do |f| + File.open(path, 'rb') do |f| f.each_line do |l| break if /^BEGIN_MAP/ =~ l end @@ -1078,11 +1078,7 @@ if __FILE__ == $0 end libs1 = $".dup - if ERB.instance_method(:initialize).parameters.assoc(:key) # Ruby 2.6+ - erb = ERB.new(src, trim_mode: '%') - else - erb = ERB.new(src, nil, '%') - end + erb = ERB.new(src, trim_mode: '%') erb.filename = arg erb_result = erb.result(binding) libs2 = $".dup diff --git a/tool/transform_mjit_header.rb b/tool/transform_mjit_header.rb deleted file mode 100644 index 8867c556f0..0000000000 --- a/tool/transform_mjit_header.rb +++ /dev/null @@ -1,327 +0,0 @@ -# Copyright (C) 2017 Vladimir Makarov, <vmakarov@redhat.com> -# This is a script to transform functions to static inline. -# Usage: transform_mjit_header.rb <c-compiler> <header file> <out> - -require 'fileutils' -require 'tempfile' - -PROGRAM = File.basename($0, ".*") - -module MJITHeader - ATTR_VALUE_REGEXP = /[^()]|\([^()]*\)/ - ATTR_REGEXP = /__attribute__\s*\(\(#{ATTR_VALUE_REGEXP}*\)\)/ - # Example: - # VALUE foo(int bar) - # VALUE __attribute__ ((foo)) bar(int baz) - # __attribute__ ((foo)) VALUE bar(int baz) - FUNC_HEADER_REGEXP = /\A[^\[{(]*(\s*#{ATTR_REGEXP})*[^\[{(]*\((#{ATTR_REGEXP}|[^()])*\)(\s*#{ATTR_REGEXP})*\s*/ - TARGET_NAME_REGEXP = /\A(rb|ruby|vm|insn|attr|Init)_/ - - # Predefined macros for compilers which are already supported by MJIT. - # We're going to support cl.exe too (WIP) but `cl.exe -E` can't produce macro. - SUPPORTED_CC_MACROS = [ - '__GNUC__', # gcc - '__clang__', # clang - ] - - # These macros are relied on this script's transformation - PREFIXED_MACROS = [ - 'ALWAYS_INLINE', - 'COLDFUNC', - 'inline', - 'RBIMPL_ATTR_COLD', - ] - - # For MinGW's ras.h. Those macros have its name in its definition and can't be preprocessed multiple times. - RECURSIVE_MACROS = %w[ - RASCTRYINFO - RASIPADDR - ] - - IGNORED_FUNCTIONS = [ - 'rb_vm_search_method_slowpath', # This increases the time to compile when inlined. So we use it as external function. - 'rb_equal_opt', # Not used from VM and not compilable - 'ruby_abi_version', - ] - - ALWAYS_INLINED_FUNCTIONS = [ - 'vm_opt_plus', - 'vm_opt_minus', - 'vm_opt_mult', - 'vm_opt_div', - 'vm_opt_mod', - 'vm_opt_neq', - 'vm_opt_lt', - 'vm_opt_le', - 'vm_opt_gt', - 'vm_opt_ge', - 'vm_opt_ltlt', - 'vm_opt_and', - 'vm_opt_or', - 'vm_opt_aref', - 'vm_opt_aset', - 'vm_opt_aref_with', - 'vm_opt_aset_with', - 'vm_opt_not', - ] - - COLD_FUNCTIONS = %w[ - setup_parameters_complex - vm_call_iseq_setup - vm_call_iseq_setup_2 - vm_call_iseq_setup_tailcall - vm_call_method_each_type - vm_ic_update - ] - - # Return start..stop of last decl in CODE ending STOP - def self.find_decl(code, stop) - level = 0 - i = stop - while i = code.rindex(/[;{}]/, i) - if level == 0 && stop != i && decl_found?($&, i) - return decl_start($&, i)..stop - end - case $& - when '}' - level += 1 - when '{' - level -= 1 - end - i -= 1 - end - nil - end - - def self.decl_found?(code, i) - i == 0 || code == ';' || code == '}' - end - - def self.decl_start(code, i) - if i == 0 && code != ';' && code != '}' - 0 - else - i + 1 - end - end - - # Given DECL return the name of it, nil if failed - def self.decl_name_of(decl) - ident_regex = /\w+/ - decl = decl.gsub(/^#.+$/, '') # remove macros - reduced_decl = decl.gsub(ATTR_REGEXP, '') # remove attributes - su1_regex = /{[^{}]*}/ - su2_regex = /{([^{}]|#{su1_regex})*}/ - su3_regex = /{([^{}]|#{su2_regex})*}/ # 3 nested structs/unions is probably enough - reduced_decl.gsub!(su3_regex, '') # remove structs/unions in the header - id_seq_regex = /\s*(?:#{ident_regex}(?:\s+|\s*[*]+\s*))*/ - # Process function header: - match = /\A#{id_seq_regex}(?<name>#{ident_regex})\s*\(/.match(reduced_decl) - return match[:name] if match - # Process non-function declaration: - reduced_decl.gsub!(/\s*=[^;]+(?=;)/, '') # remove initialization - match = /#{id_seq_regex}(?<name>#{ident_regex})/.match(reduced_decl); - return match[:name] if match - nil - end - - # Return true if CC with CFLAGS compiles successfully the current code. - # Use STAGE in the message in case of a compilation failure - def self.check_code!(code, cc, cflags, stage) - with_code(code) do |path| - cmd = "#{cc} #{cflags} #{path}" - out = IO.popen(cmd, err: [:child, :out], &:read) - unless $?.success? - STDERR.puts "error in #{stage} header file:\n#{out}" - exit false - end - end - end - - # Remove unpreprocessable macros - def self.remove_harmful_macros!(code) - code.gsub!(/^#define #{Regexp.union(RECURSIVE_MACROS)} .*$/, '') - end - - # -dD outputs those macros, and it produces redefinition warnings or errors - # This assumes common.mk passes `-DMJIT_HEADER` first when it creates rb_mjit_header.h. - def self.remove_predefined_macros!(code) - code.sub!(/\A(#define [^\n]+|\n)*(#define MJIT_HEADER 1\n)/, '\2') - end - - # Return [macro, others]. But others include PREFIXED_MACROS to be used in code. - def self.separate_macro_and_code(code) - code.lines.partition do |l| - l.start_with?('#') && PREFIXED_MACROS.all? { |m| !l.start_with?("#define #{m}") } - end.map! { |lines| lines.join('') } - end - - def self.write(code, out) - # create with strict permission, then will install proper - # permission - FileUtils.mkdir_p(File.dirname(out), mode: 0700) - File.binwrite("#{out}.new", code, perm: 0600) - FileUtils.mv("#{out}.new", out) - end - - # Note that this checks runruby. This conservatively covers platform names. - def self.windows? - RUBY_PLATFORM =~ /mswin|mingw|msys/ - end - - def self.cl_exe?(cc) - cc =~ /\Acl(\z| |\.exe)/ - end - - # If code has macro which only supported compilers predefine, return true. - def self.supported_header?(code) - SUPPORTED_CC_MACROS.any? { |macro| code =~ /^#\s*define\s+#{Regexp.escape(macro)}\b/ } - end - - # This checks if syntax check outputs one of the following messages. - # "error: conflicting types for 'restrict'" - # "error: redefinition of parameter 'restrict'" - # If it's true, this script regards platform as AIX or Solaris and adds -std=c99 as workaround. - def self.conflicting_types?(code, cc, cflags) - with_code(code) do |path| - cmd = "#{cc} #{cflags} #{path}" - out = IO.popen(cmd, err: [:child, :out], &:read) - !$?.success? && - (out.match?(/error: conflicting types for '[^']+'/) || - out.match?(/error: redefinition of parameter '[^']+'/)) - end - end - - def self.with_code(code) - # for `system_header` pragma which can't be in the main file. - Tempfile.open(['', '.h'], mode: File::BINARY) do |f| - f.puts code - f.close - Tempfile.open(['', '.c'], mode: File::BINARY) do |c| - c.puts <<SRC -#include "#{f.path}" -SRC - c.close - return yield(c.path) - end - end - end - private_class_method :with_code -end - -if ARGV.size != 3 - abort "Usage: #{$0} <c-compiler> <header file> <out>" -end - -if STDOUT.tty? - require_relative 'lib/colorize' - color = Colorize.new -end -cc = ARGV[0] -code = File.binread(ARGV[1]) # Current version of the header file. -outfile = ARGV[2] -if MJITHeader.cl_exe?(cc) - cflags = '-DMJIT_HEADER -Zs' -else - cflags = '-S -DMJIT_HEADER -fsyntax-only -Werror=implicit-function-declaration -Werror=implicit-int -Wfatal-errors' -end - -if !MJITHeader.cl_exe?(cc) && !MJITHeader.supported_header?(code) - puts "This compiler (#{cc}) looks not supported for MJIT. Giving up to generate MJIT header." - MJITHeader.write("#error MJIT does not support '#{cc}' yet", outfile) - exit -end - -MJITHeader.remove_predefined_macros!(code) - -if MJITHeader.windows? # transformation is broken with Windows headers for now - MJITHeader.remove_harmful_macros!(code) - MJITHeader.check_code!(code, cc, cflags, 'initial') - puts "\nSkipped transforming external functions to static on Windows." - MJITHeader.write(code, outfile) - exit -end - -macro, code = MJITHeader.separate_macro_and_code(code) # note: this does not work on MinGW -code = <<header + code -#ifdef __GNUC__ -# pragma GCC system_header -#endif -header -code_to_check = "#{code}#{macro}" # macro should not affect code again - -if MJITHeader.conflicting_types?(code_to_check, cc, cflags) - cflags = "#{cflags} -std=c99" # For AIX gcc -end - -# Check initial file correctness in the manner of final output. -MJITHeader.check_code!(code_to_check, cc, cflags, 'initial') - -stop_pos = -1 -extern_names = [] -transform_logs = Hash.new { |h, k| h[k] = [] } - -# This loop changes function declarations to static inline. -while (decl_range = MJITHeader.find_decl(code, stop_pos)) - stop_pos = decl_range.begin - 1 - decl = code[decl_range] - decl_name = MJITHeader.decl_name_of(decl) - - if MJITHeader::IGNORED_FUNCTIONS.include?(decl_name) && /#{MJITHeader::FUNC_HEADER_REGEXP}{/.match(decl) - transform_logs[:def_to_decl] << decl_name - code[decl_range] = decl.sub(/{.+}/m, ';') - elsif MJITHeader::COLD_FUNCTIONS.include?(decl_name) && match = /#{MJITHeader::FUNC_HEADER_REGEXP}{/.match(decl) - header = match[0].sub(/{\z/, '').strip - header = "static #{header.sub(/\A((static|inline) )+/, '')}" - decl[match.begin(0)...match.end(0)] = '{' # remove header - code[decl_range] = "\nCOLDFUNC #{header} #{decl}" - elsif MJITHeader::ALWAYS_INLINED_FUNCTIONS.include?(decl_name) && match = /#{MJITHeader::FUNC_HEADER_REGEXP}{/.match(decl) - header = match[0].sub(/{\z/, '').strip - header = "static inline #{header.sub(/\A((static|inline) )+/, '')}" - decl[match.begin(0)...match.end(0)] = '{' # remove header - code[decl_range] = "\nALWAYS_INLINE(#{header});\n#{header} #{decl}" - elsif extern_names.include?(decl_name) && (decl =~ /#{MJITHeader::FUNC_HEADER_REGEXP};/) - decl.sub!(/(extern|static|inline) /, ' ') - unless decl_name =~ /\Aattr_\w+_\w+\z/ # skip too-many false-positive warnings in insns_info.inc. - transform_logs[:static_inline_decl] << decl_name - end - - code[decl_range] = "static inline #{decl}" - elsif (match = /#{MJITHeader::FUNC_HEADER_REGEXP}{/.match(decl)) && (header = match[0]) !~ /static/ - unless decl_name.match(MJITHeader::TARGET_NAME_REGEXP) - transform_logs[:skipped] << decl_name - next - end - - extern_names << decl_name - decl[match.begin(0)...match.end(0)] = '' - - if decl =~ /\bstatic\b/ - abort "#{PROGRAM}: a static decl was found inside external definition #{decl_name.dump}" - end - - header.sub!(/(extern|inline) /, ' ') - unless decl_name =~ /\Aattr_\w+_\w+\z/ # skip too-many false-positive warnings in insns_info.inc. - transform_logs[:static_inline_def] << decl_name - end - code[decl_range] = "static inline #{header}#{decl}" - end -end - -code << macro - -# Check the final file correctness -MJITHeader.check_code!(code, cc, cflags, 'final') - -MJITHeader.write(code, outfile) - -messages = { - def_to_decl: 'changing definition to declaration', - static_inline_def: 'making external definition static inline', - static_inline_decl: 'making declaration static inline', - skipped: 'SKIPPED to transform', -} -transform_logs.each do |key, decl_names| - decl_names = decl_names.map { |s| color.bold(s) } if color - puts("#{PROGRAM}: #{messages.fetch(key)}: #{decl_names.join(', ')}") -end diff --git a/tool/update-NEWS-gemlist.rb b/tool/update-NEWS-gemlist.rb new file mode 100755 index 0000000000..8e4d39046b --- /dev/null +++ b/tool/update-NEWS-gemlist.rb @@ -0,0 +1,41 @@ +#!/usr/bin/env ruby +require 'json' +news = File.read("NEWS.md") +prev = news[/since the \*+(\d+\.\d+\.\d+)\*+/, 1] +prevs = [prev, prev.sub(/\.\d+\z/, '')] + +update = ->(list, type, desc = "updated") do + item = ->(mark = "* ") do + "The following #{type} gem#{list.size == 1 ? ' is' : 's are'} #{desc}.\n\n" + + list.map {|g, v|"#{mark}#{g} #{v}\n"}.join("") + "\n" + end + news.sub!(/^(?:\*( +))?The following #{type} gems? (?:are|is) #{desc}\.\n+(?:(?(1) \1)\*( *).*\n)*\n*/) do + item["#{$1&.<< " "}*#{$2 || ' '}"] + end or news.sub!(/^## Stdlib updates(?:\n+The following.*(?:\n+( *\* *).*)*)*\n+\K/) do + item[$1 || "* "] + end +end +ARGV.each do |type| + last = JSON.parse(File.read("#{type}_gems.json"))['gems'].filter_map do |g| + v = g['versions'].values_at(*prevs).compact.first + g = g['gem'] + g = 'RubyGems' if g == 'rubygems' + [g, v] if v + end.to_h + changed = File.foreach("gems/#{type}_gems").filter_map do |l| + next if l.start_with?("#") + g, v = l.split(" ", 3) + next unless v + [g, v] unless last[g] == v + end + changed, added = changed.partition {|g, _| last[g]} + update[changed, type] or next + if added and !added.empty? + if type == 'bundled' + update[added, type, 'promoted from default gems'] or next + else + update[added, type, 'added'] or next + end + end + File.write("NEWS.md", news) +end diff --git a/tool/update-NEWS-refs.rb b/tool/update-NEWS-refs.rb new file mode 100644 index 0000000000..f48cac5ee1 --- /dev/null +++ b/tool/update-NEWS-refs.rb @@ -0,0 +1,38 @@ +# Usage: ruby tool/update-NEWS-refs.rb + +orig_src = File.read(File.join(__dir__, "../NEWS.md")) +lines = orig_src.lines(chomp: true) + +links = {} +while lines.last =~ %r{\A\[(.*?)\]:\s+(.*)\z} + links[$1] = $2 + lines.pop +end + +if links.empty? || lines.last != "" + raise "NEWS.md must end with a sequence of links" +end + +trackers = ["Feature", "Bug", "Misc"] +labels = links.keys.reject {|k| k.start_with?(*trackers)} +new_src = lines.join("\n").gsub(/\[?\[(#{Regexp.union(trackers)}\s+#(\d+))\]\]?/) do + links[$1] ||= "https://bugs.ruby-lang.org/issues/#$2" + "[[#$1]]" +end.gsub(/\[\[#{Regexp.union(labels)}\]\]?/) do + "[#$1]" +end.chomp + "\n\n" + +label_width = links.max_by {|k, _| k.size}.first.size + 4 +redmine_links, non_redmine_links = links.partition {|k,| k =~ /\A#{Regexp.union(trackers)}\s+#\d+\z/ } + +(redmine_links.sort_by {|k,| k[/\d+/].to_i } + non_redmine_links.reverse).each do |k, v| + new_src << "[#{k}]:".ljust(label_width) << v << "\n" +end + +if orig_src != new_src + print "Update NEWS.md? [y/N]" + $stdout.flush + if gets.chomp == "y" + File.write(File.join(__dir__, "../NEWS.md"), new_src) + end +end diff --git a/tool/update-bundled_gems.rb b/tool/update-bundled_gems.rb index bed1cfc52b..2842516cac 100755 --- a/tool/update-bundled_gems.rb +++ b/tool/update-bundled_gems.rb @@ -1,22 +1,38 @@ #!ruby -pla BEGIN { require 'rubygems' + date = nil + # STDOUT is not usable in inplace edit mode + output = $-i ? STDOUT : STDERR +} +output = STDERR if ARGF.file == STDIN +END { + output.print date.strftime("latest_date=%F") if date } unless /^[^#]/ !~ (gem = $F[0]) + ver = Gem::Version.new($F[1]) (gem, src), = Gem::SpecFetcher.fetcher.detect(:latest) {|s| s.platform == "ruby" && s.name == gem } - gem = src.fetch_spec(gem) - uri = gem.metadata["source_code_uri"] || gem.homepage - uri = uri.sub(%r[\Ahttps://github\.com/[^/]+/[^/]+\K/tree/.*], "").chomp(".git") - if $F[3] - if $F[3].include?($F[1]) - $F[3][$F[1]] = gem.version.to_s - elsif Gem::Version.new($F[1]) != gem.version and /\A\h+\z/ =~ $F[3] - $F[3..-1] = [] + if gem.version > ver + gem = src.fetch_spec(gem) + if ENV["UPDATE_BUNDLED_GEMS_ALL"] + uri = gem.metadata["source_code_uri"] || gem.homepage + uri = uri.sub(%r[\Ahttps://github\.com/[^/]+/[^/]+\K/tree/.*], "").chomp(".git") + else + uri = $F[2] + end + date = gem.date if !date or gem.date && gem.date > date + if $F[3] + if $F[3].include?($F[1]) + $F[3][$F[1]] = gem.version.to_s + elsif Gem::Version.new($F[1]) != gem.version and /\A\h+\z/ =~ $F[3] + $F[3..-1] = [] + end end + f = [gem.name, gem.version.to_s, uri, *$F[3..-1]] + $_.gsub!(/\S+\s*(?=\s|$)/) {|s| (f.shift || "").ljust(s.size)} + $_ = [$_, *f].join(" ") unless f.empty? + $_.rstrip! end - f = [gem.name, gem.version.to_s, uri, *$F[3..-1]] - $_.gsub!(/\S+\s*/) {|s| f.shift.ljust(s.size)} - $_ = [$_, *f].join(" ") unless f.empty? end diff --git a/tool/update-deps b/tool/update-deps index 27230c50e4..0b90876cd2 100755 --- a/tool/update-deps +++ b/tool/update-deps @@ -88,7 +88,6 @@ result.each {|k,v| # They can be referenced as $(top_srcdir)/filename. # % ruby -e 'def g(d) Dir.chdir(d) { Dir["**/*.{c,h,inc,dmyh}"] } end; puts((g("repo_source_dir_after_build") - g("repo_source_dir_original")).sort)' FILES_IN_SOURCE_DIRECTORY = %w[ - revision.h ] # Files built in the build directory (except extconf.h). @@ -112,6 +111,7 @@ FILES_NEED_VPATH = %w[ ext/ripper/eventids1.c ext/ripper/eventids2table.c ext/ripper/ripper.c + ext/ripper/ripper_init.c golf_prelude.c id.c id.h @@ -120,15 +120,14 @@ FILES_NEED_VPATH = %w[ known_errors.inc lex.c miniprelude.c - mjit_compile.inc newline.c node_name.inc - opt_sc.inc optinsn.inc optunifs.inc parse.c parse.h probes.dmyh + revision.h vm.inc vmtc.inc @@ -150,6 +149,16 @@ FILES_NEED_VPATH = %w[ enc/trans/single_byte.c enc/trans/utf8_mac.c enc/trans/utf_16_32.c + + prism/api_node.c + prism/ast.h + prism/diagnostic.c + prism/diagnostic.h + prism/node.c + prism/prettyprint.c + prism/serialize.c + prism/token_type.c + prism/version.h ] # Multiple files with same filename. @@ -167,13 +176,17 @@ FILES_SAME_NAME_TOP = %w[ version.h ] +# Files that may or may not exist on CI for some reason. +# Windows build generally seems to have missing dependencies. +UNSTABLE_FILES = %r{\Awin32/[^/]+\.o\z} + # Other source files exist in the source directory. def in_makefile(target, source) target = target.to_s source = source.to_s case target - when %r{\A[^/]*\z}, %r{\Acoroutine/} + when %r{\A[^/]*\z}, %r{\Acoroutine/}, %r{\Aprism/} target2 = "#{target.sub(/\.o\z/, '.$(OBJEXT)')}" case source when *FILES_IN_SOURCE_DIRECTORY then source2 = "$(top_srcdir)/#{source}" @@ -230,6 +243,10 @@ def in_makefile(target, source) else source2 = "$(top_srcdir)/#{source}" end ["#{File.dirname(target)}/depend", target2, source2] + # Files that may or may not exist on CI for some reason. + # Windows build generally seems to have missing dependencies. + when UNSTABLE_FILES + warn "warning: ignoring: #{target}" else raise "unexpected target: #{target}" end @@ -301,6 +318,7 @@ def read_make_deps(cwd) deps = deps.scan(%r{[/0-9a-zA-Z._-]+}) deps.delete_if {|dep| /\.time\z/ =~ dep} # skip timestamp next if /\.o\z/ !~ target.to_s + next if /libyjit.o\z/ =~ target.to_s # skip YJIT Rust object (no corresponding C source) next if /\.bundle\// =~ target.to_s next if /\A\./ =~ target.to_s # skip rules such as ".c.o" #p [curdir, target, deps] @@ -333,24 +351,25 @@ end # raise ArgumentError, "can not find #{filename} (hint: #{hint0})" #end -def read_single_cc_deps(path_i, cwd) +def read_single_cc_deps(path_i, cwd, fn_o) files = {} compiler_wd = nil path_i.each_line {|line| next if /\A\# \d+ "(.*)"/ !~ line dep = $1 + next if %r{\A<.*>\z} =~ dep # omit <command-line>, etc. - next if /\.erb\z/ =~ dep - compiler_wd ||= dep + next if /\.e?rb\z/ =~ dep + # gcc emits {# 1 "/absolute/directory/of/the/source/file//"} at 2nd line. + if /\/\/\z/ =~ dep + compiler_wd = Pathname(dep.sub(%r{//\z}, '')) + next + end + files[dep] = true } - # gcc emits {# 1 "/absolute/directory/of/the/source/file//"} at 2nd line. - if %r{\A/.*//\z} =~ compiler_wd - files.delete compiler_wd - compiler_wd = Pathname(compiler_wd.sub(%r{//\z}, '')) - elsif !(compiler_wd = yield) - raise "compiler working directory not found: #{path_i}" - end + compiler_wd ||= fn_o.to_s.start_with?("enc/") ? cwd : path_i.parent + deps = [] files.each_key {|dep| dep = Pathname(dep) @@ -382,13 +401,7 @@ def read_cc_deps(cwd) end path_o = cwd + fn_o path_i = cwd + fn_i - deps[path_o] = read_single_cc_deps(path_i, cwd) do - if fn_o.to_s.start_with?("enc/") - cwd - else - path_i.parent - end - end + deps[path_o] = read_single_cc_deps(path_i, cwd, fn_o) } deps end @@ -476,7 +489,7 @@ def compare_deps(make_deps, cc_deps, out=$stdout) } } - makefiles.keys.sort.each {|makefile| + makefiles.keys.compact.sort.each {|makefile| cc_lines = cc_lines_hash[makefile] || Hash.new(false) make_lines = make_lines_hash[makefile] || Hash.new(false) content = begin diff --git a/tool/ytab.sed b/tool/ytab.sed deleted file mode 100755 index 95a9b3e1eb..0000000000 --- a/tool/ytab.sed +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/sed -f -# This file is used when generating code for the Ruby parser. -/^int yydebug;/{ -i\ -#ifndef yydebug -a\ -#endif -} -/^extern int yydebug;/{ -i\ -#ifndef yydebug -a\ -#endif -} -/^yydestruct.*yymsg/,/{/{ - /^yydestruct/{ - /,$/N - /[, *]p)/!{ - H - s/^/ruby_parser_&/ - s/)$/, p)/ - /\*/s/p)$/struct parser_params *&/ - } - } - /^#endif/{ - x - /yydestruct/{ - i\ -\ struct parser_params *p; - } - x - } - /^{/{ - x - /yydestruct/{ - i\ -#define yydestruct(m, t, v) ruby_parser_yydestruct(m, t, v, p) - } - x - } -} -/^yy_stack_print /,/{/{ - /^yy_stack_print/{ - /[, *]p)/!{ - H - s/^/ruby_parser_&/ - s/)$/, p)/ - /\*/s/p)$/struct parser_params *&/ - } - } - /^#endif/{ - x - /yy_stack_print/{ - i\ -\ struct parser_params *p; - } - x - } - /^{/{ - x - /yy_stack_print/{ - i\ -#define yy_stack_print(b, t) ruby_parser_yy_stack_print(b, t, p) - } - x - } -} -/^yy_reduce_print/,/^}/{ - s/fprintf *(stderr,/YYFPRINTF (p,/g -} -s/^yysyntax_error (/&struct parser_params *p, / -s/ yysyntax_error (/&p, / -s/\( YYFPRINTF *(\)yyoutput,/\1p,/ -s/\( YYFPRINTF *(\)yyo,/\1p,/ -s/\( YYFPRINTF *(\)stderr,/\1p,/ -s/\( YYDPRINTF *((\)stderr,/\1p,/ -s/^\([ ]*\)\(yyerror[ ]*([ ]*parser,\)/\1parser_\2/ -s!^ *extern char \*getenv();!/* & */! -s/^\(#.*\)".*\.tab\.c"/\1"parse.c"/ -/^\(#.*\)".*\.y"/s:\\\\:/:g |