diff options
Diffstat (limited to 'tool')
259 files changed, 34602 insertions, 0 deletions
diff --git a/tool/asm_parse.rb b/tool/asm_parse.rb new file mode 100644 index 0000000000..32882be3ad --- /dev/null +++ b/tool/asm_parse.rb @@ -0,0 +1,53 @@ +# YARV tool to parse assembly output. + +stat = {} + +while line = ARGF.gets + if /\[start\] (\w+)/ =~ line + name = $1 + puts '--------------------------------------------------------------' + puts line + size = 0 + len = 0 + + while line = ARGF.gets + if /\[start\] (\w+)/ =~ line + puts "\t; # length: #{len}, size: #{size}" + puts "\t; # !!" + stat[name] = [len, size] + # + name = $1 + puts '--------------------------------------------------------------' + puts line + size = 0 + len = 0 + next + end + + unless /(\ALM)|(\ALB)|(\A\.)|(\A\/)/ =~ line + puts line + if /\[length = (\d+)\]/ =~ line + len += $1.to_i + size += 1 + end + end + + + if /__NEXT_INSN__/ !~ line && /\[end \] (\w+)/ =~ line + ename = $1 + if name != ename + puts "!! start with #{name}, but end with #{ename}" + end + stat[ename] = [len, size] + puts "\t; # length: #{len}, size: #{size}" + break + end + end + end +end + +stat.sort_by{|a, b| -b[0] * 1000 - a[0]}.each{|a, b| + puts "#{a}\t#{b.join("\t")}" +} +puts "total length :\t#{stat.inject(0){|r, e| r+e[1][0]}}" +puts "total size :\t#{stat.inject(0){|r, e| r+e[1][1]}}" diff --git a/tool/bisect.sh b/tool/bisect.sh new file mode 100755 index 0000000000..dfc3a64041 --- /dev/null +++ b/tool/bisect.sh @@ -0,0 +1,65 @@ +#!/bin/sh +# usage: +# edit $(srcdir)/test.rb +# git bisect start <bad> <good> +# cd <builddir> +# make bisect (or bisect-ruby for full ruby) + +if [ "x" = "x$MAKE" ]; then + MAKE=make +fi + +case $1 in + miniruby | ruby ) # (miniruby|ruby) <srcdir> + srcdir="$2" + builddir=`pwd` # assume pwd is builddir + path="$builddir/_bisect.sh" + echo "path: $path" + cp "$0" "$path" + cd "$srcdir" + set -x + exec git bisect run "$path" "run-$1" + ;; + run-miniruby ) + prep=mini + run=run + ;; + run-ruby ) + prep=program + run=runruby + ;; + "" ) + echo missing command 1>&2 + exit 1 + ;; + * ) + echo unknown command "'$1'" 1>&2 + exit 1 + ;; +esac + +# Apply $(srcdir)/bisect.patch to build if exists +# e.g., needs 5c2508060b~2..5c2508060b to use Bison 3.5.91. +if [ -f bisect.patch ]; then + if ! patch -p1 -N < bisect.patch || git diff --no-patch --exit-code; then + exit 125 + fi + git status + exec= +else + exec=exec +fi + +case "$0" in +*/*) + # assume a copy of this script is in builddir + cd `echo "$0" | sed 's:\(.*\)/.*:\1:'` || exit 125 + ;; +esac +for target in srcs Makefile $prep; do + $MAKE $target || exit 125 +done +$exec $MAKE $run +status=$? +git checkout -f HEAD +exit $status diff --git a/tool/build-transcode b/tool/build-transcode new file mode 100755 index 0000000000..fa71155530 --- /dev/null +++ b/tool/build-transcode @@ -0,0 +1,16 @@ +#!/bin/sh + +[ "$1" -a -d "$1" ] && { cd "$1" || exit $?; } && shift +[ "$#" = 0 ] && set enc/trans/*.trans +for src; do + case "$src" in + *.trans) + c="`dirname $src`/`basename $src .trans`.c" + ${BASERUBY-ruby} tool/transcode-tblgen.rb -vo "$c" "$src" + ;; + *) + echo "$0: don't know how to deal with $src" + continue + ;; + esac +done diff --git a/tool/bundler/dev_gems.rb b/tool/bundler/dev_gems.rb new file mode 100644 index 0000000000..5b3ab24f5b --- /dev/null +++ b/tool/bundler/dev_gems.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +gem "rdoc", "6.2.0" # 6.2.1 is required > Ruby 2.3 +gem "test-unit", "~> 3.0" +gem "rake", "~> 13.0" + +gem "webrick", "~> 1.6" +gem "parallel_tests", "~> 2.29" +gem "parallel", "1.19.2" # 1.20+ is required > Ruby 2.3 +gem "rspec-core", "~> 3.8" +gem "rspec-expectations", "~> 3.8" +gem "rspec-mocks", "~> 3.8" +gem "uri", "~> 0.10.1" + +group :doc do + gem "ronn", "~> 0.7.3", :platform => :ruby +end diff --git a/tool/bundler/dev_gems.rb.lock b/tool/bundler/dev_gems.rb.lock new file mode 100644 index 0000000000..9787994e95 --- /dev/null +++ b/tool/bundler/dev_gems.rb.lock @@ -0,0 +1,57 @@ +GEM + remote: https://rubygems.org/ + specs: + diff-lcs (1.5.0) + hpricot (0.8.6) + hpricot (0.8.6-java) + mustache (1.1.1) + parallel (1.19.2) + parallel_tests (2.32.0) + parallel + power_assert (2.0.3) + rake (13.1.0) + rdiscount (2.2.7.1) + rdoc (6.2.0) + ronn (0.7.3) + hpricot (>= 0.8.2) + mustache (>= 0.7.0) + rdiscount (>= 1.5.8) + rspec-core (3.12.2) + rspec-support (~> 3.12.0) + rspec-expectations (3.12.3) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-mocks (3.12.6) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-support (3.12.1) + test-unit (3.6.1) + power_assert + uri (0.10.3) + webrick (1.8.1) + +PLATFORMS + java + ruby + universal-java-11 + universal-java-18 + x64-mingw-ucrt + x64-mingw32 + x86_64-darwin-20 + x86_64-linux + +DEPENDENCIES + parallel (= 1.19.2) + parallel_tests (~> 2.29) + rake (~> 13.0) + rdoc (= 6.2.0) + ronn (~> 0.7.3) + rspec-core (~> 3.8) + rspec-expectations (~> 3.8) + rspec-mocks (~> 3.8) + test-unit (~> 3.0) + uri (~> 0.10.1) + webrick (~> 1.6) + +BUNDLED WITH + 2.3.27 diff --git a/tool/bundler/rubocop_gems.rb b/tool/bundler/rubocop_gems.rb new file mode 100644 index 0000000000..25408b70f3 --- /dev/null +++ b/tool/bundler/rubocop_gems.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +gem "rubocop", "1.31.0" +gem "parser", "3.2.2.2" + +gem "minitest" +gem "rake" +gem "rake-compiler" +gem "rspec" +gem "test-unit" diff --git a/tool/bundler/rubocop_gems.rb.lock b/tool/bundler/rubocop_gems.rb.lock new file mode 100644 index 0000000000..fd7a290019 --- /dev/null +++ b/tool/bundler/rubocop_gems.rb.lock @@ -0,0 +1,70 @@ +GEM + remote: https://rubygems.org/ + specs: + ast (2.4.2) + diff-lcs (1.5.0) + minitest (5.20.0) + parallel (1.23.0) + parser (3.2.2.2) + ast (~> 2.4.1) + power_assert (2.0.3) + rainbow (3.1.1) + rake (13.1.0) + rake-compiler (1.2.5) + rake + regexp_parser (2.8.2) + rexml (3.2.6) + rspec (3.12.0) + rspec-core (~> 3.12.0) + rspec-expectations (~> 3.12.0) + rspec-mocks (~> 3.12.0) + rspec-core (3.12.2) + rspec-support (~> 3.12.0) + rspec-expectations (3.12.3) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-mocks (3.12.6) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-support (3.12.1) + rubocop (1.31.0) + parallel (~> 1.10) + parser (>= 3.1.0.0) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.8, < 3.0) + rexml (>= 3.2.5, < 4.0) + rubocop-ast (>= 1.18.0, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 1.4.0, < 3.0) + rubocop-ast (1.30.0) + parser (>= 3.2.1.0) + ruby-progressbar (1.13.0) + test-unit (3.6.1) + power_assert + unicode-display_width (2.5.0) + +PLATFORMS + aarch64-linux + arm64-darwin-20 + arm64-darwin-21 + arm64-darwin-22 + arm64-darwin-23 + universal-java-11 + universal-java-18 + x64-mingw-ucrt + x86_64-darwin-19 + x86_64-darwin-20 + x86_64-darwin-21 + x86_64-linux + +DEPENDENCIES + minitest + parser (= 3.2.2.2) + rake + rake-compiler + rspec + rubocop (= 1.31.0) + test-unit + +BUNDLED WITH + 2.3.27 diff --git a/tool/bundler/standard_gems.rb b/tool/bundler/standard_gems.rb new file mode 100644 index 0000000000..34cdfa2a61 --- /dev/null +++ b/tool/bundler/standard_gems.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +gem "standard", "1.12.1" +gem "parser", "3.2.2.2" + +gem "minitest" +gem "rake" +gem "rake-compiler" +gem "rspec" +gem "test-unit" diff --git a/tool/bundler/standard_gems.rb.lock b/tool/bundler/standard_gems.rb.lock new file mode 100644 index 0000000000..b3861729e6 --- /dev/null +++ b/tool/bundler/standard_gems.rb.lock @@ -0,0 +1,76 @@ +GEM + remote: https://rubygems.org/ + specs: + ast (2.4.2) + diff-lcs (1.5.0) + minitest (5.20.0) + parallel (1.23.0) + parser (3.2.2.2) + ast (~> 2.4.1) + power_assert (2.0.3) + rainbow (3.1.1) + rake (13.1.0) + rake-compiler (1.2.5) + rake + regexp_parser (2.8.2) + rexml (3.2.6) + rspec (3.12.0) + rspec-core (~> 3.12.0) + rspec-expectations (~> 3.12.0) + rspec-mocks (~> 3.12.0) + rspec-core (3.12.2) + rspec-support (~> 3.12.0) + rspec-expectations (3.12.3) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-mocks (3.12.6) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-support (3.12.1) + rubocop (1.29.1) + parallel (~> 1.10) + parser (>= 3.1.0.0) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.8, < 3.0) + rexml (>= 3.2.5, < 4.0) + rubocop-ast (>= 1.17.0, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 1.4.0, < 3.0) + rubocop-ast (1.30.0) + parser (>= 3.2.1.0) + rubocop-performance (1.13.3) + rubocop (>= 1.7.0, < 2.0) + rubocop-ast (>= 0.4.0) + ruby-progressbar (1.13.0) + standard (1.12.1) + rubocop (= 1.29.1) + rubocop-performance (= 1.13.3) + test-unit (3.6.1) + power_assert + unicode-display_width (2.5.0) + +PLATFORMS + aarch64-linux + arm64-darwin-20 + arm64-darwin-21 + arm64-darwin-22 + arm64-darwin-23 + universal-java-11 + universal-java-18 + x64-mingw-ucrt + x86_64-darwin-19 + x86_64-darwin-20 + x86_64-darwin-21 + x86_64-linux + +DEPENDENCIES + minitest + parser (= 3.2.2.2) + rake + rake-compiler + rspec + standard (= 1.12.1) + test-unit + +BUNDLED WITH + 2.3.27 diff --git a/tool/bundler/test_gems.rb b/tool/bundler/test_gems.rb new file mode 100644 index 0000000000..c848ade9c7 --- /dev/null +++ b/tool/bundler/test_gems.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +gem "rack", "~> 2.0" +gem "webrick", "1.7.0" +gem "rack-test", "~> 1.1" +gem "artifice", "~> 0.6.0" +gem "compact_index", "~> 0.13.0" +gem "sinatra", "~> 2.0" +# for Ruby 2.6. bundler/spec/install/gemfile/sources_spec.rb is failed with sinatra-2.0.8.1 and tilt-2.2.0. +gem "tilt", "~> 2.0.11" +gem "rake", "13.0.1" +gem "builder", "~> 3.2" diff --git a/tool/bundler/test_gems.rb.lock b/tool/bundler/test_gems.rb.lock new file mode 100644 index 0000000000..ad8fed8044 --- /dev/null +++ b/tool/bundler/test_gems.rb.lock @@ -0,0 +1,47 @@ +GEM + remote: https://rubygems.org/ + specs: + artifice (0.6) + rack-test + builder (3.2.4) + compact_index (0.13.0) + mustermann (2.0.2) + ruby2_keywords (~> 0.0.1) + rack (2.2.8) + rack-protection (2.2.4) + rack + rack-test (1.1.0) + rack (>= 1.0, < 3) + rake (13.0.1) + ruby2_keywords (0.0.5) + sinatra (2.2.4) + mustermann (~> 2.0) + rack (~> 2.2) + rack-protection (= 2.2.4) + tilt (~> 2.0) + tilt (2.0.11) + webrick (1.7.0) + +PLATFORMS + java + ruby + universal-java-11 + universal-java-18 + x64-mingw-ucrt + x64-mingw32 + x86_64-darwin-20 + x86_64-linux + +DEPENDENCIES + artifice (~> 0.6.0) + builder (~> 3.2) + compact_index (~> 0.13.0) + rack (~> 2.0) + rack-test (~> 1.1) + rake (= 13.0.1) + sinatra (~> 2.0) + tilt (~> 2.0.11) + webrick (= 1.7.0) + +BUNDLED WITH + 2.3.27 diff --git a/tool/checksum.rb b/tool/checksum.rb new file mode 100755 index 0000000000..bcc60ee14a --- /dev/null +++ b/tool/checksum.rb @@ -0,0 +1,72 @@ +#!ruby + +require_relative 'lib/vpath' + +class Checksum + def initialize(vpath) + @vpath = vpath + end + + attr_reader :source, :target + + def source=(source) + @source = source + @checksum = File.basename(source, ".*") + ".chksum" + end + + def target=(target) + @target = target + end + + def update? + src = @vpath.read(@source) + @len = src.length + @sum = src.sum + return false unless @vpath.search(File.method(:exist?), @target) + begin + data = @vpath.read(@checksum) + rescue + return false + else + return false unless data[/src="([0-9a-z_.-]+)",/, 1] == @source + return false unless @len == data[/\blen=(\d+)/, 1].to_i + return false unless @sum == data[/\bchecksum=(\d+)/, 1].to_i + return true + end + end + + def update! + open(@checksum, "wb") {|f| + f.puts("src=\"#{@source}\", len=#{@len}, checksum=#{@sum}") + } + end + + def update + return true if update? + update! if ret = yield(self) + ret + end + + def copy(name) + @vpath.open(name, "rb") {|f| + IO.copy_stream(f, name) + } + true + end + + def make(*args) + system(@make, *args) + end + + def def_options(opt = (require 'optparse'; OptionParser.new)) + @vpath.def_options(opt) + opt.on("--make=PATH") {|v| @make = v} + opt + end + + def self.update(argv) + k = new(VPath.new) + k.source, k.target, *argv = k.def_options.parse(*argv) + k.update {|_| yield(_, *argv)} + end +end diff --git a/tool/ci_functions.sh b/tool/ci_functions.sh new file mode 100644 index 0000000000..7066bbe4ec --- /dev/null +++ b/tool/ci_functions.sh @@ -0,0 +1,29 @@ +# -*- BASH -*- +# Manage functions used on a CI. +# Run `. tool/ci_functions.sh` to use it. + +# Create options with patterns `-n !/name1/ -n !/name2/ ..` to exclude the test +# method names by the method names `name1 name2 ..`. +# See `ruby tool/test/runner.rb --help` `-n` option. +function ci_to_excluded_test_opts { + local tests_str="${1}" + # Use the backward matching `!/name$/`, as the perfect matching doesn't work. + # https://bugs.ruby-lang.org/issues/16936 + ruby <<EOF + opts = "${tests_str}".split.map { |test| "-n \!/#{test}\$$/" } + puts opts.join(' ') +EOF + return 0 +} + +# Create options with patterns `-n name1 -n name2 ..` to include the test +# method names by the method names `name1 name2 ..`. +# See `ruby tool/test/runner.rb --help` `-n` option. +function ci_to_included_test_opts { + local tests_str="${1}" + ruby <<EOF + opts = "${tests_str}".split.map { |test| "-n #{test}" } + puts opts.join(' ') +EOF + return 0 +} diff --git a/tool/colors b/tool/colors new file mode 100644 index 0000000000..a65c326ade --- /dev/null +++ b/tool/colors @@ -0,0 +1,3 @@ +pass=36;7 +fail=31;1;7 +skip=33;1 diff --git a/tool/darwin-cc b/tool/darwin-cc new file mode 100755 index 0000000000..6eee96e435 --- /dev/null +++ b/tool/darwin-cc @@ -0,0 +1,6 @@ +#!/bin/bash +exec 2> >(exec grep -v \ + -e '^ld: warning: The [a-z0-9_][a-z0-9_]* architecture is deprecated for macOS' \ + -e '^ld: warning: text-based stub file /System/Library/Frameworks/' \ + >&2) +exec "$@" diff --git a/tool/disable_ipv6.sh b/tool/disable_ipv6.sh new file mode 100755 index 0000000000..ce1cc0da68 --- /dev/null +++ b/tool/disable_ipv6.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -ex +sysctl -w net.ipv6.conf.all.disable_ipv6=1 +sysctl -w net.ipv6.conf.default.disable_ipv6=1 +sysctl -w net.ipv6.conf.lo.disable_ipv6=1 + +cat /etc/hosts +ruby -e "hosts = File.read('/etc/hosts').sub(/^::1\s*localhost.*$/, ''); File.write('/etc/hosts', hosts)" +cat /etc/hosts diff --git a/tool/downloader.rb b/tool/downloader.rb new file mode 100644 index 0000000000..d3a9f75637 --- /dev/null +++ b/tool/downloader.rb @@ -0,0 +1,415 @@ +# Used by configure and make to download or update mirrored Ruby and GCC +# files. This will use HTTPS if possible, falling back to HTTP. + +require 'fileutils' +require 'open-uri' +require 'pathname' +begin + require 'net/https' +rescue LoadError + https = 'http' +else + https = 'https' + + # open-uri of ruby 2.2.0 accepts an array of PEMs as ssl_ca_cert, but old + # versions do not. so, patching OpenSSL::X509::Store#add_file instead. + class OpenSSL::X509::Store + alias orig_add_file add_file + def add_file(pems) + Array(pems).each do |pem| + if File.directory?(pem) + add_path pem + else + orig_add_file pem + end + end + end + end + # since open-uri internally checks ssl_ca_cert using File.directory?, + # allow to accept an array. + class <<File + alias orig_directory? directory? + def File.directory? files + files.is_a?(Array) ? false : orig_directory?(files) + end + end +end + +class Downloader + def self.https=(https) + @@https = https + end + + def self.https? + @@https == 'https' + end + + def self.https + @@https + end + + class GNU < self + def self.download(name, *rest) + if https? + begin + super("https://cdn.jsdelivr.net/gh/gcc-mirror/gcc@master/#{name}", name, *rest) + rescue => e + STDERR.puts "Download failed (#{e.message}), try another URL" + super("https://raw.githubusercontent.com/gcc-mirror/gcc/master/#{name}", name, *rest) + end + else + super("https://repo.or.cz/official-gcc.git/blob_plain/HEAD:/#{name}", name, *rest) + end + end + end + + class RubyGems < self + def self.download(name, dir = nil, since = true, options = {}) + require 'rubygems' + options = options.dup + options[:ssl_ca_cert] = Dir.glob(File.expand_path("../lib/rubygems/ssl_certs/**/*.pem", File.dirname(__FILE__))) + super("https://rubygems.org/downloads/#{name}", name, dir, since, options) + end + end + + Gems = RubyGems + + class Unicode < self + INDEX = {} # cache index file information across files in the same directory + UNICODE_PUBLIC = "https://www.unicode.org/Public/" + + def self.download(name, dir = nil, since = true, options = {}) + options = options.dup + unicode_beta = options.delete(:unicode_beta) + name_dir_part = name.sub(/[^\/]+$/, '') + if unicode_beta == 'YES' + if INDEX.size == 0 + index_options = options.dup + index_options[:cache_save] = false # TODO: make sure caching really doesn't work for index file + index_data = File.read(under(dir, "index.html")) rescue nil + index_file = super(UNICODE_PUBLIC+name_dir_part, "#{name_dir_part}index.html", dir, true, index_options) + INDEX[:index] = File.read(index_file) + since = true unless INDEX[:index] == index_data + end + file_base = File.basename(name, '.txt') + return if file_base == '.' # Use pre-generated headers and tables + beta_name = INDEX[:index][/#{Regexp.quote(file_base)}(-[0-9.]+d\d+)?\.txt/] + # make sure we always check for new versions of files, + # because they can easily change in the beta period + super(UNICODE_PUBLIC+name_dir_part+beta_name, name, dir, since, options) + else + index_file = Pathname.new(under(dir, name_dir_part+'index.html')) + if index_file.exist? and name_dir_part !~ /^(12\.1\.0|emoji\/12\.0)/ + raise "Although Unicode is not in beta, file #{index_file} exists. " + + "Remove all files in this directory and in .downloaded-cache/ " + + "because they may be leftovers from the beta period." + end + super(UNICODE_PUBLIC+name, name, dir, since, options) + end + end + end + + def self.mode_for(data) + /\A#!/ =~ data ? 0755 : 0644 + end + + def self.http_options(file, since) + options = {} + if since + case since + when true + since = (File.mtime(file).httpdate rescue nil) + when Time + since = since.httpdate + end + if since + options['If-Modified-Since'] = since + end + end + options['Accept-Encoding'] = 'identity' # to disable Net::HTTP::GenericRequest#decode_content + options + end + + def self.httpdate(date) + Time.httpdate(date) + rescue ArgumentError => e + # Some hosts (e.g., zlib.net) return similar to RFC 850 but 4 + # digit year, sometimes. + /\A\s* + (?:Mon|Tues|Wednes|Thurs|Fri|Satur|Sun)day,\x20 + (\d\d)-(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-(\d{4})\x20 + (\d\d):(\d\d):(\d\d)\x20 + GMT + \s*\z/ix =~ date or raise + warn e.message + Time.utc($3, $2, $1, $4, $5, $6) + end + + # Downloader.download(url, name, [dir, [since]]) + # + # Update a file from url if newer version is available. + # Creates the file if the file doesn't yet exist; however, the + # directory where the file is being created has to exist already. + # The +since+ parameter can take the following values, with associated meanings: + # true :: + # Take the last-modified time of the current file on disk, and only download + # if the server has a file that was modified later. Download unconditionally + # if we don't have the file yet. Default. + # +some time value+ :: + # Use this time value instead of the time of modification of the file on disk. + # nil :: + # Only download the file if it doesn't exist yet. + # false :: + # always download url regardless of whether we already have a file, + # and regardless of modification times. (This is essentially just a waste of + # network resources, except in the case that the file we have is somehow damaged. + # Please note that using this recurringly might create or be seen as a + # denial of service attack.) + # + # Example usage: + # download 'http://www.unicode.org/Public/UCD/latest/ucd/UnicodeData.txt', + # 'UnicodeData.txt', 'enc/unicode/data' + def self.download(url, name, dir = nil, since = true, options = {}) + options = options.dup + url = URI(url) + dryrun = options.delete(:dryrun) + options.delete(:unicode_beta) # just to be on the safe side for gems and gcc + + if name + file = Pathname.new(under(dir, name)) + else + name = File.basename(url.path) + end + cache_save = options.delete(:cache_save) { + ENV["CACHE_SAVE"] != "no" + } + cache = cache_file(url, name, options.delete(:cache_dir)) + file ||= cache + if since.nil? and file.exist? + if $VERBOSE + $stdout.puts "#{file} already exists" + $stdout.flush + end + if cache_save + save_cache(cache, file, name) + end + return file.to_path + end + if dryrun + puts "Download #{url} into #{file}" + return + end + if link_cache(cache, file, name, $VERBOSE) + return file.to_path + end + if !https? and URI::HTTPS === url + warn "*** using http instead of https ***" + url.scheme = 'http' + url = URI(url.to_s) + end + if $VERBOSE + $stdout.print "downloading #{name} ... " + $stdout.flush + end + mtime = nil + options = options.merge(http_options(file, since.nil? ? true : since)) + begin + data = with_retry(10) do + data = url.read(options) + if mtime = data.meta["last-modified"] + mtime = Time.httpdate(mtime) + end + data + end + rescue OpenURI::HTTPError => http_error + if http_error.message =~ /^304 / # 304 Not Modified + if $VERBOSE + $stdout.puts "#{name} not modified" + $stdout.flush + end + return file.to_path + end + raise + rescue Timeout::Error + if since.nil? and file.exist? + puts "Request for #{url} timed out, using old version." + return file.to_path + end + raise + rescue SocketError + if since.nil? and file.exist? + puts "No network connection, unable to download #{url}, using old version." + return file.to_path + end + raise + end + dest = (cache_save && cache && !cache.exist? ? cache : file) + dest.parent.mkpath + dest.open("wb", 0600) do |f| + f.write(data) + f.chmod(mode_for(data)) + end + if mtime + dest.utime(mtime, mtime) + end + if $VERBOSE + $stdout.puts "done" + $stdout.flush + end + if dest.eql?(cache) + link_cache(cache, file, name) + elsif cache_save + save_cache(cache, file, name) + end + return file.to_path + rescue => e + raise "failed to download #{name}\n#{e.class}: #{e.message}: #{url}" + end + + def self.under(dir, name) + dir ? File.join(dir, File.basename(name)) : name + end + + def self.cache_file(url, name, cache_dir = nil) + case cache_dir + when false + return nil + when nil + cache_dir = ENV['CACHE_DIR'] + if !cache_dir or cache_dir.empty? + cache_dir = ".downloaded-cache" + end + end + Pathname.new(cache_dir) + (name || File.basename(URI(url).path)) + end + + def self.link_cache(cache, file, name, verbose = false) + return false unless cache and cache.exist? + return true if cache.eql?(file) + if /cygwin/ !~ RUBY_PLATFORM or /winsymlink:nativestrict/ =~ ENV['CYGWIN'] + begin + file.make_symlink(cache.relative_path_from(file.parent)) + rescue SystemCallError + else + if verbose + $stdout.puts "made symlink #{name} to #{cache}" + $stdout.flush + end + return true + end + end + begin + file.make_link(cache) + rescue SystemCallError + else + if verbose + $stdout.puts "made link #{name} to #{cache}" + $stdout.flush + end + return true + end + end + + def self.save_cache(cache, file, name) + return unless cache or cache.eql?(file) + begin + st = cache.stat + rescue + begin + file.rename(cache) + rescue + return + end + else + return unless st.mtime > file.lstat.mtime + file.unlink + end + link_cache(cache, file, name) + end + + def self.with_retry(max_times, &block) + times = 0 + begin + block.call + rescue Errno::ETIMEDOUT, SocketError, OpenURI::HTTPError, Net::ReadTimeout, Net::OpenTimeout, ArgumentError => e + raise if e.is_a?(OpenURI::HTTPError) && e.message !~ /^50[023] / # retry only 500, 502, 503 for http error + times += 1 + if times <= max_times + $stderr.puts "retrying #{e.class} (#{e.message}) after #{times ** 2} seconds..." + sleep(times ** 2) + retry + else + raise + end + end + end + private_class_method :with_retry +end + +Downloader.https = https.freeze + +if $0 == __FILE__ + since = true + options = {} + until ARGV.empty? + case ARGV[0] + when '-d' + destdir = ARGV[1] + ARGV.shift + when '-p' + # strip directory names from the name to download, and add the + # prefix instead. + prefix = ARGV[1] + ARGV.shift + when '-e' + since = nil + when '-a' + since = false + when '-n', '--dryrun' + options[:dryrun] = true + when '--cache-dir' + options[:cache_dir] = ARGV[1] + ARGV.shift + when '--unicode-beta' + options[:unicode_beta] = ARGV[1] + ARGV.shift + when /\A--cache-dir=(.*)/m + options[:cache_dir] = $1 + when /\A-/ + abort "#{$0}: unknown option #{ARGV[0]}" + else + break + end + ARGV.shift + end + dl = Downloader.constants.find do |name| + ARGV[0].casecmp(name.to_s) == 0 + end unless ARGV.empty? + $VERBOSE = true + if dl + dl = Downloader.const_get(dl) + ARGV.shift + ARGV.each do |name| + dir = destdir + if prefix + name = name.sub(/\A\.\//, '') + destdir2 = destdir.sub(/\A\.\//, '') + if name.start_with?(destdir2+"/") + name = name[(destdir2.size+1)..-1] + if (dir = File.dirname(name)) == '.' + dir = destdir + else + dir = File.join(destdir, dir) + end + else + name = File.basename(name) + end + name = "#{prefix}/#{name}" + end + dl.download(name, dir, since, options) + end + else + abort "usage: #{$0} url name" unless ARGV.size == 2 + Downloader.download(ARGV[0], ARGV[1], destdir, since, options) + end +end diff --git a/tool/enc-emoji-citrus-gen.rb b/tool/enc-emoji-citrus-gen.rb new file mode 100644 index 0000000000..da9c8a6b62 --- /dev/null +++ b/tool/enc-emoji-citrus-gen.rb @@ -0,0 +1,131 @@ +require File.expand_path('../lib/jisx0208', __FILE__) + +ENCODES = [ + { + :name => "SHIFT_JIS-DOCOMO", + :src_zone => [0xF8..0xFC, 0x40..0xFC, 8], + :dst_ilseq => 0xFFFE, + :map => [ + [0xE63E..0xE757, JISX0208::Char.from_sjis(0xF89F)], + ], + }, + { + :name => "ISO-2022-JP-KDDI", + :src_zone => [0x21..0x7E, 0x21..0x7E, 8], + :dst_ilseq => 0xFFFE, + :map => [ + [0xE468..0xE5B4, JISX0208::Char.new(0x7521)], + [0xE5B5..0xE5CC, JISX0208::Char.new(0x7867)], + [0xE5CD..0xE5DF, JISX0208::Char.new(0x7921)], + [0xEA80..0xEAFA, JISX0208::Char.new(0x7934)], + [0xEAFB..0xEB0D, JISX0208::Char.new(0x7854)], + [0xEB0E..0xEB8E, JISX0208::Char.new(0x7A51)], + ], + }, + { + :name => "SHIFT_JIS-KDDI", + :src_zone => [0xF3..0xFC, 0x40..0xFC, 8], + :dst_ilseq => 0xFFFE, + :map => [ + [0xE468..0xE5B4, JISX0208::Char.from_sjis(0xF640)], + [0xE5B5..0xE5CC, JISX0208::Char.from_sjis(0xF7E5)], + [0xE5CD..0xE5DF, JISX0208::Char.from_sjis(0xF340)], + [0xEA80..0xEAFA, JISX0208::Char.from_sjis(0xF353)], + [0xEAFB..0xEB0D, JISX0208::Char.from_sjis(0xF7D2)], + [0xEB0E..0xEB8E, JISX0208::Char.from_sjis(0xF3CF)], + ], + }, + { + :name => "SHIFT_JIS-SOFTBANK", + :src_zone => [0xF3..0xFC, 0x40..0xFC, 8], + :dst_ilseq => 0xFFFE, + :map => [ + [0xE001..0xE05A, JISX0208::Char.from_sjis(0xF941)], + [0xE101..0xE15A, JISX0208::Char.from_sjis(0xF741)], + [0xE201..0xE25A, JISX0208::Char.from_sjis(0xF7A1)], + [0xE301..0xE34D, JISX0208::Char.from_sjis(0xF9A1)], + [0xE401..0xE44C, JISX0208::Char.from_sjis(0xFB41)], + [0xE501..0xE53E, JISX0208::Char.from_sjis(0xFBA1)], + ], + }, +] + +def zone(*args) + bits = args.pop + [*args.map{|range| "0x%02X-0x%02X" % [range.begin, range.end] }, bits].join(' / ') +end + +def header(params) + (<<END_HEADER_TEMPLATE % [params[:name], zone(*params[:src_zone]), params[:dst_ilseq]]) +# DO NOT EDIT THIS FILE DIRECTLY + +TYPE ROWCOL +NAME %s +SRC_ZONE %s +OOB_MODE ILSEQ +DST_ILSEQ 0x%04X +DST_UNIT_BITS 16 +END_HEADER_TEMPLATE +end + +def generate_to_ucs(params, pairs) + pairs.sort_by! {|u, c| c } + name = "EMOJI_#{params[:name]}%UCS" + open("#{name}.src", "w") do |io| + io.print header(params.merge(name: name.tr('%', '/'))) + io.puts + io.puts "BEGIN_MAP" + io.print pairs.inject("") {|acc, uc| acc += "0x%04X = 0x%04X\n" % uc.reverse } + io.puts "END_MAP" + end +end + +def generate_from_ucs(params, pairs) + pairs.sort_by! {|u, c| u } + name = "UCS%EMOJI_#{params[:name]}" + open("#{name}.src", "w") do |io| + io.print header(params.merge(name: name.tr('%', '/'))) + io.puts + io.puts "BEGIN_MAP" + io.print pairs.inject("") {|acc, uc| acc += "0x%04X = 0x%04X\n" % uc } + io.puts "END_MAP" + end +end + +def make_pairs(code_map) + code_map.inject([]) {|acc, (range, ch)| + acc += range.map{|uni| pair = [uni, Integer(ch)]; ch = ch.succ; next pair } + } +end + +ENCODES.each do |params| + pairs = make_pairs(params[:map], ¶ms[:conv]) + generate_to_ucs(params, pairs) + generate_from_ucs(params, pairs) +end + +# generate KDDI-UNDOC for Shift_JIS-KDDI +kddi_sjis_map = ENCODES.select{|enc| enc[:name] == "SHIFT_JIS-KDDI"}.first[:map] +pairs = kddi_sjis_map.inject([]) {|acc, (range, ch)| + acc += range.map{|uni| pair = [ch.to_sjis - 0x700, Integer(ch)]; ch = ch.succ; next pair } +} +params = { + :name => "SHIFT_JIS-KDDI-UNDOC", + :src_zone => [0xF3..0xFC, 0x40..0xFC, 8], + :dst_ilseq => 0xFFFE, +} +generate_from_ucs(params, pairs) +generate_to_ucs(params, pairs) + +# generate KDDI-UNDOC for ISO-2022-JP-KDDI +kddi_2022_map = ENCODES.select{|enc| enc[:name] == "ISO-2022-JP-KDDI"}.first[:map] +pairs = kddi_2022_map.each_with_index.inject([]) {|acc, ((range, ch), i)| + sjis = kddi_sjis_map[i][1] + acc += range.map{|uni| pair = [sjis.to_sjis - 0x700, Integer(ch)]; ch = ch.succ; sjis = sjis.succ; next pair } +} +params = { + :name => "ISO-2022-JP-KDDI-UNDOC", + :src_zone => [0x21..0x7E, 0x21..0x7E, 8], + :dst_ilseq => 0xFFFE, +} +generate_from_ucs(params, pairs) diff --git a/tool/enc-emoji4unicode.rb b/tool/enc-emoji4unicode.rb new file mode 100644 index 0000000000..1e7d45901f --- /dev/null +++ b/tool/enc-emoji4unicode.rb @@ -0,0 +1,133 @@ +#!/usr/bin/env ruby + +# example: +# ./enc-emoji4unicode.rb emoji4unicode.xml > ../enc/trans/emoji-exchange-tbl.rb + +require 'rexml/document' +require File.expand_path("../transcode-tblgen", __FILE__) + +class EmojiTable + VERBOSE_MODE = false + + def initialize(xml_path) + @doc = REXML::Document.new File.open(xml_path) + @kddi_undoc = make_kddi_undoc_map() + end + + def conversion(from_carrier, to_carrier, &block) + REXML::XPath.each(@doc.root, '//e') do |e| + from = e.attribute(from_carrier.downcase).to_s + to = e.attribute(to_carrier.downcase).to_s + text_fallback = e.attribute('text_fallback').to_s + name = e.attribute('name').to_s + if from =~ /^(?:\*|\+)(.+)$/ # proposed or unified + from = $1 + end + if from.empty? || from !~ /^[0-9A-F]+$/ + # do nothing + else + from_utf8 = [from.hex].pack("U").unpack("H*").first + if to =~ /^(?:>|\*)?([0-9A-F\+]+)$/ + str_to = $1 + if str_to =~ /^\+/ # unicode "proposed" begins at "+" + proposal = true + str_to.sub!(/^\+/, '') + else + proposal = false + end + tos = str_to.split('+') + to_utf8 = tos.map(&:hex).pack("U*").unpack("H*").first + comment = "[%s] U+%X -> %s" % [name, from.hex, tos.map{|c| "U+%X"%c.hex}.join(' ')] + block.call(:from => from_utf8, + :to => to_utf8, + :comment => comment, + :fallback => false, + :proposal => proposal) + elsif to.empty? + if text_fallback.empty? + comment = "[%s] U+%X -> U+3013 (GETA)" % [name, from.hex] + block.call(:from => from_utf8, + :to => "\u{3013}".unpack("H*").first, + :comment => comment, # geta + :fallback => true, + :proposal => false) + else + to_utf8 = text_fallback.unpack("H*").first + comment = %([%s] U+%X -> "%s") % [name, from.hex, text_fallback] + block.call(:from => from_utf8, + :to => to_utf8, + :comment => comment, + :fallback => true, + :proposal => false) + end + else + raise "something wrong: %s -> %s" % [from, to] + end + end + end + end + + def generate(io, from_carrier, to_carrier) + from_encoding = (from_carrier == "Unicode") ? "UTF-8" : "UTF8-"+from_carrier + to_encoding = (to_carrier == "Unicode" ) ? "UTF-8" : "UTF8-"+to_carrier + io.puts "EMOJI_EXCHANGE_TBL['#{from_encoding}']['#{to_encoding}'] = [" + io.puts " # for documented codepoints" if from_carrier == "KDDI" + self.conversion(from_carrier, to_carrier) do |params| + from, to = params[:from], %Q{"#{params[:to]}"} + to = ":undef" if params[:fallback] || params[:proposal] + io.puts %{ ["#{from}", #{to}], # #{params[:comment]}} + end + if from_carrier == "KDDI" + io.puts " # for undocumented codepoints" + self.conversion(from_carrier, to_carrier) do |params| + from, to = params[:from], %Q{"#{params[:to]}"} + to = ":undef" if params[:fallback] || params[:proposal] + unicode = utf8_to_ucs(from) + undoc = ucs_to_utf8(@kddi_undoc[unicode]) + io.puts %{ ["#{undoc}", #{to}], # #{params[:comment]}} + end + end + io.puts "]" + io.puts + end + + private + + def utf8_to_ucs(cp) + return [cp].pack("H*").unpack("U*").first + end + + def ucs_to_utf8(cp) + return [cp].pack("U*").unpack("H*").first + end + + def make_kddi_undoc_map() + pub_to_sjis = citrus_decode_mapsrc( + "mskanji", 2, "UCS/EMOJI_SHIFT_JIS-KDDI").sort_by{|u, s| s} + sjis_to_undoc = citrus_decode_mapsrc( + "mskanji", 2, "EMOJI_SHIFT_JIS-KDDI-UNDOC/UCS").sort_by{|s, u| s} + return pub_to_sjis.zip(sjis_to_undoc).inject({}) {|h, rec| + raise "no match sjis codepoint" if rec[0][1] != rec[1][0] + h[rec[0][0]] = rec[1][1] + next h + } + end +end + +if ARGV.empty? + puts "usage: #$0 [emoji4unicode.xml]" + exit 1 +end +$srcdir = File.expand_path("../../enc/trans", __FILE__) +emoji_table = EmojiTable.new(ARGV[0]) + +companies = %w(DoCoMo KDDI SoftBank Unicode) + +io = STDOUT +io.puts "EMOJI_EXCHANGE_TBL = Hash.new{|h,k| h[k] = {}}" +companies.each do |from_company| + companies.each do |to_company| + next if from_company == to_company + emoji_table.generate(io, from_company, to_company) + end +end diff --git a/tool/enc-unicode.rb b/tool/enc-unicode.rb new file mode 100755 index 0000000000..93f6e869f8 --- /dev/null +++ b/tool/enc-unicode.rb @@ -0,0 +1,577 @@ +#!/usr/bin/env ruby + +# Creates the data structures needed by Oniguruma to map Unicode codepoints to +# property names and POSIX character classes +# +# To use this, get UnicodeData.txt, Scripts.txt, PropList.txt, +# PropertyAliases.txt, PropertyValueAliases.txt, DerivedCoreProperties.txt, +# DerivedAge.txt and Blocks.txt from unicode.org. +# (http://unicode.org/Public/UNIDATA/) And run following command. +# ruby1.9 tool/enc-unicode.rb data_dir > enc/unicode/name2ctype.kwd +# You can get source file for gperf. After this, simply make ruby. + +if ARGV[0] == "--header" + header = true + ARGV.shift +end +unless ARGV.size == 2 + abort "Usage: #{$0} data_directory emoji_data_directory" +end + +pat = /(?:\A|\/)([.\d]+)\z/ +$versions = { + :Unicode => ARGV[0][pat, 1], + :Emoji => ARGV[1][pat, 1], +} + +POSIX_NAMES = %w[NEWLINE Alpha Blank Cntrl Digit Graph Lower Print XPosixPunct Space Upper XDigit Word Alnum ASCII Punct] + +def pair_codepoints(codepoints) + + # We have a sorted Array of codepoints that we wish to partition into + # ranges such that the start- and endpoints form an inclusive set of + # codepoints with property _property_. Note: It is intended that some ranges + # will begin with the value with which they end, e.g. 0x0020 -> 0x0020 + + codepoints.sort! + last_cp = codepoints.first + pairs = [[last_cp, nil]] + codepoints[1..-1].each do |codepoint| + next if last_cp == codepoint + + # If the current codepoint does not follow directly on from the last + # codepoint, the last codepoint represents the end of the current range, + # and the current codepoint represents the start of the next range. + if last_cp.next != codepoint + pairs[-1][-1] = last_cp + pairs << [codepoint, nil] + end + last_cp = codepoint + end + + # The final pair has as its endpoint the last codepoint for this property + pairs[-1][-1] = codepoints.last + pairs +end + +def parse_unicode_data(file) + last_cp = 0 + data = {'Any' => (0x0000..0x10ffff).to_a, 'Assigned' => [], + 'ASCII' => (0..0x007F).to_a, 'NEWLINE' => [0x0a], 'Cn' => []} + beg_cp = nil + IO.foreach(file) do |line| + fields = line.split(';') + cp = fields[0].to_i(16) + + case fields[1] + when /\A<(.*),\s*First>\z/ + beg_cp = cp + next + when /\A<(.*),\s*Last>\z/ + cps = (beg_cp..cp).to_a + else + beg_cp = cp + cps = [cp] + end + + # The Cn category represents unassigned characters. These are not listed in + # UnicodeData.txt so we must derive them by looking for 'holes' in the range + # of listed codepoints. We increment the last codepoint seen and compare it + # with the current codepoint. If the current codepoint is less than + # last_cp.next we have found a hole, so we add the missing codepoint to the + # Cn category. + data['Cn'].concat((last_cp.next...beg_cp).to_a) + + # Assigned - Defined in unicode.c; interpreted as every character in the + # Unicode range minus the unassigned characters + data['Assigned'].concat(cps) + + # The third field denotes the 'General' category, e.g. Lu + (data[fields[2]] ||= []).concat(cps) + + # The 'Major' category is the first letter of the 'General' category, e.g. + # 'Lu' -> 'L' + (data[fields[2][0,1]] ||= []).concat(cps) + last_cp = cp + end + + # The last Cn codepoint should be 0x10ffff. If it's not, append the missing + # codepoints to Cn and C + cn_remainder = (last_cp.next..0x10ffff).to_a + data['Cn'] += cn_remainder + data['C'] += data['Cn'] + + # Special case for LC (Cased_Letter). LC = Ll + Lt + Lu + data['LC'] = data['Ll'] + data['Lt'] + data['Lu'] + + # Define General Category properties + gcps = data.keys.sort - POSIX_NAMES + + # Returns General Category Property names and the data + [gcps, data] +end + +def define_posix_props(data) + # We now derive the character classes (POSIX brackets), e.g. [[:alpha:]] + # + + data['Alpha'] = data['Alphabetic'] + data['Upper'] = data['Uppercase'] + data['Lower'] = data['Lowercase'] + data['Punct'] = data['Punctuation'] + data['XPosixPunct'] = data['Punctuation'] + [0x24, 0x2b, 0x3c, 0x3d, 0x3e, 0x5e, 0x60, 0x7c, 0x7e] + data['Digit'] = data['Decimal_Number'] + data['XDigit'] = (0x0030..0x0039).to_a + (0x0041..0x0046).to_a + + (0x0061..0x0066).to_a + data['Alnum'] = data['Alpha'] + data['Digit'] + data['Space'] = data['White_Space'] + data['Blank'] = data['Space_Separator'] + [0x0009] + data['Cntrl'] = data['Cc'] + data['Word'] = data['Alpha'] + data['Mark'] + data['Digit'] + data['Connector_Punctuation'] + data['Graph'] = data['Any'] - data['Space'] - data['Cntrl'] - + data['Surrogate'] - data['Unassigned'] + data['Print'] = data['Graph'] + data['Space_Separator'] +end + +def parse_scripts(data, categories) + files = [ + {:fn => 'DerivedCoreProperties.txt', :title => 'Derived Property'}, + {:fn => 'Scripts.txt', :title => 'Script'}, + {:fn => 'PropList.txt', :title => 'Binary Property'}, + {:fn => 'emoji/emoji-data.txt', :title => 'Emoji'} + ] + current = nil + cps = [] + names = {} + files.each do |file| + data_foreach(file[:fn]) do |line| + if /^# Total (?:code points|elements): / =~ line + data[current] = cps + categories[current] = file[:title] + (names[file[:title]] ||= []) << current + cps = [] + elsif /^([0-9a-fA-F]+)(?:\.\.([0-9a-fA-F]+))?\s*;\s*(\w+)/ =~ line + current = $3 + $2 ? cps.concat(($1.to_i(16)..$2.to_i(16)).to_a) : cps.push($1.to_i(16)) + end + end + end + # All code points not explicitly listed for Script + # have the value Unknown (Zzzz). + data['Unknown'] = (0..0x10ffff).to_a - data.values_at(*names['Script']).flatten + categories['Unknown'] = 'Script' + names.values.flatten << 'Unknown' +end + +def parse_aliases(data) + kv = {} + data_foreach('PropertyAliases.txt') do |line| + next unless /^(\w+)\s*; (\w+)/ =~ line + data[$1] = data[$2] + kv[normalize_propname($1)] = normalize_propname($2) + end + data_foreach('PropertyValueAliases.txt') do |line| + next unless /^(sc|gc)\s*; (\w+)\s*; (\w+)(?:\s*; (\w+))?/ =~ line + if $1 == 'gc' + data[$3] = data[$2] + data[$4] = data[$2] + kv[normalize_propname($3)] = normalize_propname($2) + kv[normalize_propname($4)] = normalize_propname($2) if $4 + else + data[$2] = data[$3] + data[$4] = data[$3] + kv[normalize_propname($2)] = normalize_propname($3) + kv[normalize_propname($4)] = normalize_propname($3) if $4 + end + end + kv +end + +# According to Unicode6.0.0/ch03.pdf, Section 3.1, "An update version +# never involves any additions to the character repertoire." Versions +# in DerivedAge.txt should always be /\d+\.\d+/ +def parse_age(data) + current = nil + last_constname = nil + cps = [] + ages = [] + data_foreach('DerivedAge.txt') do |line| + if /^# Total code points: / =~ line + constname = constantize_agename(current) + # each version matches all previous versions + cps.concat(data[last_constname]) if last_constname + data[constname] = cps + make_const(constname, cps, "Derived Age #{current}") + ages << current + last_constname = constname + cps = [] + elsif /^([0-9a-fA-F]+)(?:\.\.([0-9a-fA-F]+))?\s*;\s*(\d+\.\d+)/ =~ line + current = $3 + $2 ? cps.concat(($1.to_i(16)..$2.to_i(16)).to_a) : cps.push($1.to_i(16)) + end + end + ages +end + +def parse_GraphemeBreakProperty(data) + current = nil + cps = [] + ages = [] + data_foreach('auxiliary/GraphemeBreakProperty.txt') do |line| + if /^# Total code points: / =~ line + constname = constantize_Grapheme_Cluster_Break(current) + data[constname] = cps + make_const(constname, cps, "Grapheme_Cluster_Break=#{current}") + ages << current + cps = [] + elsif /^([0-9a-fA-F]+)(?:\.\.([0-9a-fA-F]+))?\s*;\s*(\w+)/ =~ line + current = $3 + $2 ? cps.concat(($1.to_i(16)..$2.to_i(16)).to_a) : cps.push($1.to_i(16)) + end + end + ages +end + +def parse_block(data) + cps = [] + blocks = [] + data_foreach('Blocks.txt') do |line| + if /^([0-9a-fA-F]+)\.\.([0-9a-fA-F]+);\s*(.*)/ =~ line + cps = ($1.to_i(16)..$2.to_i(16)).to_a + constname = constantize_blockname($3) + data[constname] = cps + make_const(constname, cps, "Block") + blocks << constname + end + end + + # All code points not belonging to any of the named blocks + # have the value No_Block. + no_block = (0..0x10ffff).to_a - data.values_at(*blocks).flatten + constname = constantize_blockname("No_Block") + make_const(constname, no_block, "Block") + blocks << constname +end + +# shim for Ruby 1.8 +unless {}.respond_to?(:key) + class Hash + alias key index + end +end + +$const_cache = {} +# make_const(property, pairs, name): Prints a 'static const' structure for a +# given property, group of paired codepoints, and a human-friendly name for +# the group +def make_const(prop, data, name) + if name.empty? + puts "\n/* '#{prop}' */" + else + puts "\n/* '#{prop}': #{name} */" + end + if origprop = $const_cache.key(data) + puts "#define CR_#{prop} CR_#{origprop}" + else + $const_cache[prop] = data + pairs = pair_codepoints(data) + puts "static const OnigCodePoint CR_#{prop}[] = {" + # The first element of the constant is the number of pairs of codepoints + puts "\t#{pairs.size}," + pairs.each do |pair| + pair.map! { |c| c == 0 ? '0x0000' : sprintf("%0#6x", c) } + puts "\t#{pair.first}, #{pair.last}," + end + puts "}; /* CR_#{prop} */" + end +end + +def normalize_propname(name) + name = name.downcase + name.delete!('- _') + name +end + +def constantize_agename(name) + "Age_#{name.sub(/\./, '_')}" +end + +def constantize_Grapheme_Cluster_Break(name) + "Grapheme_Cluster_Break_#{name}" +end + +def constantize_blockname(name) + "In_#{name.gsub(/\W/, '_')}" +end + +def get_file(name) + File.join(ARGV[name.start_with?("emoji-[stz]") ? 1 : 0], name) +end + +def data_foreach(name, &block) + fn = get_file(name) + warn "Reading #{name}" + if /^emoji/ =~ name + sep = "" + pat = /^# #{Regexp.quote(File.basename(name))}.*^# Version: ([\d.]+)/m + type = :Emoji + else + sep = "\n" + pat = /^# #{File.basename(name).sub(/\./, '-([\\d.]+)\\.')}/ + type = :Unicode + end + File.open(fn, 'rb') do |f| + line = f.gets(sep) + unless version = line[pat, 1] + raise ArgumentError, <<-ERROR +#{name}: no #{type} version +#{line.gsub(/^/, '> ')} + ERROR + end + if !(v = $versions[type]) + $versions[type] = version + elsif v != version + raise ArgumentError, <<-ERROR +#{name}: #{type} version mismatch: #{version} to #{v} +#{line.gsub(/^/, '> ')} + ERROR + end + f.each(&block) + end +end + +# Write Data +class Unifdef + attr_accessor :output, :top, :stack, :stdout, :kwdonly + def initialize(out) + @top = @output = [] + @stack = [] + $stdout, @stdout = self, out + end + def restore + $stdout = @stdout + end + def ifdef(sym) + if @kwdonly + @stdout.puts "#ifdef #{sym}" + else + @stack << @top + @top << tmp = [sym] + @top = tmp + end + if block_given? + begin + return yield + ensure + endif(sym) + end + end + end + def endif(sym) + if @kwdonly + @stdout.puts "#endif /* #{sym} */" + else + unless sym == @top[0] + restore + raise ArgumentError, "#{sym} unmatch to #{@top[0]}" + end + @top = @stack.pop + end + end + def show(dest, *syms) + _show(dest, @output, syms) + end + def _show(dest, ary, syms) + if Symbol === (sym = ary[0]) + unless syms.include?(sym) + return + end + end + ary.each do |e| + case e + when Array + _show(dest, e, syms) + when String + dest.print e + end + end + end + def write(str) + if @kwdonly + @stdout.write(str) + else + @top << str + end + self + end + alias << write +end + +output = Unifdef.new($stdout) +output.kwdonly = !header + +puts '%{' +props, data = parse_unicode_data(get_file('UnicodeData.txt')) +categories = {} +props.concat parse_scripts(data, categories) +aliases = parse_aliases(data) +ages = blocks = graphemeBreaks = nil +define_posix_props(data) +POSIX_NAMES.each do |name| + if name == 'XPosixPunct' + make_const(name, data[name], "[[:Punct:]]") + elsif name == 'Punct' + make_const(name, data[name], "") + else + make_const(name, data[name], "[[:#{name}:]]") + end +end +output.ifdef :USE_UNICODE_PROPERTIES do + props.each do |name| + category = categories[name] || + case name.size + when 1 then 'Major Category' + when 2 then 'General Category' + else '-' + end + make_const(name, data[name], category) + end + output.ifdef :USE_UNICODE_AGE_PROPERTIES do + ages = parse_age(data) + end + graphemeBreaks = parse_GraphemeBreakProperty(data) + blocks = parse_block(data) +end +puts(<<'__HEREDOC') + +static const OnigCodePoint* const CodeRanges[] = { +__HEREDOC +POSIX_NAMES.each{|name|puts" CR_#{name},"} +output.ifdef :USE_UNICODE_PROPERTIES do + props.each{|name| puts" CR_#{name},"} + output.ifdef :USE_UNICODE_AGE_PROPERTIES do + ages.each{|name| puts" CR_#{constantize_agename(name)},"} + end + graphemeBreaks.each{|name| puts" CR_#{constantize_Grapheme_Cluster_Break(name)},"} + blocks.each{|name|puts" CR_#{name},"} +end + +puts(<<'__HEREDOC') +}; +struct uniname2ctype_struct { + short name; + unsigned short ctype; +}; +#define uniname2ctype_offset(str) offsetof(struct uniname2ctype_pool_t, uniname2ctype_pool_##str) + +static const struct uniname2ctype_struct *uniname2ctype_p( +#if !(/*ANSI*/+0) /* if ANSI, old style not to conflict with generated prototype */ + const char *, unsigned int +#endif +); +%} +struct uniname2ctype_struct; +%% +__HEREDOC + +i = -1 +name_to_index = {} +POSIX_NAMES.each do |name| + i += 1 + next if name == 'NEWLINE' + name = normalize_propname(name) + name_to_index[name] = i + puts"%-40s %3d" % [name + ',', i] +end +output.ifdef :USE_UNICODE_PROPERTIES do + props.each do |name| + i += 1 + name = normalize_propname(name) + name_to_index[name] = i + puts "%-40s %3d" % [name + ',', i] + end + aliases.each_pair do |k, v| + next if name_to_index[k] + next unless v = name_to_index[v] + puts "%-40s %3d" % [k + ',', v] + end + output.ifdef :USE_UNICODE_AGE_PROPERTIES do + ages.each do |name| + i += 1 + name = "age=#{name}" + name_to_index[name] = i + puts "%-40s %3d" % [name + ',', i] + end + end + graphemeBreaks.each do |name| + i += 1 + name = "graphemeclusterbreak=#{name.delete('_').downcase}" + name_to_index[name] = i + puts "%-40s %3d" % [name + ',', i] + end + blocks.each do |name| + i += 1 + name = normalize_propname(name) + name_to_index[name] = i + puts "%-40s %3d" % [name + ',', i] + end +end +puts(<<'__HEREDOC') +%% +static int +uniname2ctype(const UChar *name, unsigned int len) +{ + const struct uniname2ctype_struct *p = uniname2ctype_p((const char *)name, len); + if (p) return p->ctype; + return -1; +} +__HEREDOC +$versions.each do |type, ver| + name = type == :Unicode ? "ONIG_UNICODE_VERSION" : "ONIG_UNICODE_EMOJI_VERSION" + versions = ver.scan(/\d+/) + print("#if defined #{name}_STRING && !( \\\n") + versions.zip(%w[MAJOR MINOR TEENY]) do |v, n| + print(" #{name}_#{n} == #{v} && \\\n") + end + print(" 1)\n") + print("# error #{name}_STRING mismatch\n") + print("#endif\n") + print("#define #{name}_STRING #{ver.dump}\n") + versions.zip(%w[MAJOR MINOR TEENY]) do |v, n| + print("#define #{name}_#{n} #{v}\n") + end +end + +output.restore + +if header + require 'tempfile' + + NAME2CTYPE = %w[gperf -7 -c -j1 -i1 -t -C -P -T -H uniname2ctype_hash -Q uniname2ctype_pool -N uniname2ctype_p] + + fds = [] + syms = %i[USE_UNICODE_PROPERTIES USE_UNICODE_AGE_PROPERTIES] + begin + fds << (tmp = Tempfile.new(%w"name2ctype .h")) + IO.popen([*NAME2CTYPE, out: tmp], "w") {|f| output.show(f, *syms)} + end while syms.pop + fds.each(&:close) + ff = nil + IO.popen(%W[diff -DUSE_UNICODE_AGE_PROPERTIES #{fds[1].path} #{fds[0].path}], "r") {|age| + IO.popen(%W[diff -DUSE_UNICODE_PROPERTIES #{fds[2].path} -], "r", in: age) {|f| + ansi = false + f.each {|line| + if /ANSI-C code produced by gperf/ =~ line + ansi = true + end + line.sub!(/\/\*ANSI\*\//, '1') if ansi + line.gsub!(/\(int\)\((?:long|size_t)\)&\(\(struct uniname2ctype_pool_t \*\)0\)->uniname2ctype_pool_(str\d+),\s+/, + 'uniname2ctype_offset(\1), ') + if ff = (!ff ? /^(uniname2ctype_hash) /=~line : /^\}/!~line) # no line can match both, exclusive flip-flop + line.sub!(/^( *(?:register\s+)?(.*\S)\s+hval\s*=\s*)(?=len;)/, '\1(\2)') + end + puts line + } + } + } +end diff --git a/tool/eval.rb b/tool/eval.rb new file mode 100644 index 0000000000..9153573e6e --- /dev/null +++ b/tool/eval.rb @@ -0,0 +1,158 @@ +# VM checking and benchmarking code + +require './rbconfig' +require 'fileutils' +require 'pp' + +Ruby = ENV['RUBY'] || RbConfig.ruby +# + +OPTIONS = %w{ + opt-direct-threaded-code + opt-basic-operations + opt-operands-unification + opt-instructions-unification + opt-inline-method-cache + opt-stack-caching +}.map{|opt| + '--disable-' + opt +} + +opts = OPTIONS.dup +Configs = OPTIONS.map{|opt| + o = opts.dup + opts.delete(opt) + o +} + [[]] + +pp Configs if $DEBUG + + +def exec_cmd(cmd) + puts cmd + unless system(cmd) + p cmd + raise "error" + end +end + +def dirname idx + "ev-#{idx}" +end + +def build + Configs.each_with_index{|config, idx| + dir = dirname(idx) + FileUtils.rm_rf(dir) if FileTest.exist?(dir) + Dir.mkdir(dir) + FileUtils.cd(dir){ + exec_cmd("#{Ruby} ../extconf.rb " + config.join(" ")) + exec_cmd("make clean test-all") + } + } +end + +def check + Configs.each_with_index{|c, idx| + puts "= #{idx}" + system("#{Ruby} -r ev-#{idx}/yarvcore -e 'puts YARVCore::OPTS'") + } +end + +def bench_each idx + puts "= #{idx}" + 5.times{|count| + print count + FileUtils.cd(dirname(idx)){ + exec_cmd("make benchmark OPT=-y ITEMS=#{ENV['ITEMS']} > ../b#{idx}-#{count}") + } + } + puts +end + +def bench + # return bench_each(6) + Configs.each_with_index{|c, idx| + bench_each idx + } +end + +def parse_result data + flag = false + stat = [] + data.each{|line| + if flag + if /(\w+)\t([\d\.]+)/ =~ line + stat << [$1, $2.to_f] + else + raise "not a data" + end + + end + if /benchmark summary/ =~ line + flag = true + end + } + stat +end + +def calc_each data + data.sort! + data.pop # remove max + data.shift # remove min + + data.inject(0.0){|res, e| + res += e + } / data.size +end + +def calc_stat stats + stats[0].each_with_index{|e, idx| + bm = e[0] + vals = stats.map{|st| + st[idx][1] + } + [bm, calc_each(vals)] + } +end + +def stat + total = [] + Configs.each_with_index{|c, idx| + stats = [] + 5.times{|count| + file = "b#{idx}-#{count}" + # p file + open(file){|f| + stats << parse_result(f.read) + } + } + # merge stats + total << calc_stat(stats) + total + } + # pp total + total[0].each_with_index{|e, idx| + # print "#{e[0]}\t" + total.each{|st| + print st[idx][1], "\t" + } + puts + } +end + +ARGV.each{|cmd| + case cmd + when 'build' + build + when 'check' + check + when 'bench' + bench + when 'stat' + stat + else + raise + end +} + diff --git a/tool/expand-config.rb b/tool/expand-config.rb new file mode 100755 index 0000000000..81ffa6cb98 --- /dev/null +++ b/tool/expand-config.rb @@ -0,0 +1,33 @@ +#!./miniruby -s + +# Used to expand Ruby config entries for Win32 Makefiles. + +config = File.read(conffile = $config) +config.sub!(/^(\s*)RUBY_VERSION\b.*(\sor\s*)$/, '\1true\2') +rbconfig = Module.new {module_eval(config, conffile)}::RbConfig +config = $expand ? rbconfig::CONFIG : rbconfig::MAKEFILE_CONFIG +config["RUBY_RELEASE_DATE"] ||= + File.read(File.expand_path("../../version.h", __FILE__))[/^\s*#\s*define\s+RUBY_RELEASE_DATE\s+"(.*)"/, 1] + +while /\A(\w+)=(.*)/ =~ ARGV[0] + config[$1] = $2 + config[$1].tr!(File::ALT_SEPARATOR, File::SEPARATOR) if File::ALT_SEPARATOR + ARGV.shift +end + +if $output + output = open($output, "wb", $mode &&= $mode.oct) + output.chmod($mode) if $mode +else + output = STDOUT + output.binmode +end + +ARGF.each do |line| + line.gsub!(/@([a-z_]\w*)@/i) { + s = config.fetch($1, $expand ? $& : "") + s = s.gsub(/\$\((.+?)\)/, %Q[${\\1}]) unless $expand + s + } + output.puts line +end diff --git a/tool/extlibs.rb b/tool/extlibs.rb new file mode 100755 index 0000000000..cd8e5239b3 --- /dev/null +++ b/tool/extlibs.rb @@ -0,0 +1,263 @@ +#!/usr/bin/ruby + +# Used to download, extract and patch extension libraries (extlibs) +# for Ruby. See common.mk for Ruby's usage. + +require 'digest' +require_relative 'downloader' +require_relative 'lib/colorize' + +class Vars < Hash + def pattern + /\$\((#{Regexp.union(keys)})\)/ + end + + def expand(str) + if empty? + str + else + str.gsub(pattern) {self[$1]} + end + end +end + +class ExtLibs + def initialize + @colorize = Colorize.new + end + + def cache_file(url, cache_dir) + Downloader.cache_file(url, nil, cache_dir).to_path + end + + def do_download(url, cache_dir) + Downloader.download(url, nil, nil, nil, :cache_dir => cache_dir) + end + + def do_checksum(cache, chksums) + chksums.each do |sum| + name, sum = sum.split(/:/) + if $VERBOSE + $stdout.print "checking #{name} of #{cache} ..." + $stdout.flush + end + hd = Digest(name.upcase).file(cache).hexdigest + if $VERBOSE + $stdout.print " " + $stdout.puts hd == sum ? @colorize.pass("OK") : @colorize.fail("NG") + $stdout.flush + end + unless hd == sum + raise "checksum mismatch: #{cache}, #{name}:#{hd}, expected #{sum}" + end + end + end + + def do_extract(cache, dir) + if $VERBOSE + $stdout.puts "extracting #{cache} into #{dir}" + $stdout.flush + end + ext = File.extname(cache) + case ext + when '.gz', '.tgz' + f = IO.popen(["gzip", "-dc", cache]) + cache = cache.chomp('.gz') + when '.bz2', '.tbz' + f = IO.popen(["bzip2", "-dc", cache]) + cache = cache.chomp('.bz2') + when '.xz', '.txz' + f = IO.popen(["xz", "-dc", cache]) + cache = cache.chomp('.xz') + else + inp = cache + end + inp ||= f.binmode + ext = File.extname(cache) + case ext + when '.tar', /\A\.t[gbx]z\z/ + pid = Process.spawn("tar", "xpf", "-", in: inp, chdir: dir) + when '.zip' + pid = Process.spawn("unzip", inp, "-d", dir) + end + f.close if f + Process.wait(pid) + $?.success? or raise "failed to extract #{cache}" + end + + def do_patch(dest, patch, args) + if $VERBOSE + $stdout.puts "applying #{patch} under #{dest}" + $stdout.flush + end + Process.wait(Process.spawn(ENV.fetch("PATCH", "patch"), "-d", dest, "-i", patch, *args)) + $?.success? or raise "failed to patch #{patch}" + end + + def do_link(file, src, dest) + file = File.join(dest, file) + if (target = src).start_with?("/") + target = File.join([".."] * file.count("/"), src) + end + return unless File.exist?(File.expand_path(target, File.dirname(file))) + File.unlink(file) rescue nil + begin + File.symlink(target, file) + rescue + else + if $VERBOSE + $stdout.puts "linked #{target} to #{file}" + $stdout.flush + end + return + end + begin + src = src.sub(/\A\//, '') + File.copy_stream(src, file) + rescue + if $VERBOSE + $stdout.puts "failed to link #{src} to #{file}: #{$!.message}" + end + else + if $VERBOSE + $stdout.puts "copied #{src} to #{file}" + end + end + end + + def do_exec(command, dir, dest) + dir = dir ? File.join(dest, dir) : dest + if $VERBOSE + $stdout.puts "running #{command.dump} under #{dir}" + $stdout.flush + end + system(command, chdir: dir) or raise "failed #{command.dump}" + end + + def do_command(mode, dest, url, cache_dir, chksums) + extracted = false + base = /.*(?=\.tar(?:\.\w+)?\z)/ + + case mode + when :download + cache = do_download(url, cache_dir) + do_checksum(cache, chksums) + when :extract + cache = cache_file(url, cache_dir) + target = File.join(dest, File.basename(cache)[base]) + unless File.directory?(target) + do_checksum(cache, chksums) + extracted = do_extract(cache, dest) + end + when :all + cache = do_download(url, cache_dir) + target = File.join(dest, File.basename(cache)[base]) + unless File.directory?(target) + do_checksum(cache, chksums) + extracted = do_extract(cache, dest) + end + end + extracted + end + + def run(argv) + cache_dir = nil + mode = :all + until argv.empty? + case argv[0] + when '--download' + mode = :download + when '--extract' + mode = :extract + when '--patch' + mode = :patch + when '--all' + mode = :all + when '--cache' + argv.shift + cache_dir = argv[0] + when /\A--cache=/ + cache_dir = $' + when '--' + argv.shift + break + when /\A-/ + warn "unknown option: #{argv[0]}" + return false + else + break + end + argv.shift + end + + success = true + argv.each do |dir| + Dir.glob("#{dir}/**/extlibs") do |list| + if $VERBOSE + $stdout.puts "downloading for #{list}" + $stdout.flush + end + vars = Vars.new + extracted = false + dest = File.dirname(list) + url = chksums = nil + IO.foreach(list) do |line| + line.sub!(/\s*#.*/, '') + if /^(\w+)\s*=\s*(.*)/ =~ line + vars[$1] = vars.expand($2) + next + end + if chksums + chksums.concat(line.split) + elsif /^\t/ =~ line + if extracted and (mode == :all or mode == :patch) + patch, *args = line.split.map {|s| vars.expand(s)} + do_patch(dest, patch, args) + end + next + elsif /^!\s*(?:chdir:\s*([^|\s]+)\|\s*)?(.*)/ =~ line + if extracted and (mode == :all or mode == :patch) + command = vars.expand($2.strip) + chdir = $1 and chdir = vars.expand(chdir) + do_exec(command, chdir, dest) + end + next + elsif /->/ =~ line + if extracted and (mode == :all or mode == :patch) + link, file = $`.strip, $'.strip + do_link(vars.expand(link), vars.expand(file), dest) + end + next + else + url, *chksums = line.split(' ') + end + if chksums.last == '\\' + chksums.pop + next + end + unless url + chksums = nil + next + end + url = vars.expand(url) + begin + extracted = do_command(mode, dest, url, cache_dir, chksums) + rescue => e + warn e.full_message + success = false + end + url = chksums = nil + end + end + end + success + end + + def self.run(argv) + self.new.run(argv) + end +end + +if $0 == __FILE__ + exit ExtLibs.run(ARGV) +end diff --git a/tool/fake.rb b/tool/fake.rb new file mode 100644 index 0000000000..91dfb041c4 --- /dev/null +++ b/tool/fake.rb @@ -0,0 +1,61 @@ +# Used by Makefile and configure for building Ruby. +# See common.mk and Makefile.in for details. + +class File + sep = ("\\" if RUBY_PLATFORM =~ /mswin|bccwin|mingw/) + if sep != ALT_SEPARATOR + remove_const :ALT_SEPARATOR + ALT_SEPARATOR = sep + end +end + +static = !!(defined?($static) && $static) +$:.unshift(builddir) +posthook = proc do + RbConfig.fire_update!("top_srcdir", $top_srcdir) + RbConfig.fire_update!("topdir", $topdir) + $hdrdir.sub!(/\A#{Regexp.quote($top_srcdir)}(?=\/)/, "$(top_srcdir)") + if $extmk + $ruby = "$(topdir)/miniruby -I'$(topdir)' -I'$(top_srcdir)/lib' -I'$(extout)/$(arch)' -I'$(extout)/common'" + else + $ruby = baseruby + end + $static = static + untrace_var(:$ruby, posthook) +end +prehook = proc do |extmk| +=begin + pat = %r[(?:\A(?:\w:|//[^/]+)|\G)/[^/]*] + dir = builddir.scan(pat) + pwd = Dir.pwd.scan(pat) + if dir[0] == pwd[0] + while dir[0] and dir[0] == pwd[0] + dir.shift + pwd.shift + end + builddir = File.join((pwd.empty? ? ["."] : [".."]*pwd.size) + dir) + builddir = "." if builddir.empty? + end +=end + join = proc {|*args| File.join(*args).sub!(/\A(?:\.\/)*/, '')} + $topdir ||= builddir + $top_srcdir ||= (File.identical?(top_srcdir, dir = join[$topdir, srcdir]) ? + dir : top_srcdir) + $extout = '$(topdir)/.ext' + $extout_prefix = '$(extout)$(target_prefix)/' + config = RbConfig::CONFIG + mkconfig = RbConfig::MAKEFILE_CONFIG + $builtruby ||= File.join(builddir, config['RUBY_INSTALL_NAME'] + config['EXEEXT']) + RbConfig.fire_update!("builddir", builddir) + RbConfig.fire_update!("buildlibdir", builddir) + RbConfig.fire_update!("libdir", builddir) + RbConfig.fire_update!("prefix", $topdir) + RbConfig.fire_update!("top_srcdir", $top_srcdir ||= top_srcdir) + RbConfig.fire_update!("extout", $extout) + RbConfig.fire_update!("rubyhdrdir", "$(top_srcdir)/include") + RbConfig.fire_update!("rubyarchhdrdir", "$(extout)/include/$(arch)") + RbConfig.fire_update!("libdirname", "buildlibdir") + trace_var(:$ruby, posthook) + untrace_var(:$extmk, prehook) +end +trace_var(:$extmk, prehook) diff --git a/tool/fetch-bundled_gems.rb b/tool/fetch-bundled_gems.rb new file mode 100755 index 0000000000..12d6f3d9cd --- /dev/null +++ b/tool/fetch-bundled_gems.rb @@ -0,0 +1,27 @@ +#!ruby -an +BEGIN { + require 'fileutils' + + dir = ARGV.shift + ARGF.eof? + FileUtils.mkdir_p(dir) + Dir.chdir(dir) +} + +n, v, u, r = $F + +next if n =~ /^#/ + +if File.directory?(n) + puts "updating #{n} ..." + system("git", "fetch", chdir: n) or abort +else + puts "retrieving #{n} ..." + system(*%W"git clone #{u} #{n}") or abort +end +c = r || "v#{v}" +checkout = %w"git -c advice.detachedHead=false checkout" +puts "checking out #{c} (v=#{v}, r=#{r}) ..." +unless system(*checkout, c, "--", chdir: n) + abort if r or !system(*checkout, v, "--", chdir: n) +end diff --git a/tool/file2lastrev.rb b/tool/file2lastrev.rb new file mode 100755 index 0000000000..3d8c69357d --- /dev/null +++ b/tool/file2lastrev.rb @@ -0,0 +1,124 @@ +#!/usr/bin/env ruby + +# Gets the most recent revision of a file in a VCS-agnostic way. +# Used by Doxygen, Makefiles and merger.rb. + +require 'optparse' + +# this file run with BASERUBY, which may be older than 1.9, so no +# require_relative +require File.expand_path('../lib/vcs', __FILE__) + +Program = $0 + +@output = nil +def self.output=(output) + if @output and @output != output + raise "you can specify only one of --changed, --revision.h and --doxygen" + end + @output = output +end +@suppress_not_found = false +@limit = 20 + +format = '%Y-%m-%dT%H:%M:%S%z' +vcs = nil +OptionParser.new {|opts| + opts.banner << " paths..." + vcs_options = VCS.define_options(opts) + new_vcs = proc do |path| + begin + vcs = VCS.detect(path, vcs_options, opts.new) + rescue VCS::NotFoundError => e + abort "#{File.basename(Program)}: #{e.message}" unless @suppress_not_found + opts.remove + end + nil + end + opts.new + opts.on("--srcdir=PATH", "use PATH as source directory") do |path| + abort "#{File.basename(Program)}: srcdir is already set" if vcs + new_vcs[path] + end + opts.on("--changed", "changed rev") do + self.output = :changed + end + opts.on("--revision.h", "RUBY_REVISION macro") do + self.output = :revision_h + end + opts.on("--doxygen", "Doxygen format") do + self.output = :doxygen + end + opts.on("--modified[=FORMAT]", "modified time") do |fmt| + self.output = :modified + format = fmt if fmt + end + opts.on("--limit=NUM", "limit branch name length (#@limit)", Integer) do |n| + @limit = n + end + opts.on("-q", "--suppress_not_found") do + @suppress_not_found = true + end + opts.order! rescue abort "#{File.basename(Program)}: #{$!}\n#{opts}" + if vcs + vcs.set_options(vcs_options) # options after --srcdir + else + new_vcs["."] + end +} +exit unless vcs + +@output = + case @output + when :changed, nil + Proc.new {|last, changed| + changed + } + when :revision_h + Proc.new {|last, changed, modified, branch, title| + short = vcs.short_revision(last) + if /[^\x00-\x7f]/ =~ title and title.respond_to?(:force_encoding) + title = title.dup.force_encoding("US-ASCII") + end + [ + "#define RUBY_REVISION #{short.inspect}", + ("#define RUBY_FULL_REVISION #{last.inspect}" unless short == last), + if branch + e = '..' + limit = @limit + name = branch.sub(/\A(.{#{limit-e.size}}).{#{e.size+1},}/o) {$1+e} + name = name.dump.sub(/\\#/, '#') + "#define RUBY_BRANCH_NAME #{name}" + end, + if title + title = title.dump.sub(/\\#/, '#') + "#define RUBY_LAST_COMMIT_TITLE #{title}" + end, + if modified + modified.utc.strftime('#define RUBY_RELEASE_DATETIME "%FT%TZ"') + end, + ].compact + } + when :doxygen + Proc.new {|last, changed| + "r#{changed}/r#{last}" + } + when :modified + Proc.new {|last, changed, modified| + modified.strftime(format) + } + else + raise "unknown output format `#{@output}'" + end + +ok = true +(ARGV.empty? ? [nil] : ARGV).each do |arg| + begin + puts @output[*vcs.get_revisions(arg)] + rescue => e + next if @suppress_not_found and VCS::NotFoundError === e + warn "#{File.basename(Program)}: #{e.message}" + ok = false + end +end +exit ok diff --git a/tool/format-release b/tool/format-release new file mode 100755 index 0000000000..e0de841127 --- /dev/null +++ b/tool/format-release @@ -0,0 +1,262 @@ +#!/usr/bin/env ruby +# https://rubygems.org/gems/diffy +require "diffy" +require "open-uri" +require "yaml" + +Diffy::Diff.default_options.merge!( + include_diff_info: true, + context: 1, +) + +class Tarball + attr_reader :version, :size, :sha1, :sha256, :sha512 + + def initialize(version, url, size, sha1, sha256, sha512) + @url = url + @size = size + @sha1 = sha1 + @sha256 = sha256 + @sha512 = sha512 + @version = version + @xy = version[/\A\d+\.\d+/] + end + + def gz?; @url.end_with?('.gz'); end + def zip?; @url.end_with?('.zip'); end + def bz2?; @url.end_with?('.bz2'); end + def xz?; @url.end_with?('.xz'); end + + def ext; @url[/(?:zip|tar\.(?:gz|bz2|xz))\z/]; end + + def to_md + <<eom +* <https://cache.ruby-lang.org/pub/ruby/#{@xy}/ruby-#{@version}.#{ext}> + + SIZE: #{@size} bytes + SHA1: #{@sha1} + SHA256: #{@sha256} + SHA512: #{@sha512} +eom + end + + # * /home/naruse/obj/ruby-trunk/tmp/ruby-2.6.0-preview3.tar.gz + # SIZE: 17116009 bytes + # SHA1: 21f62c369661a2ab1b521fd2fa8191a4273e12a1 + # SHA256: 97cea8aa63dfa250ba6902b658a7aa066daf817b22f82b7ee28f44aec7c2e394 + # SHA512: 1e2042324821bb4e110af7067f52891606dcfc71e640c194ab1c117f0b941550e0b3ac36ad3511214ac80c536b9e5cfaf8789eec74cf56971a832ea8fc4e6d94 + def self.parse(wwwdir, version, rubydir) + unless /\A(\d+)\.(\d+)\.(\d+)(?:-(?:preview|rc)\d+)?\z/ =~ version + raise "unexpected version string '#{version}'" + end + x = $1.to_i + y = $2.to_i + z = $3.to_i + # previous tag for git diff --shortstat + # It's only for x.y.0 release + if z != 0 + prev_tag = nil + elsif y != 0 + prev_tag = "v#{x}_#{y-1}_0" + prev_ver = "#{x}.#{y-1}.0" + elsif x == 3 && y == 0 && z == 0 + prev_tag = "v2_7_0" + prev_ver = "2.7.0" + else + raise "unexpected version for prev_ver '#{version}'" + end + + uri = "https://cache.ruby-lang.org/pub/tmp/ruby-info-#{version}-draft.yml" + info = YAML.load(URI(uri).read) + if info.size != 1 + raise "unexpected info.yml '#{uri}'" + end + tarballs = [] + info[0]["size"].each_key do |ext| + url = info[0]["url"][ext] + size = info[0]["size"][ext] + sha1 = info[0]["sha1"][ext] + sha256 = info[0]["sha256"][ext] + sha512 = info[0]["sha512"][ext] + tarball = Tarball.new(version, url, size, sha1, sha256, sha512) + tarballs << tarball + end + + if prev_tag + # show diff shortstat + tag = "v#{version.gsub(/[.\-]/, '_')}" + stat = `git -C #{rubydir} diff --shortstat #{prev_tag}..#{tag}` + files_changed, insertions, deletions = stat.scan(/\d+/) + end + + xy = version[/\A\d+\.\d+/] + #puts "## Download\n\n" + #tarballs.each do |tarball| + # puts tarball.to_md + #end + update_branches_yml(version, xy, wwwdir) + update_downloads_yml(version, xy, wwwdir) + update_releases_yml(version, xy, tarballs, wwwdir, files_changed, insertions, deletions) + tarballs + end + + def self.update_branches_yml(ver, xy, wwwdir) + filename = "_data/branches.yml" + orig_data = File.read(File.join(wwwdir, filename)) + data = orig_data.dup + if data.include?("\n- name: #{xy}\n") + data.sub!(/\n- name: #{Regexp.escape(xy)}\n(?: .*\n)*/) do |node| + unless ver.include?("-") + # assume this is X.Y.0 release + node.sub!(/^ status: preview\n/, " status: normal maintenance\n") + node.sub!(/^ date:\n/, " date: #{Time.now.year}-12-25\n") + end + node + end + else + if ver.include?("-") + status = "preview" + year = nil + else + status = "normal maintenance" + year = Time.now.year + end + entry = <<eom +- name: #{xy} + status: #{status} + date:#{ year && " #{year}-12-25" } + eol_date: + +eom + data.sub!(/(?=^- name)/, entry) + end + if data != orig_data + diff = Diffy::Diff.new(orig_data, data) + show_diff(filename, diff) + end + end + + def self.update_downloads_yml(ver, xy, wwwdir) + filename = "_data/downloads.yml" + orig_data = File.read(File.join(wwwdir, filename)) + data = orig_data.dup + + if /^preview:\n\n(?: .*\n)* - #{Regexp.escape(xy)}\./ =~ data + if ver.include?("-") + data.sub!(/^ - #{Regexp.escape(xy)}\..*/, " - #{ver}") + else + data.sub!(/^ - #{Regexp.escape(xy)}\..*\n/, "") + data.sub!(/(?<=^stable:\n\n)/, " - #{ver}\n") + end + else + unless data.sub!(/^ - #{Regexp.escape(xy)}\..*/, " - #{ver}") + if ver.include?("-") + data.sub!(/(?<=^preview:\n\n)/, " - #{ver}\n") + else + data.sub!(/(?<=^stable:\n\n)/, " - #{ver}\n") + end + end + end + if data != orig_data + diff = Diffy::Diff.new(orig_data, data) + show_diff(filename, diff) + end + end + + def self.update_releases_yml(ver, xy, ary, wwwdir, files_changed, insertions, deletions) + filename = "_data/releases.yml" + orig_data = File.read(File.join(wwwdir, filename)) + data = orig_data.dup + + date = Time.now.utc # use utc to use previous day in midnight + entry = <<eom +- version: #{ver} + date: #{date.strftime("%Y-%m-%d")} + post: /en/news/#{date.strftime("%Y/%m/%d")}/ruby-#{ver.tr('.', '-')}-released/ +eom + + if /\.0(?:-\w+)?\z/ =~ ver + # preview, rc, or first release + entry <<= <<eom + tag: ruby_#{ver.tr('.-', '_')} + stats: + files_changed: #{files_changed} + insertions: #{insertions} + deletions: #{deletions} +eom + end + + entry <<= <<eom + url: + gz: https://cache.ruby-lang.org/pub/ruby/#{xy}/ruby-#{ver}.tar.gz + zip: https://cache.ruby-lang.org/pub/ruby/#{xy}/ruby-#{ver}.zip + bz2: https://cache.ruby-lang.org/pub/ruby/#{xy}/ruby-#{ver}.tar.bz2 + xz: https://cache.ruby-lang.org/pub/ruby/#{xy}/ruby-#{ver}.tar.xz + size: + gz: #{ary.find{|x|x.gz? }.size} + zip: #{ary.find{|x|x.zip?}.size} + bz2: #{ary.find{|x|x.bz2?}&.size} + xz: #{ary.find{|x|x.xz? }.size} + sha1: + gz: #{ary.find{|x|x.gz? }.sha1} + zip: #{ary.find{|x|x.zip?}.sha1} + bz2: #{ary.find{|x|x.bz2?}&.sha1} + xz: #{ary.find{|x|x.xz? }.sha1} + sha256: + gz: #{ary.find{|x|x.gz? }.sha256} + zip: #{ary.find{|x|x.zip?}.sha256} + bz2: #{ary.find{|x|x.bz2?}&.sha256} + xz: #{ary.find{|x|x.xz? }.sha256} + sha512: + gz: #{ary.find{|x|x.gz? }.sha512} + zip: #{ary.find{|x|x.zip?}.sha512} + bz2: #{ary.find{|x|x.bz2?}&.sha512} + xz: #{ary.find{|x|x.xz? }.sha512} +eom + + if ver.start_with?("3.") + entry = entry.gsub(/ bz2: .*\n/, "") + end + + if data.include?("\n- version: #{ver}\n") + # update existing entry + data.sub!(/\n- version: #{ver}\n(^ .*\n)*\n/, "\n#{entry}\n") + elsif data.sub!(/\n# #{Regexp.escape(xy)} series\n/, "\\&\n#{entry}") + else + data.sub!(/^$/, "\n# #{xy} series\n\n#{entry}") + end + if data != orig_data + diff = Diffy::Diff.new(orig_data, data) + show_diff(filename, diff) + end + end + + def self.show_diff(filename, diff) + diff.each_with_index do |line, index| + case index + when 0 + line.sub!(/\A--- (.*)\t(\d+-\d+-\d+ [0-9:.]+ [\-+]\d+)\Z/) do + "--- a/#{filename}\t#{$2}" + end + when 1 + line.sub!(/\A\+\+\+ (.*)\t(\d+-\d+-\d+ [0-9:.]+ [\-+]\d+)\Z/) do + "+++ b/#{filename}\t#{$2}" + end + end + puts line + end + end +end + +def main + wwwdir = ARGV.shift + version = ARGV.shift + rubydir = ARGV.shift + unless rubydir + STDERR.puts "usage: format-release <dir-of-w.r-l.o> <version> <ruby-dir>" + exit + end + Tarball.parse(wwwdir, version, rubydir) +end + +main diff --git a/tool/gen-mailmap.rb b/tool/gen-mailmap.rb new file mode 100755 index 0000000000..27b7abf8de --- /dev/null +++ b/tool/gen-mailmap.rb @@ -0,0 +1,47 @@ +#!/usr/bin/env ruby + +require "open-uri" +require "yaml" + +EMAIL_YML_URL = "https://cdn.jsdelivr.net/gh/ruby/ruby-commit-hook/config/email.yml" + +email_yml = URI(EMAIL_YML_URL).read.sub(/\A(?:#.*\n)+/, "").gsub(/^# +(.+)$/) { $1 + ": []" } + +email = YAML.load(email_yml) +YAML.load(DATA.read).each do |name, mails| + email[name] ||= [] + email[name] |= mails +end + +open(File.join(__dir__, "../.mailmap"), "w") do |f| + email.each do |name, mails| + canonical = "#{ name }@ruby-lang.org" + mails.delete(canonical) + svn = "#{ name }@b2dd03c8-39d4-4d8f-98ff-823fe69b080e" + ((mails | [canonical]) + [svn]).each do |mail| + f.puts "#{ name } <#{ canonical }> <#{ mail }>" + end + end +end + +puts "You'll see canonical names (SVN account names) by the following commands:" +puts +puts " git shortlog -ce" +puts " git log --pretty=format:'%cN <%cE>'" +puts " git log --use-mailmap --pretty=full" + +__END__ +git: +- svn@b2dd03c8-39d4-4d8f-98ff-823fe69b080e +- "(no author)@b2dd03c8-39d4-4d8f-98ff-823fe69b080e" +kazu: +- znz@users.noreply.github.com +marcandre: +- github@marc-andre.ca +mrkn: +- mrkn@users.noreply.github.com +- muraken@b2dd03c8-39d4-4d8f-98ff-823fe69b080e +naruse: +- nurse@users.noreply.github.com +tenderlove: +- tenderlove@github.com diff --git a/tool/gen_dummy_probes.rb b/tool/gen_dummy_probes.rb new file mode 100755 index 0000000000..45222830f3 --- /dev/null +++ b/tool/gen_dummy_probes.rb @@ -0,0 +1,32 @@ +#!/usr/bin/ruby +# -*- coding: us-ascii -*- + +# Used to create dummy probes (as for systemtap and DTrace) by Makefiles. +# See common.mk. + +text = ARGF.read + +# remove comments +text.gsub!(%r'(?:^ *)?/\*.*?\*/\n?'m, '') + +# remove the pragma declarations and ifdefs +text.gsub!(/^#(?:pragma|include|if|endif).*\n/, '') + +# replace the provider section with the start of the header file +text.gsub!(/provider ruby \{/, "#ifndef\t_PROBES_H\n#define\t_PROBES_H\n#define DTRACE_PROBES_DISABLED 1\n") + +# finish up the #ifndef sandwich +text.gsub!(/\};/, "\n#endif\t/* _PROBES_H */") + +# expand probes to DTRACE macros +text.gsub!(/^ *probe ([^\(]*)\(([^\)]*)\);/) { + name, args = $1, $2 + name.upcase! + name.gsub!(/__/, '_') + args.gsub!(/(\A|, *)[^,]*\b(?=\w+(?=,|\z))/, '\1') + "#define RUBY_DTRACE_#{name}_ENABLED() 0\n" \ + "#define RUBY_DTRACE_#{name}(#{args}) do {} while (0)" +} + +puts "/* -*- c -*- */" +print text diff --git a/tool/gen_ruby_tapset.rb b/tool/gen_ruby_tapset.rb new file mode 100755 index 0000000000..ae3c1eccd2 --- /dev/null +++ b/tool/gen_ruby_tapset.rb @@ -0,0 +1,105 @@ +#!/usr/bin/ruby +# -*- coding: us-ascii -*- +# Create a tapset for systemtap and DTrace +# usage: ./ruby gen_ruby_tapset.rb --ruby-path=/path/to/ruby probes.d > output + +require "optparse" + +def set_argument(argname, nth) + # remove C style type info + argname.gsub!(/.+ (.+)/, '\1') # e.g. char *hoge -> *hoge + argname.gsub!(/^\*/, '') # e.g. *filename -> filename + + "#{argname} = $arg#{nth}" +end + +ruby_path = "/usr/local/ruby" + +opts = OptionParser.new +opts.on("--ruby-path=PATH"){|v| ruby_path = v} +opts.parse!(ARGV) + +text = ARGF.read + +# remove preprocessor directives +text.gsub!(/^#.*$/, '') + +# remove provider name +text.gsub!(/^provider ruby \{/, "") +text.gsub!(/^\};/, "") + +# probename() +text.gsub!(/probe (.+)\( *\);/) { + probe_name = $1 + <<-End + probe #{probe_name} = process("ruby").provider("ruby").mark("#{probe_name}") + { + } + End +} + +# probename(arg1) +text.gsub!(/ *probe (.+)\(([^,)]+)\);/) { + probe_name = $1 + arg1 = $2 + + <<-End + probe #{probe_name} = process("ruby").provider("ruby").mark("#{probe_name}") + { + #{set_argument(arg1, 1)} + } + End +} + +# probename(arg1, arg2) +text.gsub!(/ *probe (.+)\(([^,)]+),([^,)]+)\);/) { + probe_name = $1 + arg1 = $2 + arg2 = $3 + + <<-End + probe #{probe_name} = process("#{ruby_path}").provider("ruby").mark("#{probe_name}") + { + #{set_argument(arg1, 1)} + #{set_argument(arg2, 2)} + } + End +} + +# probename(arg1, arg2, arg3) +text.gsub!(/ *probe (.+)\(([^,)]+),([^,)]+),([^,)]+)\);/) { + probe_name = $1 + arg1 = $2 + arg2 = $3 + arg3 = $4 + + <<-End + probe #{probe_name} = process("#{ruby_path}").provider("ruby").mark("#{probe_name}") + { + #{set_argument(arg1, 1)} + #{set_argument(arg2, 2)} + #{set_argument(arg3, 3)} + } + End +} + +# probename(arg1, arg2, arg3, arg4) +text.gsub!(/ *probe (.+)\(([^,)]+),([^,)]+),([^,)]+),([^,)]+)\);/) { + probe_name = $1 + arg1 = $2 + arg2 = $3 + arg3 = $4 + arg4 = $5 + + <<-End + probe #{probe_name} = process("#{ruby_path}").provider("ruby").mark("#{probe_name}") + { + #{set_argument(arg1, 1)} + #{set_argument(arg2, 2)} + #{set_argument(arg3, 3)} + #{set_argument(arg4, 4)} + } + End +} + +print text diff --git a/tool/generic_erb.rb b/tool/generic_erb.rb new file mode 100644 index 0000000000..6af995fc13 --- /dev/null +++ b/tool/generic_erb.rb @@ -0,0 +1,61 @@ +# -*- coding: us-ascii -*- + +# Used to expand Ruby template files by common.mk, uncommon.mk and +# some Ruby extension libraries. + +require 'erb' +require 'optparse' +require_relative 'lib/vpath' +require_relative 'lib/colorize' + +vpath = VPath.new +timestamp = nil +output = nil +ifchange = nil +source = false +color = nil +templates = [] + +ARGV.options do |o| + o.on('-t', '--timestamp[=PATH]') {|v| timestamp = v || true} + o.on('-i', '--input=PATH') {|v| template << v} + o.on('-o', '--output=PATH') {|v| output = v} + o.on('-c', '--[no-]if-change') {|v| ifchange = v} + o.on('-x', '--source') {source = true} + o.on('--color') {color = true} + vpath.def_options(o) + o.order!(ARGV) + templates << (ARGV.shift or abort o.to_s) if templates.empty? +end +color = Colorize.new(color) +unchanged = color.pass("unchanged") +updated = color.fail("updated") + +result = templates.map do |template| + if ERB.instance_method(:initialize).parameters.assoc(:key) # Ruby 2.6+ + erb = ERB.new(File.read(template), trim_mode: '%-') + else + erb = ERB.new(File.read(template), nil, '%-') + end + erb.filename = template + source ? erb.src : proc{erb.result(binding)}.call +end +result = result.size == 1 ? result[0] : result.join("") +if output + if ifchange and (vpath.open(output, "rb") {|f| f.read} rescue nil) == result + puts "#{output} #{unchanged}" + else + open(output, "wb") {|f| f.print result} + puts "#{output} #{updated}" + end + if timestamp + if timestamp == true + dir, base = File.split(output) + timestamp = File.join(dir, ".time." + base) + end + File.open(timestamp, 'a') {} + File.utime(nil, nil, timestamp) + end +else + print result +end diff --git a/tool/git-refresh b/tool/git-refresh new file mode 100755 index 0000000000..9ed7d7c76e --- /dev/null +++ b/tool/git-refresh @@ -0,0 +1,46 @@ +#!/bin/sh +set -e + +if (cd -P .) 2>/dev/null; then + CHDIR='cd -P' +else + CHDIR='cd' +fi + +quiet= +branch= + +until [ $# = 0 ]; do + case "$1" in + --) shift; break;; + -C|--directory) shift; $CHDIR "$1";; + -C*) $CHDIR `expr "$1" : '-C\(.*\)'`;; + --directory=*) $CHDIR `expr "$1" : '[^=]*=\(.*\)'`;; + -q) quiet=1;; + -b|--branch) shift; branch="$1";; + -b*) branch=`expr "$1" : '-b\(.*\)'`;; + --branch=*) branch=`expr "$1" : '[^=]*=\(.*\)'`;; + -*) echo "unknown option: $1" 1>&2; exit 1;; + *) break;; + esac + shift +done + +url="$1" +dir="$2" +shift 2 +[ x"$branch" = x ] && unset branch || : +if [ -d "$dir" ]; then + if [ x"$(git -C "$dir" describe --tags)" = x"$branch" ]; then + exit 0 # already up-to-date + fi + echo updating `expr "/$dir/" : '.*/\([^/][^/]*\)/'` ... + [ $quiet ] || set -x + $CHDIR "$dir" + ${branch+git} ${branch+fetch} ${branch+"$@"} + exec git ${branch+checkout} "${branch-pull}" "$@" +else + echo retrieving `expr "/$dir/" : '.*/\([^/][^/]*\)/'` ... + [ $quiet ] || set -x + exec git clone ${branch+--branch} ${branch+"$branch"} "$url" "$dir" "$@" +fi diff --git a/tool/gperf.sed b/tool/gperf.sed new file mode 100644 index 0000000000..6b3e1980be --- /dev/null +++ b/tool/gperf.sed @@ -0,0 +1,22 @@ +/ANSI-C code/{ + h + s/.*/ANSI:offset:/ + x +} +/\/\*!ANSI{\*\//{ + G + s/\/\*!ANSI{\*\/\(.*\)\/\*}!ANSI\*\/\(.*\)\nANSI:.*/\/\*\1\*\/\2/ +} +s/(int)([a-z_]*)&((struct \([a-zA-Z_0-9][a-zA-Z_0-9]*\)_t *\*)0)->\1_str\([1-9][0-9]*\),/gperf_offsetof(\1, \2),/g +/^#line/{ + G + x + s/:offset:/:/ + x + s/\(.*\)\(\n\).*:offset:.*/#define gperf_offsetof(s, n) (short)offsetof(struct s##_t, s##_str##n)\2\1/ + s/\n[^#].*// +} +/^[a-zA-Z_0-9]*hash/,/^}/{ + s/ hval = / hval = (unsigned int)/ + s/ return / return (unsigned int)/ +} diff --git a/tool/id2token.rb b/tool/id2token.rb new file mode 100755 index 0000000000..d12ea9c08e --- /dev/null +++ b/tool/id2token.rb @@ -0,0 +1,26 @@ +#! /usr/bin/ruby -p +# -*- coding: us-ascii -*- + +# Used to build the Ruby parsing code in common.mk and Ripper. + +BEGIN { + require 'optparse' + require_relative 'lib/vpath' + vpath = VPath.new + header = nil + + opt = OptionParser.new do |o| + vpath.def_options(o) + header = o.order!(ARGV).shift + end or abort opt.opt_s + + TOKENS = {} + h = vpath.read(header) rescue abort("#{header} not found in #{vpath.inspect}") + h.scan(/^#define\s+RUBY_TOKEN_(\w+)\s+(\d+)/) do |token, id| + TOKENS[token] = id + end + + TOKENS_RE = /\bRUBY_TOKEN\((#{TOKENS.keys.join('|')})\)\s*(?=\s)/ +} + +$_.gsub!(TOKENS_RE) {TOKENS[$1]} if /^%token/ =~ $_ diff --git a/tool/ifchange b/tool/ifchange new file mode 100755 index 0000000000..5af41e0156 --- /dev/null +++ b/tool/ifchange @@ -0,0 +1,119 @@ +#!/bin/sh +# usage: ifchange target temporary + +# Used in generating revision.h via Makefiles. + +help() { + cat <<HELP +usage: $0 [options] target new-file +options: + --timestamp[=file] touch timestamp file. (default: prefixed with ".time". + under the directory of the target) + --keep[=suffix] keep old file with suffix. (default: '.old') + --empty assume unchanged if the new file is empty. + --color[=always|auto|never] colorize output. +HELP +} + +set -e +timestamp= +keepsuffix= +empty= +color=auto +until [ $# -eq 0 ]; do + case "$1" in + --) + shift + break; + ;; + --timestamp) + timestamp=. + ;; + --timestamp=*) + timestamp=`expr \( "$1" : '[^=]*=\(.*\)' \)` + ;; + --keep) + keepsuffix=.old + ;; + --keep=*) + keepsuffix=`expr \( "$1" : '[^=]*=\(.*\)' \)` + ;; + --empty) + empty=yes + ;; + --color) + color=always + ;; + --color=*) + color=`expr \( "$1" : '[^=]*=\(.*\)' \)` + ;; + --debug) + set -x + ;; + --help) + help + exit + ;; + --*) + echo "$0: unknown option: $1" 1>&2 + exit 1 + ;; + *) + break + ;; + esac + shift +done + +if [ "$#" != 2 ]; then + help + exit 1 +fi + +target="$1" +temp="$2" +if [ "$temp" = - ]; then + temp="tmpdata$$.tmp~" + cat > "$temp" + trap 'rm -f "$temp"' 0 +fi + +msg_begin= msg_unchanged= msg_updated= msg_reset= +if [ "$color" = always -o \( "$color" = auto -a -t 1 \) ]; then + msg_begin="[" + case "`tput smso 2>/dev/null`" in + "$msg_begin"*m) + if [ ${TEST_COLORS:+set} ]; then + msg_unchanged=`expr ":$TEST_COLORS:" : ".*:pass=\([^:]*\):"` || : + msg_updated=`expr ":$TEST_COLORS:" : ".*:fail=\([^:]*\):"` || : + fi + msg_unchanged="${msg_begin}${msg_unchanged:-32}m" + msg_updated="${msg_begin}${msg_updated:-31;1}m" + msg_reset="${msg_begin}m" + ;; + esac + unset msg_begin +fi + +targetdir= +case "$target" in */*) targetdir=`dirname "$target"`;; esac +if [ -f "$target" -a ! -${empty:+f}${empty:-s} "$temp" ] || cmp "$target" "$temp" >/dev/null 2>&1; then + echo "$target ${msg_unchanged}unchanged${msg_reset}" + rm -f "$temp" +else + echo "$target ${msg_updated}updated${msg_reset}" + [ x"${targetdir}" = x -o -d "${targetdir}" ] || mkdir -p "${targetdir}" + [ x"${keepsuffix}" != x -a -f "$target" ] && mv -f "$target" "${target}${keepsuffix}" + mv -f "$temp" "$target" +fi + +if [ -n "${timestamp}" ]; then + if [ x"${timestamp}" = x. ]; then + if [ x"$targetdir" = x ]; then + timestamp=.time."$target" + else + timestamp="$targetdir"/.time.`basename "$target"` + fi + fi + : > "$timestamp" +fi diff --git a/tool/insns2vm.rb b/tool/insns2vm.rb new file mode 100755 index 0000000000..027dc4e380 --- /dev/null +++ b/tool/insns2vm.rb @@ -0,0 +1,15 @@ +#!ruby + +# This is used by Makefile.in to generate .inc files. +# See Makefile.in for details. + +require_relative 'ruby_vm/scripts/insns2vm' + +if $0 == __FILE__ + RubyVM::Insns2VM.router(ARGV).each do |(path, generator)| + str = generator.generate path + path.open 'wb:utf-8' do |fp| + fp.write str + end + end +end diff --git a/tool/install-sh b/tool/install-sh new file mode 100644 index 0000000000..11e502f56d --- /dev/null +++ b/tool/install-sh @@ -0,0 +1,17 @@ +#!/bin/sh + +# Just only for using AC_PROG_INSTALL in configure.ac. +# See autoconf.info for more detail. + +cat <<EOF >&2 +Ruby uses a BSD-compatible install(1) if possible. If not, Ruby +provides its own install(1) alternative. + +This script is a place holder for AC_PROG_INSTALL in configure.ac. + +Please report a bug in Ruby to http://bugs.ruby-lang.org if you see +this message. + +Thank you. +EOF +exit 1 diff --git a/tool/intern_ids.rb b/tool/intern_ids.rb new file mode 100755 index 0000000000..20483195e0 --- /dev/null +++ b/tool/intern_ids.rb @@ -0,0 +1,35 @@ +#!/usr/bin/ruby -sp +# $ ruby -i tool/intern_ids.rb -prefix=_ foo.c + +BEGIN { + $prefix ||= nil + + defs = File.join(File.dirname(__dir__), "defs/id.def") + ids = eval(File.read(defs), binding, defs) + table = {} + ids[:predefined].each {|v, t| table[t] = "id#{v}"} + ids[:token_op].each {|v, t, *| table[t] = "id#{v}"} + predefined = table.keys +} + +$_.gsub!(/rb_intern\("([^\"]+)"\)/) do + token = $1 + table[token] ||= "id" + id2varname(token, $prefix) +end + +END { + predefined.each {|t| table.delete(t)} + unless table.empty? + table = table.sort_by {|t, v| v} + + # Append at the last, then edit and move appropriately. + puts + puts "==== defs" + table.each {|t, v| puts "static ID #{v};"} + puts ">>>>" + puts + puts "==== init" + table.each {|t, v|puts "#{v} = rb_intern_const(\"#{t}\");"} + puts ">>>>" + end +} diff --git a/tool/leaked-globals b/tool/leaked-globals new file mode 100755 index 0000000000..d95f3794e8 --- /dev/null +++ b/tool/leaked-globals @@ -0,0 +1,65 @@ +#!/usr/bin/ruby +require_relative 'lib/colorize' + +until ARGV.empty? + case ARGV[0] + when /\ASYMBOL_PREFIX=(.*)/ + SYMBOL_PREFIX = $1 + when /\ANM=(.*)/ # may be multiple words + NM = $1 + when /\APLATFORM=(.+)?/ + platform = $1 + else + break + end + ARGV.shift +end + +config = ARGV.shift +count = 0 +col = Colorize.new +config_code = File.read(config) +REPLACE = config_code.scan(/\bAC_(?:REPLACE|CHECK)_FUNCS?\((\w+)/).flatten +# REPLACE << 'memcmp' if /\bAC_FUNC_MEMCMP\b/ =~ config_code +REPLACE.push('main', 'DllMain') +if platform and !platform.empty? + begin + h = File.read(platform) + rescue Errno::ENOENT + else + REPLACE.concat( + h .gsub(%r[/\*.*?\*/]m, " ") # delete block comments + .gsub(%r[//.*], ' ') # delete oneline comments + .gsub(/^\s*#.*(?:\\\n.*)*/, "") # delete preprocessor directives + .scan(/\b((?!rb_|DEPRECATED|_)\w+)\s*\(.*\);/) + .flatten) + end +end +missing = File.dirname(config) + "/missing/" +ARGV.reject! do |n| + unless (src = Dir.glob(missing + File.basename(n, ".*") + ".[cS]")).empty? + puts "Ignore #{n} because of #{src.map {|s| File.basename(s)}.join(', ')} under missing" + true + end +end +print "Checking leaked global symbols..." +STDOUT.flush +IO.foreach("|#{NM} -Pgp #{ARGV.join(' ')}") do |line| + n, t, = line.split + next unless /[A-TV-Z]/ =~ t + next unless n.sub!(/^#{SYMBOL_PREFIX}/o, "") + next if n.include?(".") + next if /\A(?:Init_|InitVM_|RUBY_|ruby_|rb_|[Oo]nig|dln_|mjit_|coroutine_)/ =~ n + next if REPLACE.include?(n) + puts col.fail("leaked") if count.zero? + count += 1 + puts " #{n}" +end +case count +when 0 + puts col.pass("none") +when 1 + abort col.fail("1 un-prefixed symbol leaked") +else + abort col.fail("#{count} un-prefixed symbols leaked") +end diff --git a/tool/lib/-test-/integer.rb b/tool/lib/-test-/integer.rb new file mode 100644 index 0000000000..e60abf03a0 --- /dev/null +++ b/tool/lib/-test-/integer.rb @@ -0,0 +1,14 @@ +require 'test/unit' +require '-test-/integer.so' + +module Test::Unit::Assertions + def assert_fixnum(v, msg=nil) + assert_instance_of(Integer, v, msg) + assert_send([Bug::Integer, :fixnum?, v], msg) + end + + def assert_bignum(v, msg=nil) + assert_instance_of(Integer, v, msg) + assert_send([Bug::Integer, :bignum?, v], msg) + end +end diff --git a/tool/lib/bundled_gem.rb b/tool/lib/bundled_gem.rb new file mode 100644 index 0000000000..895aed4510 --- /dev/null +++ b/tool/lib/bundled_gem.rb @@ -0,0 +1,68 @@ +require 'fileutils' +require 'rubygems' +require 'rubygems/package' + +# This library is used by "make extract-gems" to +# unpack bundled gem files. + +module BundledGem + DEFAULT_GEMS_DEPENDENCIES = [ + "net-protocol", # net-ftp + "time", # net-ftp + "singleton", # prime + "ipaddr", # rinda + "forwardable", # prime, rinda + "ruby2_keywords", # drb + "strscan" # rexml + ] + + module_function + + def unpack(file, *rest) + pkg = Gem::Package.new(file) + prepare_test(pkg.spec, *rest) {|dir| pkg.extract_files(dir)} + puts "Unpacked #{file}" + end + + def copy(path, *rest) + spec = Gem::Specification.load(path) + path = File.dirname(path) + prepare_test(spec, *rest) do |dir| + FileUtils.rm_rf(dir) + files = spec.files.reject {|f| f.start_with?(".git")} + dirs = files.map {|f| File.dirname(f) if f.include?("/")}.uniq + FileUtils.mkdir_p(dirs.map {|d| d ? "#{dir}/#{d}" : dir}.sort_by {|d| d.count("/")}) + files.each do |f| + File.copy_stream(File.join(path, f), File.join(dir, f)) + end + end + puts "Copied #{path}" + end + + def prepare_test(spec, dir = ".") + target = spec.full_name + Gem.ensure_gem_subdirectories(dir) + gem_dir = File.join(dir, "gems", target) + yield gem_dir + spec_dir = spec.extensions.empty? ? "specifications" : File.join("gems", target) + if spec.extensions.empty? + spec.dependencies.reject! {|dep| DEFAULT_GEMS_DEPENDENCIES.include?(dep.name)} + end + File.binwrite(File.join(dir, spec_dir, "#{target}.gemspec"), spec.to_ruby) + unless spec.extensions.empty? + spec.dependencies.clear + File.binwrite(File.join(dir, spec_dir, ".bundled.#{target}.gemspec"), spec.to_ruby) + end + if spec.bindir and spec.executables + bindir = File.join(dir, "bin") + Dir.mkdir(bindir) rescue nil + spec.executables.each do |exe| + File.open(File.join(bindir, exe), "wb", 0o777) {|f| + f.print "#!ruby\n", + %[load File.realpath("../gems/#{target}/#{spec.bindir}/#{exe}", __dir__)\n] + } + end + end + FileUtils.rm_rf(Dir.glob("#{gem_dir}/.git*")) + end +end diff --git a/tool/lib/colorize.rb b/tool/lib/colorize.rb new file mode 100644 index 0000000000..11b878d318 --- /dev/null +++ b/tool/lib/colorize.rb @@ -0,0 +1,55 @@ +# frozen-string-literal: true + +class Colorize + # call-seq: + # Colorize.new(colorize = nil) + # Colorize.new(color: color, colors_file: colors_file) + def initialize(color = nil, opts = ((_, color = color, nil)[0] if Hash === color)) + @colors = @reset = nil + @color = (opts[:color] if opts) + if color or (color == nil && STDOUT.tty?) + if (%w[smso so].any? {|attr| /\A\e\[.*m\z/ =~ IO.popen("tput #{attr}", "r", :err => IO::NULL, &:read)} rescue nil) + @beg = "\e[" + colors = (colors = ENV['TEST_COLORS']) ? Hash[colors.scan(/(\w+)=([^:\n]*)/)] : {} + if opts and colors_file = opts[:colors_file] + begin + File.read(colors_file).scan(/(\w+)=([^:\n]*)/) do |n, c| + colors[n] ||= c + end + rescue Errno::ENOENT + end + end + @colors = colors + @reset = "#{@beg}m" + end + end + self + end + + DEFAULTS = { + "pass"=>"32", "fail"=>"31;1", "skip"=>"33;1", + "black"=>"30", "red"=>"31", "green"=>"32", "yellow"=>"33", + "blue"=>"34", "magenta"=>"35", "cyan"=>"36", "white"=>"37", + "bold"=>"1", "underline"=>"4", "reverse"=>"7", + } + + # colorize.decorate(str, name = color_name) + def decorate(str, name = @color) + if @colors and color = (@colors[name] || DEFAULTS[name]) + "#{@beg}#{color}m#{str}#{@reset}" + else + str + end + end + + DEFAULTS.each_key do |name| + define_method(name) {|str| + decorate(str, name) + } + end +end + +if $0 == __FILE__ + colorize = Colorize.new(ARGV.shift) + ARGV.each {|str| puts colorize.decorate(str)} +end diff --git a/tool/lib/core_assertions.rb b/tool/lib/core_assertions.rb new file mode 100644 index 0000000000..acfaf00cef --- /dev/null +++ b/tool/lib/core_assertions.rb @@ -0,0 +1,809 @@ +# frozen_string_literal: true + +module Test + module Unit + module Assertions + def _assertions= n # :nodoc: + @_assertions = n + end + + def _assertions # :nodoc: + @_assertions ||= 0 + end + + ## + # Returns a proc that will output +msg+ along with the default message. + + def message msg = nil, ending = nil, &default + proc { + ending ||= (ending_pattern = /(?<!\.)\z/; ".") + ending_pattern ||= /(?<!#{Regexp.quote(ending)})\z/ + msg = msg.call if Proc === msg + ary = [msg, (default.call if default)].compact.reject(&:empty?) + ary.map! {|str| str.to_s.sub(ending_pattern, ending) } + begin + ary.join("\n") + rescue Encoding::CompatibilityError + ary.map(&:b).join("\n") + end + } + end + end + + module CoreAssertions + require_relative 'envutil' + require 'pp' + nil.pretty_inspect + + def mu_pp(obj) #:nodoc: + obj.pretty_inspect.chomp + end + + def assert_file + AssertFile + end + + FailDesc = proc do |status, message = "", out = ""| + now = Time.now + proc do + EnvUtil.failure_description(status, now, message, out) + end + end + + def assert_in_out_err(args, test_stdin = "", test_stdout = [], test_stderr = [], message = nil, + success: nil, **opt) + args = Array(args).dup + args.insert((Hash === args[0] ? 1 : 0), '--disable=gems') + stdout, stderr, status = EnvUtil.invoke_ruby(args, test_stdin, true, true, **opt) + desc = FailDesc[status, message, stderr] + if block_given? + raise "test_stdout ignored, use block only or without block" if test_stdout != [] + raise "test_stderr ignored, use block only or without block" if test_stderr != [] + yield(stdout.lines.map {|l| l.chomp }, stderr.lines.map {|l| l.chomp }, status) + else + all_assertions(desc) do |a| + [["stdout", test_stdout, stdout], ["stderr", test_stderr, stderr]].each do |key, exp, act| + a.for(key) do + if exp.is_a?(Regexp) + assert_match(exp, act) + elsif exp.all? {|e| String === e} + assert_equal(exp, act.lines.map {|l| l.chomp }) + else + assert_pattern_list(exp, act) + end + end + end + unless success.nil? + a.for("success?") do + if success + assert_predicate(status, :success?) + else + assert_not_predicate(status, :success?) + end + end + end + end + status + end + end + + if defined?(RubyVM::InstructionSequence) + def syntax_check(code, fname, line) + code = code.dup.force_encoding(Encoding::UTF_8) + RubyVM::InstructionSequence.compile(code, fname, fname, line) + :ok + ensure + raise if SyntaxError === $! + end + else + def syntax_check(code, fname, line) + code = code.b + code.sub!(/\A(?:\xef\xbb\xbf)?(\s*\#.*$)*(\n)?/n) { + "#$&#{"\n" if $1 && !$2}BEGIN{throw tag, :ok}\n" + } + code = code.force_encoding(Encoding::UTF_8) + catch {|tag| eval(code, binding, fname, line - 1)} + end + end + + def assert_no_memory_leak(args, prepare, code, message=nil, limit: 2.0, rss: false, **opt) + # TODO: consider choosing some appropriate limit for MJIT and stop skipping this once it does not randomly fail + pend 'assert_no_memory_leak may consider MJIT memory usage as leak' if defined?(RubyVM::MJIT) && RubyVM::MJIT.enabled? + + require_relative 'memory_status' + raise Test::Unit::PendedError, "unsupported platform" unless defined?(Memory::Status) + + token_dump, token_re = new_test_token + envs = args.shift if Array === args and Hash === args.first + args = [ + "--disable=gems", + "-r", File.expand_path("../memory_status", __FILE__), + *args, + "-v", "-", + ] + if defined? Memory::NO_MEMORY_LEAK_ENVS then + envs ||= {} + newenvs = envs.merge(Memory::NO_MEMORY_LEAK_ENVS) { |_, _, _| break } + envs = newenvs if newenvs + end + args.unshift(envs) if envs + cmd = [ + 'END {STDERR.puts '"#{token_dump}"'"FINAL=#{Memory::Status.new}"}', + prepare, + 'STDERR.puts('"#{token_dump}"'"START=#{$initial_status = Memory::Status.new}")', + '$initial_size = $initial_status.size', + code, + 'GC.start', + ].join("\n") + _, err, status = EnvUtil.invoke_ruby(args, cmd, true, true, **opt) + before = err.sub!(/^#{token_re}START=(\{.*\})\n/, '') && Memory::Status.parse($1) + after = err.sub!(/^#{token_re}FINAL=(\{.*\})\n/, '') && Memory::Status.parse($1) + assert(status.success?, FailDesc[status, message, err]) + ([:size, (rss && :rss)] & after.members).each do |n| + b = before[n] + a = after[n] + next unless a > 0 and b > 0 + assert_operator(a.fdiv(b), :<, limit, message(message) {"#{n}: #{b} => #{a}"}) + end + rescue LoadError + pend + end + + # :call-seq: + # assert_nothing_raised( *args, &block ) + # + #If any exceptions are given as arguments, the assertion will + #fail if one of those exceptions are raised. Otherwise, the test fails + #if any exceptions are raised. + # + #The final argument may be a failure message. + # + # assert_nothing_raised RuntimeError do + # raise Exception #Assertion passes, Exception is not a RuntimeError + # end + # + # assert_nothing_raised do + # raise Exception #Assertion fails + # end + def assert_nothing_raised(*args) + self._assertions += 1 + if Module === args.last + msg = nil + else + msg = args.pop + end + begin + yield + rescue Test::Unit::PendedError, *(Test::Unit::AssertionFailedError if args.empty?) + raise + rescue *(args.empty? ? Exception : args) => e + msg = message(msg) { + "Exception raised:\n<#{mu_pp(e)}>\n""Backtrace:\n" << + Test.filter_backtrace(e.backtrace).map{|frame| " #{frame}"}.join("\n") + } + raise Test::Unit::AssertionFailedError, msg.call, e.backtrace + end + end + + def prepare_syntax_check(code, fname = nil, mesg = nil, verbose: nil) + fname ||= caller_locations(2, 1)[0] + mesg ||= fname.to_s + verbose, $VERBOSE = $VERBOSE, verbose + case + when Array === fname + fname, line = *fname + when defined?(fname.path) && defined?(fname.lineno) + fname, line = fname.path, fname.lineno + else + line = 1 + end + yield(code, fname, line, message(mesg) { + if code.end_with?("\n") + "```\n#{code}```\n" + else + "```\n#{code}\n```\n""no-newline" + end + }) + ensure + $VERBOSE = verbose + end + + def assert_valid_syntax(code, *args, **opt) + prepare_syntax_check(code, *args, **opt) do |src, fname, line, mesg| + yield if defined?(yield) + assert_nothing_raised(SyntaxError, mesg) do + assert_equal(:ok, syntax_check(src, fname, line), mesg) + end + end + end + + def assert_normal_exit(testsrc, message = '', child_env: nil, **opt) + assert_valid_syntax(testsrc, caller_locations(1, 1)[0]) + if child_env + child_env = [child_env] + else + child_env = [] + end + out, _, status = EnvUtil.invoke_ruby(child_env + %W'-W0', testsrc, true, :merge_to_stdout, **opt) + assert !status.signaled?, FailDesc[status, message, out] + end + + def assert_ruby_status(args, test_stdin="", message=nil, **opt) + out, _, status = EnvUtil.invoke_ruby(args, test_stdin, true, :merge_to_stdout, **opt) + desc = FailDesc[status, message, out] + assert(!status.signaled?, desc) + message ||= "ruby exit status is not success:" + assert(status.success?, desc) + end + + ABORT_SIGNALS = Signal.list.values_at(*%w"ILL ABRT BUS SEGV TERM") + + def separated_runner(token, out = nil) + include(*Test::Unit::TestCase.ancestors.select {|c| !c.is_a?(Class) }) + out = out ? IO.new(out, 'w') : STDOUT + at_exit { + out.puts "#{token}<error>", [Marshal.dump($!)].pack('m'), "#{token}</error>", "#{token}assertions=#{self._assertions}" + } + Test::Unit::Runner.class_variable_set(:@@stop_auto_run, true) if defined?(Test::Unit::Runner) + end + + def assert_separately(args, file = nil, line = nil, src, ignore_stderr: nil, **opt) + unless file and line + loc, = caller_locations(1,1) + file ||= loc.path + line ||= loc.lineno + end + capture_stdout = true + unless /mswin|mingw/ =~ RUBY_PLATFORM + capture_stdout = false + opt[:out] = Test::Unit::Runner.output if defined?(Test::Unit::Runner) + res_p, res_c = IO.pipe + opt[:ios] = [res_c] + end + token_dump, token_re = new_test_token + src = <<eom +# -*- coding: #{line += __LINE__; src.encoding}; -*- +BEGIN { + require "test/unit";include Test::Unit::Assertions;require #{__FILE__.dump};include Test::Unit::CoreAssertions + separated_runner #{token_dump}, #{res_c&.fileno || 'nil'} +} +#{line -= __LINE__; src} +eom + args = args.dup + args.insert((Hash === args.first ? 1 : 0), "-w", "--disable=gems", *$:.map {|l| "-I#{l}"}) + args << "--debug" if RUBY_ENGINE == 'jruby' # warning: tracing (e.g. set_trace_func) will not capture all events without --debug flag + stdout, stderr, status = EnvUtil.invoke_ruby(args, src, capture_stdout, true, **opt) + ensure + if res_c + res_c.close + res = res_p.read + res_p.close + else + res = stdout + end + raise if $! + abort = status.coredump? || (status.signaled? && ABORT_SIGNALS.include?(status.termsig)) + assert(!abort, FailDesc[status, nil, stderr]) + self._assertions += res[/^#{token_re}assertions=(\d+)/, 1].to_i + begin + res = Marshal.load(res[/^#{token_re}<error>\n\K.*\n(?=#{token_re}<\/error>$)/m].unpack1("m")) + rescue => marshal_error + ignore_stderr = nil + res = nil + end + if res and !(SystemExit === res) + if bt = res.backtrace + bt.each do |l| + l.sub!(/\A-:(\d+)/){"#{file}:#{line + $1.to_i}"} + end + bt.concat(caller) + else + res.set_backtrace(caller) + end + raise res + end + + # really is it succeed? + unless ignore_stderr + # the body of assert_separately must not output anything to detect error + assert(stderr.empty?, FailDesc[status, "assert_separately failed with error message", stderr]) + end + assert(status.success?, FailDesc[status, "assert_separately failed", stderr]) + raise marshal_error if marshal_error + end + + # Run Ractor-related test without influencing the main test suite + def assert_ractor(src, args: [], require: nil, require_relative: nil, file: nil, line: nil, ignore_stderr: nil, **opt) + return unless defined?(Ractor) + + require = "require #{require.inspect}" if require + if require_relative + dir = File.dirname(caller_locations[0,1][0].absolute_path) + full_path = File.expand_path(require_relative, dir) + require = "#{require}; require #{full_path.inspect}" + end + + assert_separately(args, file, line, <<~RUBY, ignore_stderr: ignore_stderr, **opt) + #{require} + previous_verbose = $VERBOSE + $VERBOSE = nil + Ractor.new {} # trigger initial warning + $VERBOSE = previous_verbose + #{src} + RUBY + end + + # :call-seq: + # assert_throw( tag, failure_message = nil, &block ) + # + #Fails unless the given block throws +tag+, returns the caught + #value otherwise. + # + #An optional failure message may be provided as the final argument. + # + # tag = Object.new + # assert_throw(tag, "#{tag} was not thrown!") do + # throw tag + # end + def assert_throw(tag, msg = nil) + ret = catch(tag) do + begin + yield(tag) + rescue UncaughtThrowError => e + thrown = e.tag + end + msg = message(msg) { + "Expected #{mu_pp(tag)} to have been thrown"\ + "#{%Q[, not #{thrown}] if thrown}" + } + assert(false, msg) + end + assert(true) + ret + end + + # :call-seq: + # assert_raise( *args, &block ) + # + #Tests if the given block raises an exception. Acceptable exception + #types may be given as optional arguments. If the last argument is a + #String, it will be used as the error message. + # + # assert_raise do #Fails, no Exceptions are raised + # end + # + # assert_raise NameError do + # puts x #Raises NameError, so assertion succeeds + # end + def assert_raise(*exp, &b) + case exp.last + when String, Proc + msg = exp.pop + end + + begin + yield + rescue Test::Unit::PendedError => e + return e if exp.include? Test::Unit::PendedError + raise e + rescue Exception => e + expected = exp.any? { |ex| + if ex.instance_of? Module then + e.kind_of? ex + else + e.instance_of? ex + end + } + + assert expected, proc { + flunk(message(msg) {"#{mu_pp(exp)} exception expected, not #{mu_pp(e)}"}) + } + + return e + ensure + unless e + exp = exp.first if exp.size == 1 + + flunk(message(msg) {"#{mu_pp(exp)} expected but nothing was raised"}) + end + end + end + + # :call-seq: + # assert_raise_with_message(exception, expected, msg = nil, &block) + # + #Tests if the given block raises an exception with the expected + #message. + # + # assert_raise_with_message(RuntimeError, "foo") do + # nil #Fails, no Exceptions are raised + # end + # + # assert_raise_with_message(RuntimeError, "foo") do + # raise ArgumentError, "foo" #Fails, different Exception is raised + # end + # + # assert_raise_with_message(RuntimeError, "foo") do + # raise "bar" #Fails, RuntimeError is raised but the message differs + # end + # + # assert_raise_with_message(RuntimeError, "foo") do + # raise "foo" #Raises RuntimeError with the message, so assertion succeeds + # end + def assert_raise_with_message(exception, expected, msg = nil, &block) + case expected + when String + assert = :assert_equal + when Regexp + assert = :assert_match + else + raise TypeError, "Expected #{expected.inspect} to be a kind of String or Regexp, not #{expected.class}" + end + + ex = m = nil + EnvUtil.with_default_internal(expected.encoding) do + ex = assert_raise(exception, msg || proc {"Exception(#{exception}) with message matches to #{expected.inspect}"}) do + yield + end + m = ex.message + end + msg = message(msg, "") {"Expected Exception(#{exception}) was raised, but the message doesn't match"} + + if assert == :assert_equal + assert_equal(expected, m, msg) + else + msg = message(msg) { "Expected #{mu_pp expected} to match #{mu_pp m}" } + assert expected =~ m, msg + block.binding.eval("proc{|_|$~=_}").call($~) + end + ex + end + + TEST_DIR = File.join(__dir__, "test/unit") #:nodoc: + + # :call-seq: + # assert(test, [failure_message]) + # + #Tests if +test+ is true. + # + #+msg+ may be a String or a Proc. If +msg+ is a String, it will be used + #as the failure message. Otherwise, the result of calling +msg+ will be + #used as the message if the assertion fails. + # + #If no +msg+ is given, a default message will be used. + # + # assert(false, "This was expected to be true") + def assert(test, *msgs) + case msg = msgs.first + when String, Proc + when nil + msgs.shift + else + bt = caller.reject { |s| s.start_with?(TEST_DIR) } + raise ArgumentError, "assertion message must be String or Proc, but #{msg.class} was given.", bt + end unless msgs.empty? + super + end + + # :call-seq: + # assert_respond_to( object, method, failure_message = nil ) + # + #Tests if the given Object responds to +method+. + # + #An optional failure message may be provided as the final argument. + # + # assert_respond_to("hello", :reverse) #Succeeds + # assert_respond_to("hello", :does_not_exist) #Fails + def assert_respond_to(obj, (meth, *priv), msg = nil) + unless priv.empty? + msg = message(msg) { + "Expected #{mu_pp(obj)} (#{obj.class}) to respond to ##{meth}#{" privately" if priv[0]}" + } + return assert obj.respond_to?(meth, *priv), msg + end + #get rid of overcounting + if caller_locations(1, 1)[0].path.start_with?(TEST_DIR) + return if obj.respond_to?(meth) + end + super(obj, meth, msg) + end + + # :call-seq: + # assert_not_respond_to( object, method, failure_message = nil ) + # + #Tests if the given Object does not respond to +method+. + # + #An optional failure message may be provided as the final argument. + # + # assert_not_respond_to("hello", :reverse) #Fails + # assert_not_respond_to("hello", :does_not_exist) #Succeeds + def assert_not_respond_to(obj, (meth, *priv), msg = nil) + unless priv.empty? + msg = message(msg) { + "Expected #{mu_pp(obj)} (#{obj.class}) to not respond to ##{meth}#{" privately" if priv[0]}" + } + return assert !obj.respond_to?(meth, *priv), msg + end + #get rid of overcounting + if caller_locations(1, 1)[0].path.start_with?(TEST_DIR) + return unless obj.respond_to?(meth) + end + refute_respond_to(obj, meth, msg) + end + + # pattern_list is an array which contains regexp and :*. + # :* means any sequence. + # + # pattern_list is anchored. + # Use [:*, regexp, :*] for non-anchored match. + def assert_pattern_list(pattern_list, actual, message=nil) + rest = actual + anchored = true + pattern_list.each_with_index {|pattern, i| + if pattern == :* + anchored = false + else + if anchored + match = /\A#{pattern}/.match(rest) + else + match = pattern.match(rest) + end + unless match + msg = message(msg) { + expect_msg = "Expected #{mu_pp pattern}\n" + if /\n[^\n]/ =~ rest + actual_mesg = +"to match\n" + rest.scan(/.*\n+/) { + actual_mesg << ' ' << $&.inspect << "+\n" + } + actual_mesg.sub!(/\+\n\z/, '') + else + actual_mesg = "to match " + mu_pp(rest) + end + actual_mesg << "\nafter #{i} patterns with #{actual.length - rest.length} characters" + expect_msg + actual_mesg + } + assert false, msg + end + rest = match.post_match + anchored = true + end + } + if anchored + assert_equal("", rest) + end + end + + def assert_warning(pat, msg = nil) + result = nil + stderr = EnvUtil.with_default_internal(pat.encoding) { + EnvUtil.verbose_warning { + result = yield + } + } + msg = message(msg) {diff pat, stderr} + assert(pat === stderr, msg) + result + end + + def assert_warn(*args) + assert_warning(*args) {$VERBOSE = false; yield} + end + + def assert_deprecated_warning(mesg = /deprecated/) + assert_warning(mesg) do + Warning[:deprecated] = true + yield + end + end + + def assert_deprecated_warn(mesg = /deprecated/) + assert_warn(mesg) do + Warning[:deprecated] = true + yield + end + end + + class << (AssertFile = Struct.new(:failure_message).new) + include Assertions + include CoreAssertions + def assert_file_predicate(predicate, *args) + if /\Anot_/ =~ predicate + predicate = $' + neg = " not" + end + result = File.__send__(predicate, *args) + result = !result if neg + mesg = "Expected file ".dup << args.shift.inspect + mesg << "#{neg} to be #{predicate}" + mesg << mu_pp(args).sub(/\A\[(.*)\]\z/m, '(\1)') unless args.empty? + mesg << " #{failure_message}" if failure_message + assert(result, mesg) + end + alias method_missing assert_file_predicate + + def for(message) + clone.tap {|a| a.failure_message = message} + end + end + + class AllFailures + attr_reader :failures + + def initialize + @count = 0 + @failures = {} + end + + def for(key) + @count += 1 + yield key + rescue Exception => e + @failures[key] = [@count, e] + end + + def foreach(*keys) + keys.each do |key| + @count += 1 + begin + yield key + rescue Exception => e + @failures[key] = [@count, e] + end + end + end + + def message + i = 0 + total = @count.to_s + fmt = "%#{total.size}d" + @failures.map {|k, (n, v)| + v = v.message + "\n#{i+=1}. [#{fmt%n}/#{total}] Assertion for #{k.inspect}\n#{v.b.gsub(/^/, ' | ').force_encoding(v.encoding)}" + }.join("\n") + end + + def pass? + @failures.empty? + end + end + + # threads should respond to shift method. + # Array can be used. + def assert_join_threads(threads, message = nil) + errs = [] + values = [] + while th = threads.shift + begin + values << th.value + rescue Exception + errs << [th, $!] + th = nil + end + end + values + ensure + if th&.alive? + th.raise(Timeout::Error.new) + th.join rescue errs << [th, $!] + end + if !errs.empty? + msg = "exceptions on #{errs.length} threads:\n" + + errs.map {|t, err| + "#{t.inspect}:\n" + + RUBY_VERSION >= "2.5.0" ? err.full_message(highlight: false, order: :top) : err.message + }.join("\n---\n") + if message + msg = "#{message}\n#{msg}" + end + raise Test::Unit::AssertionFailedError, msg + end + end + + def assert_all?(obj, m = nil, &blk) + failed = [] + obj.each do |*a, &b| + unless blk.call(*a, &b) + failed << (a.size > 1 ? a : a[0]) + end + end + assert(failed.empty?, message(m) {failed.pretty_inspect}) + end + + def assert_all_assertions(msg = nil) + all = AllFailures.new + yield all + ensure + assert(all.pass?, message(msg) {all.message.chomp(".")}) + end + alias all_assertions assert_all_assertions + + def assert_all_assertions_foreach(msg = nil, *keys, &block) + all = AllFailures.new + all.foreach(*keys, &block) + ensure + assert(all.pass?, message(msg) {all.message.chomp(".")}) + end + alias all_assertions_foreach assert_all_assertions_foreach + + %w[ + CLOCK_THREAD_CPUTIME_ID CLOCK_PROCESS_CPUTIME_ID + CLOCK_MONOTONIC + ].find do |c| + if Process.const_defined?(c) + [c.to_sym, Process.const_get(c)].find do |clk| + begin + Process.clock_gettime(clk) + rescue + # Constants may be defined but not implemented, e.g., mingw. + else + PERFORMANCE_CLOCK = clk + end + end + end + end + + # Expect +seq+ to respond to +first+ and +each+ methods, e.g., + # Array, Range, Enumerator::ArithmeticSequence and other + # Enumerable-s, and each elements should be size factors. + # + # :yield: each elements of +seq+. + def assert_linear_performance(seq, rehearsal: nil, pre: ->(n) {n}) + pend "No PERFORMANCE_CLOCK found" unless defined?(PERFORMANCE_CLOCK) + + # Timeout testing generally doesn't work when RJIT compilation happens. + rjit_enabled = defined?(RubyVM::RJIT) && RubyVM::RJIT.enabled? + measure = proc do |arg, message| + st = Process.clock_gettime(PERFORMANCE_CLOCK) + yield(*arg) + t = (Process.clock_gettime(PERFORMANCE_CLOCK) - st) + assert_operator 0, :<=, t, message unless rjit_enabled + t + end + + first = seq.first + *arg = pre.call(first) + times = (0..(rehearsal || (2 * first))).map do + measure[arg, "rehearsal"].nonzero? + end + times.compact! + tmin, tmax = times.minmax + + # safe_factor * tmax * rehearsal_time_variance_factor(equals to 1 when variance is small) + tbase = 10 * tmax * [(tmax / tmin) ** 2 / 4, 1].max + info = "(tmin: #{tmin}, tmax: #{tmax}, tbase: #{tbase})" + + seq.each do |i| + next if i == first + t = tbase * i.fdiv(first) + *arg = pre.call(i) + message = "[#{i}]: in #{t}s #{info}" + Timeout.timeout(t, Timeout::Error, message) do + measure[arg, message] + end + end + end + + def diff(exp, act) + require 'pp' + q = PP.new(+"") + q.guard_inspect_key do + q.group(2, "expected: ") do + q.pp exp + end + q.text q.newline + q.group(2, "actual: ") do + q.pp act + end + q.flush + end + q.output + end + + def new_test_token + token = "\e[7;1m#{$$.to_s}:#{Time.now.strftime('%s.%L')}:#{rand(0x10000).to_s(16)}:\e[m" + return token.dump, Regexp.quote(token) + end + end + end +end diff --git a/tool/lib/envutil.rb b/tool/lib/envutil.rb new file mode 100644 index 0000000000..0391b90c1c --- /dev/null +++ b/tool/lib/envutil.rb @@ -0,0 +1,367 @@ +# -*- coding: us-ascii -*- +# frozen_string_literal: true +require "open3" +require "timeout" +require_relative "find_executable" +begin + require 'rbconfig' +rescue LoadError +end +begin + require "rbconfig/sizeof" +rescue LoadError +end + +module EnvUtil + def rubybin + if ruby = ENV["RUBY"] + return ruby + end + ruby = "ruby" + exeext = RbConfig::CONFIG["EXEEXT"] + rubyexe = (ruby + exeext if exeext and !exeext.empty?) + 3.times do + if File.exist? ruby and File.executable? ruby and !File.directory? ruby + return File.expand_path(ruby) + end + if rubyexe and File.exist? rubyexe and File.executable? rubyexe + return File.expand_path(rubyexe) + end + ruby = File.join("..", ruby) + end + if defined?(RbConfig.ruby) + RbConfig.ruby + else + "ruby" + end + end + module_function :rubybin + + LANG_ENVS = %w"LANG LC_ALL LC_CTYPE" + + DEFAULT_SIGNALS = Signal.list + DEFAULT_SIGNALS.delete("TERM") if /mswin|mingw/ =~ RUBY_PLATFORM + + RUBYLIB = ENV["RUBYLIB"] + + class << self + attr_accessor :timeout_scale + attr_reader :original_internal_encoding, :original_external_encoding, + :original_verbose, :original_warning + + def capture_global_values + @original_internal_encoding = Encoding.default_internal + @original_external_encoding = Encoding.default_external + @original_verbose = $VERBOSE + @original_warning = defined?(Warning.[]) ? %i[deprecated experimental].to_h {|i| [i, Warning[i]]} : nil + end + end + + def apply_timeout_scale(t) + if scale = EnvUtil.timeout_scale + t * scale + else + t + end + end + module_function :apply_timeout_scale + + def timeout(sec, klass = nil, message = nil, &blk) + return yield(sec) if sec == nil or sec.zero? + sec = apply_timeout_scale(sec) + Timeout.timeout(sec, klass, message, &blk) + end + module_function :timeout + + def terminate(pid, signal = :TERM, pgroup = nil, reprieve = 1) + reprieve = apply_timeout_scale(reprieve) if reprieve + + signals = Array(signal).select do |sig| + DEFAULT_SIGNALS[sig.to_s] or + DEFAULT_SIGNALS[Signal.signame(sig)] rescue false + end + signals |= [:ABRT, :KILL] + case pgroup + when 0, true + pgroup = -pid + when nil, false + pgroup = pid + end + + lldb = true if /darwin/ =~ RUBY_PLATFORM + + while signal = signals.shift + + if lldb and [:ABRT, :KILL].include?(signal) + lldb = false + # sudo -n: --non-interactive + # lldb -p: attach + # -o: run command + system(*%W[sudo -n lldb -p #{pid} --batch -o bt\ all -o call\ rb_vmdebug_stack_dump_all_threads() -o quit]) + true + end + + begin + Process.kill signal, pgroup + rescue Errno::EINVAL + next + rescue Errno::ESRCH + break + end + if signals.empty? or !reprieve + Process.wait(pid) + else + begin + Timeout.timeout(reprieve) {Process.wait(pid)} + rescue Timeout::Error + else + break + end + end + end + $? + end + module_function :terminate + + def invoke_ruby(args, stdin_data = "", capture_stdout = false, capture_stderr = false, + encoding: nil, timeout: 10, reprieve: 1, timeout_error: Timeout::Error, + stdout_filter: nil, stderr_filter: nil, ios: nil, + signal: :TERM, + rubybin: EnvUtil.rubybin, precommand: nil, + **opt) + timeout = apply_timeout_scale(timeout) + + in_c, in_p = IO.pipe + out_p, out_c = IO.pipe if capture_stdout + err_p, err_c = IO.pipe if capture_stderr && capture_stderr != :merge_to_stdout + opt[:in] = in_c + opt[:out] = out_c if capture_stdout + opt[:err] = capture_stderr == :merge_to_stdout ? out_c : err_c if capture_stderr + if encoding + out_p.set_encoding(encoding) if out_p + err_p.set_encoding(encoding) if err_p + end + ios.each {|i, o = i|opt[i] = o} if ios + + c = "C" + child_env = {} + LANG_ENVS.each {|lc| child_env[lc] = c} + if Array === args and Hash === args.first + child_env.update(args.shift) + end + if RUBYLIB and lib = child_env["RUBYLIB"] + child_env["RUBYLIB"] = [lib, RUBYLIB].join(File::PATH_SEPARATOR) + end + child_env['ASAN_OPTIONS'] = ENV['ASAN_OPTIONS'] if ENV['ASAN_OPTIONS'] + args = [args] if args.kind_of?(String) + pid = spawn(child_env, *precommand, rubybin, *args, opt) + in_c.close + out_c&.close + out_c = nil + err_c&.close + err_c = nil + if block_given? + return yield in_p, out_p, err_p, pid + else + th_stdout = Thread.new { out_p.read } if capture_stdout + th_stderr = Thread.new { err_p.read } if capture_stderr && capture_stderr != :merge_to_stdout + in_p.write stdin_data.to_str unless stdin_data.empty? + in_p.close + if (!th_stdout || th_stdout.join(timeout)) && (!th_stderr || th_stderr.join(timeout)) + timeout_error = nil + else + status = terminate(pid, signal, opt[:pgroup], reprieve) + terminated = Time.now + end + stdout = th_stdout.value if capture_stdout + stderr = th_stderr.value if capture_stderr && capture_stderr != :merge_to_stdout + out_p.close if capture_stdout + err_p.close if capture_stderr && capture_stderr != :merge_to_stdout + status ||= Process.wait2(pid)[1] + stdout = stdout_filter.call(stdout) if stdout_filter + stderr = stderr_filter.call(stderr) if stderr_filter + if timeout_error + bt = caller_locations + msg = "execution of #{bt.shift.label} expired timeout (#{timeout} sec)" + msg = failure_description(status, terminated, msg, [stdout, stderr].join("\n")) + raise timeout_error, msg, bt.map(&:to_s) + end + return stdout, stderr, status + end + ensure + [th_stdout, th_stderr].each do |th| + th.kill if th + end + [in_c, in_p, out_c, out_p, err_c, err_p].each do |io| + io&.close + end + [th_stdout, th_stderr].each do |th| + th.join if th + end + end + module_function :invoke_ruby + + def verbose_warning + class << (stderr = "".dup) + alias write concat + def flush; end + end + stderr, $stderr = $stderr, stderr + $VERBOSE = true + yield stderr + return $stderr + ensure + stderr, $stderr = $stderr, stderr + $VERBOSE = EnvUtil.original_verbose + EnvUtil.original_warning&.each {|i, v| Warning[i] = v} + end + module_function :verbose_warning + + def default_warning + $VERBOSE = false + yield + ensure + $VERBOSE = EnvUtil.original_verbose + end + module_function :default_warning + + def suppress_warning + $VERBOSE = nil + yield + ensure + $VERBOSE = EnvUtil.original_verbose + end + module_function :suppress_warning + + def under_gc_stress(stress = true) + stress, GC.stress = GC.stress, stress + yield + ensure + GC.stress = stress + end + module_function :under_gc_stress + + def with_default_external(enc) + suppress_warning { Encoding.default_external = enc } + yield + ensure + suppress_warning { Encoding.default_external = EnvUtil.original_external_encoding } + end + module_function :with_default_external + + def with_default_internal(enc) + suppress_warning { Encoding.default_internal = enc } + yield + ensure + suppress_warning { Encoding.default_internal = EnvUtil.original_internal_encoding } + end + module_function :with_default_internal + + def labeled_module(name, &block) + Module.new do + singleton_class.class_eval { + define_method(:to_s) {name} + alias inspect to_s + alias name to_s + } + class_eval(&block) if block + end + end + module_function :labeled_module + + def labeled_class(name, superclass = Object, &block) + Class.new(superclass) do + singleton_class.class_eval { + define_method(:to_s) {name} + alias inspect to_s + alias name to_s + } + class_eval(&block) if block + end + end + module_function :labeled_class + + if /darwin/ =~ RUBY_PLATFORM + DIAGNOSTIC_REPORTS_PATH = File.expand_path("~/Library/Logs/DiagnosticReports") + DIAGNOSTIC_REPORTS_TIMEFORMAT = '%Y-%m-%d-%H%M%S' + @ruby_install_name = RbConfig::CONFIG['RUBY_INSTALL_NAME'] + + def self.diagnostic_reports(signame, pid, now) + return unless %w[ABRT QUIT SEGV ILL TRAP].include?(signame) + cmd = File.basename(rubybin) + cmd = @ruby_install_name if "ruby-runner#{RbConfig::CONFIG["EXEEXT"]}" == cmd + path = DIAGNOSTIC_REPORTS_PATH + timeformat = DIAGNOSTIC_REPORTS_TIMEFORMAT + pat = "#{path}/#{cmd}_#{now.strftime(timeformat)}[-_]*.crash" + first = true + 30.times do + first ? (first = false) : sleep(0.1) + Dir.glob(pat) do |name| + log = File.read(name) rescue next + if /\AProcess:\s+#{cmd} \[#{pid}\]$/ =~ log + File.unlink(name) + File.unlink("#{path}/.#{File.basename(name)}.plist") rescue nil + return log + end + end + end + nil + end + else + def self.diagnostic_reports(signame, pid, now) + end + end + + def self.failure_description(status, now, message = "", out = "") + pid = status.pid + if signo = status.termsig + signame = Signal.signame(signo) + sigdesc = "signal #{signo}" + end + log = diagnostic_reports(signame, pid, now) + if signame + sigdesc = "SIG#{signame} (#{sigdesc})" + end + if status.coredump? + sigdesc = "#{sigdesc} (core dumped)" + end + full_message = ''.dup + message = message.call if Proc === message + if message and !message.empty? + full_message << message << "\n" + end + full_message << "pid #{pid}" + full_message << " exit #{status.exitstatus}" if status.exited? + full_message << " killed by #{sigdesc}" if sigdesc + if out and !out.empty? + full_message << "\n" << out.b.gsub(/^/, '| ') + full_message.sub!(/(?<!\n)\z/, "\n") + end + if log + full_message << "Diagnostic reports:\n" << log.b.gsub(/^/, '| ') + end + full_message + end + + def self.gc_stress_to_class? + unless defined?(@gc_stress_to_class) + _, _, status = invoke_ruby(["-e""exit GC.respond_to?(:add_stress_to_class)"]) + @gc_stress_to_class = status.success? + end + @gc_stress_to_class + end +end + +if defined?(RbConfig) + module RbConfig + @ruby = EnvUtil.rubybin + class << self + undef ruby if method_defined?(:ruby) + attr_reader :ruby + end + dir = File.dirname(ruby) + CONFIG['bindir'] = dir + end +end + +EnvUtil.capture_global_values diff --git a/tool/lib/find_executable.rb b/tool/lib/find_executable.rb new file mode 100644 index 0000000000..89c6fb8f3b --- /dev/null +++ b/tool/lib/find_executable.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true +require "rbconfig" + +module EnvUtil + def find_executable(cmd, *args) + exts = RbConfig::CONFIG["EXECUTABLE_EXTS"].split | [RbConfig::CONFIG["EXEEXT"]] + ENV["PATH"].split(File::PATH_SEPARATOR).each do |path| + next if path.empty? + path = File.join(path, cmd) + exts.each do |ext| + cmdline = [path + ext, *args] + begin + return cmdline if yield(IO.popen(cmdline, "r", err: [:child, :out], &:read)) + rescue + next + end + end + end + nil + end + module_function :find_executable +end diff --git a/tool/lib/gc_checker.rb b/tool/lib/gc_checker.rb new file mode 100644 index 0000000000..719da8cac0 --- /dev/null +++ b/tool/lib/gc_checker.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module GCDisabledChecker + def before_setup + if @__gc_disabled__ = GC.enable # return true if GC is disabled + GC.disable + end + + super + end + + def after_teardown + super + + disabled = GC.enable + GC.disable if @__gc_disabled__ + + if @__gc_disabled__ != disabled + label = { + true => 'disabled', + false => 'enabled', + } + raise "GC was #{label[@__gc_disabled__]}, but is #{label[disabled]} after the test." + end + end +end + +module GCCompactChecker + def after_teardown + super + GC.compact + end +end + +Test::Unit::TestCase.include GCDisabledChecker +Test::Unit::TestCase.include GCCompactChecker if ENV['RUBY_TEST_GC_COMPACT'] diff --git a/tool/lib/iseq_loader_checker.rb b/tool/lib/iseq_loader_checker.rb new file mode 100644 index 0000000000..3f07b3a999 --- /dev/null +++ b/tool/lib/iseq_loader_checker.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +begin + require '-test-/iseq_load/iseq_load' +rescue LoadError +end +require 'tempfile' + +class RubyVM::InstructionSequence + def disasm_if_possible + begin + self.disasm + rescue Encoding::CompatibilityError, EncodingError, SecurityError + nil + end + end + + def self.compare_dump_and_load i1, dumper, loader + dump = dumper.call(i1) + return i1 unless dump + i2 = loader.call(dump) + + # compare disassembled result + d1 = i1.disasm_if_possible + d2 = i2.disasm_if_possible + + if d1 != d2 + STDERR.puts "expected:" + STDERR.puts d1 + STDERR.puts "actual:" + STDERR.puts d2 + + t1 = Tempfile.new("expected"); t1.puts d1; t1.close + t2 = Tempfile.new("actual"); t2.puts d2; t2.close + system("diff -u #{t1.path} #{t2.path}") # use diff if available + exit(1) + end + i2 + end + + opt = ENV['RUBY_ISEQ_DUMP_DEBUG'] + + if opt && caller.any?{|e| /test\/runner\.rb/ =~ e} + puts "RUBY_ISEQ_DUMP_DEBUG = #{opt}" if opt + end + + CHECK_TO_A = 'to_a' == opt + CHECK_TO_BINARY = 'to_binary' == opt + + def self.translate i1 + # check to_a/load_iseq + compare_dump_and_load(i1, + proc{|iseq| + ary = iseq.to_a + ary[9] == :top ? ary : nil + }, + proc{|ary| + RubyVM::InstructionSequence.iseq_load(ary) + }) if CHECK_TO_A && defined?(RubyVM::InstructionSequence.iseq_load) + + # check to_binary + i2_bin = compare_dump_and_load(i1, + proc{|iseq| + begin + iseq.to_binary + rescue RuntimeError # not a toplevel + # STDERR.puts [:failed, $!, iseq].inspect + nil + end + }, + proc{|bin| + iseq = RubyVM::InstructionSequence.load_from_binary(bin) + # STDERR.puts iseq.inspect + iseq + }) if CHECK_TO_BINARY + # return value + i2_bin if CHECK_TO_BINARY + end if CHECK_TO_A || CHECK_TO_BINARY +end + +#require_relative 'x'; exit(1) diff --git a/tool/lib/jisx0208.rb b/tool/lib/jisx0208.rb new file mode 100644 index 0000000000..30185fb81b --- /dev/null +++ b/tool/lib/jisx0208.rb @@ -0,0 +1,86 @@ +# Library used by tools/enc-emoji-citrus-gen.rb + +module JISX0208 + class Char + class << self + def from_sjis(sjis) + unless 0x8140 <= sjis && sjis <= 0xFCFC + raise ArgumentError, "out of the range of JIS X 0208: 0x#{sjis.to_s(16)}" + end + sjis_hi, sjis_lo = sjis >> 8, sjis & 0xFF + sjis_hi = (sjis_hi - ((sjis_hi <= 0x9F) ? 0x80 : 0xC0)) << 1 + if sjis_lo <= 0x9E + sjis_hi -= 1 + sjis_lo -= (sjis_lo <= 0x7E) ? 0x3F : 0x40 + else + sjis_lo -= 0x9E + end + return self.new(sjis_hi, sjis_lo) + end + end + + def initialize(row, cell=nil) + if cell + @code = row_cell_to_code(row, cell) + else + @code = row.to_int + end + end + + def ==(other) + if self.class === other + return Integer(self) == Integer(other) + end + return super(other) + end + + def to_int + return @code + end + + def hi + Integer(self) >> 8 + end + + def lo + Integer(self) & 0xFF + end + + def row + self.hi - 0x20 + end + + def cell + self.lo - 0x20 + end + + def succ + succ_hi, succ_lo = self.hi, self.lo + 1 + if succ_lo > 0x7E + succ_lo = 0x21 + succ_hi += 1 + end + return self.class.new(succ_hi << 8 | succ_lo) + end + + def to_sjis + h, l = self.hi, self.lo + h = (h + 1) / 2 + ((0x21..0x5E).include?(h) ? 0x70 : 0xB0) + l += self.hi.odd? ? 0x1F + ((l >= 0x60) ? 1 : 0) : 0x7E + return h << 8 | l + end + + def inspect + "#<JISX0208::Char:#{self.object_id.to_s(16)} sjis=#{self.to_sjis.to_s(16)} jis=#{self.to_int.to_s(16)}>" + end + + private + + def row_cell_to_code(row, cell) + unless 0 < row && (1..94).include?(cell) + raise ArgumentError, "out of row-cell range: #{row}-#{cell}" + end + return (row + 0x20) << 8 | (cell + 0x20) + end + end +end diff --git a/tool/lib/leakchecker.rb b/tool/lib/leakchecker.rb new file mode 100644 index 0000000000..ed50796940 --- /dev/null +++ b/tool/lib/leakchecker.rb @@ -0,0 +1,314 @@ +# frozen_string_literal: true +class LeakChecker + @@try_lsof = nil # not-tried-yet + + def initialize + @fd_info = find_fds + @@skip = false + @tempfile_info = find_tempfiles + @thread_info = find_threads + @env_info = find_env + @encoding_info = find_encodings + @old_verbose = $VERBOSE + @old_warning_flags = find_warning_flags + end + + def check(test_name) + if /i386-solaris/ =~ RUBY_PLATFORM && /TestGem/ =~ test_name + GC.verify_internal_consistency + end + + leaks = [ + check_fd_leak(test_name), + check_thread_leak(test_name), + check_tempfile_leak(test_name), + check_env(test_name), + check_encodings(test_name), + check_verbose(test_name), + check_warning_flags(test_name), + ] + GC.start if leaks.any? + end + + def check_verbose test_name + puts "#{test_name}: $VERBOSE == #{$VERBOSE}" unless @old_verbose == $VERBOSE + end + + def find_fds + if IO.respond_to?(:console) and (m = IO.method(:console)).arity.nonzero? + m[:close] + end + %w"/proc/self/fd /dev/fd".each do |fd_dir| + if File.directory?(fd_dir) + fds = Dir.open(fd_dir) {|d| + a = d.grep(/\A\d+\z/, &:to_i) + if d.respond_to? :fileno + a -= [d.fileno] + end + a + } + return fds.sort + end + end + [] + end + + def check_fd_leak(test_name) + leaked = false + live1 = @fd_info + live2 = find_fds + fd_closed = live1 - live2 + if !fd_closed.empty? + fd_closed.each {|fd| + puts "Closed file descriptor: #{test_name}: #{fd}" + } + end + fd_leaked = live2 - live1 + if !@@skip && !fd_leaked.empty? + leaked = true + h = {} + ObjectSpace.each_object(IO) {|io| + inspect = io.inspect + begin + autoclose = io.autoclose? + fd = io.fileno + rescue IOError # closed IO object + next + end + (h[fd] ||= []) << [io, autoclose, inspect] + } + fd_leaked.select! {|fd| + str = ''.dup + pos = nil + if h[fd] + str << ' :' + h[fd].map {|io, autoclose, inspect| + if ENV["LEAK_CHECKER_TRACE_OBJECT_ALLOCATION"] + pos = "#{ObjectSpace.allocation_sourcefile(io)}:#{ObjectSpace.allocation_sourceline(io)}" + end + s = ' ' + inspect + s << "(not-autoclose)" if !autoclose + s + }.sort.each {|s| + str << s + } + else + begin + io = IO.for_fd(fd, autoclose: false) + s = io.stat + rescue Errno::EBADF + # something un-stat-able + next + else + next if /darwin/ =~ RUBY_PLATFORM and [0, -1].include?(s.dev) + str << ' ' << s.inspect + ensure + io&.close + end + end + puts "Leaked file descriptor: #{test_name}: #{fd}#{str}" + puts " The IO was created at #{pos}" if pos + true + } + unless fd_leaked.empty? + unless @@try_lsof == false + @@try_lsof |= system(*%W[lsof -a -d #{fd_leaked.minmax.uniq.join("-")} -p #$$], out: Test::Unit::Runner.output) + end + end + h.each {|fd, list| + next if list.length <= 1 + if 1 < list.count {|io, autoclose, inspect| autoclose } + str = list.map {|io, autoclose, inspect| " #{inspect}" + (autoclose ? "(autoclose)" : "") }.sort.join + puts "Multiple autoclose IO objects for a file descriptor in: #{test_name}: #{str}" + end + } + end + @fd_info = live2 + @@skip = false + return leaked + end + + def extend_tempfile_counter + return if defined? LeakChecker::TempfileCounter + m = Module.new { + @count = 0 + class << self + attr_accessor :count + end + + def new(data) + LeakChecker::TempfileCounter.count += 1 + super(data) + end + } + LeakChecker.const_set(:TempfileCounter, m) + + class << Tempfile::Remover + prepend LeakChecker::TempfileCounter + end + end + + def find_tempfiles(prev_count=-1) + return [prev_count, []] unless defined? Tempfile + extend_tempfile_counter + count = TempfileCounter.count + if prev_count == count + [prev_count, []] + else + tempfiles = ObjectSpace.each_object(Tempfile).find_all {|t| + t.instance_variable_defined?(:@tmpfile) and t.path + } + [count, tempfiles] + end + end + + def check_tempfile_leak(test_name) + return false unless defined? Tempfile + count1, initial_tempfiles = @tempfile_info + count2, current_tempfiles = find_tempfiles(count1) + leaked = false + tempfiles_leaked = current_tempfiles - initial_tempfiles + if !tempfiles_leaked.empty? + leaked = true + list = tempfiles_leaked.map {|t| t.inspect }.sort + list.each {|str| + puts "Leaked tempfile: #{test_name}: #{str}" + } + tempfiles_leaked.each {|t| t.close! } + end + @tempfile_info = [count2, initial_tempfiles] + return leaked + end + + def find_threads + Thread.list.find_all {|t| + t != Thread.current && t.alive? + } + end + + def check_thread_leak(test_name) + live1 = @thread_info + live2 = find_threads + thread_finished = live1 - live2 + leaked = false + if !thread_finished.empty? + list = thread_finished.map {|t| t.inspect }.sort + list.each {|str| + puts "Finished thread: #{test_name}: #{str}" + } + end + thread_leaked = live2 - live1 + if !thread_leaked.empty? + leaked = true + list = thread_leaked.map {|t| t.inspect }.sort + list.each {|str| + puts "Leaked thread: #{test_name}: #{str}" + } + end + @thread_info = live2 + return leaked + end + + e = ENV["_Ruby_Env_Ignorecase_"], ENV["_RUBY_ENV_IGNORECASE_"] + begin + ENV["_Ruby_Env_Ignorecase_"] = ENV["_RUBY_ENV_IGNORECASE_"] = nil + ENV["_RUBY_ENV_IGNORECASE_"] = "ENV_CASE_TEST" + ENV_IGNORECASE = ENV["_Ruby_Env_Ignorecase_"] == "ENV_CASE_TEST" + ensure + ENV["_Ruby_Env_Ignorecase_"], ENV["_RUBY_ENV_IGNORECASE_"] = e + end + + if ENV_IGNORECASE + def find_env + ENV.to_h {|k, v| [k.upcase, v]} + end + else + def find_env + ENV.to_h + end + end + + def check_env(test_name) + old_env = @env_info + new_env = find_env + return false if old_env == new_env + (old_env.keys | new_env.keys).sort.each {|k| + if old_env.has_key?(k) + if new_env.has_key?(k) + if old_env[k] != new_env[k] + puts "Environment variable changed: #{test_name} : #{k.inspect} changed : #{old_env[k].inspect} -> #{new_env[k].inspect}" + end + else + puts "Environment variable changed: #{test_name} : #{k.inspect} deleted" + end + else + if new_env.has_key?(k) + puts "Environment variable changed: #{test_name} : #{k.inspect} added" + else + flunk "unreachable" + end + end + } + @env_info = new_env + return true + end + + def find_encodings + { + 'Encoding.default_internal' => Encoding.default_internal, + 'Encoding.default_external' => Encoding.default_external, + 'STDIN.internal_encoding' => STDIN.internal_encoding, + 'STDIN.external_encoding' => STDIN.external_encoding, + 'STDOUT.internal_encoding' => STDOUT.internal_encoding, + 'STDOUT.external_encoding' => STDOUT.external_encoding, + 'STDERR.internal_encoding' => STDERR.internal_encoding, + 'STDERR.external_encoding' => STDERR.external_encoding, + } + end + + def check_encodings(test_name) + old_encoding_info = @encoding_info + @encoding_info = find_encodings + leaked = false + @encoding_info.each do |key, new_encoding| + old_encoding = old_encoding_info[key] + if new_encoding != old_encoding + leaked = true + puts "#{key} changed: #{test_name} : #{old_encoding.inspect} to #{new_encoding.inspect}" + end + end + leaked + end + + WARNING_CATEGORIES = (Warning.respond_to?(:[]) ? %i[deprecated experimental] : []).freeze + + def find_warning_flags + WARNING_CATEGORIES.to_h do |category| + [category, Warning[category]] + end + end + + def check_warning_flags(test_name) + new_warning_flags = find_warning_flags + leaked = false + WARNING_CATEGORIES.each do |category| + if new_warning_flags[category] != @old_warning_flags[category] + leaked = true + puts "Warning[#{category.inspect}] changed: #{test_name} : #{@old_warning_flags[category]} to #{new_warning_flags[category]}" + end + end + return leaked + end + + def puts(*a) + output = Test::Unit::Runner.output + if defined?(output.set_encoding) + output.set_encoding(nil, nil) + end + output.puts(*a) + end + + def self.skip + @@skip = true + end +end diff --git a/tool/lib/memory_status.rb b/tool/lib/memory_status.rb new file mode 100644 index 0000000000..5e9e80a68a --- /dev/null +++ b/tool/lib/memory_status.rb @@ -0,0 +1,151 @@ +# frozen_string_literal: true +begin + require '-test-/memory_status.so' +rescue LoadError +end + +module Memory + keys = [] + + case + when File.exist?(procfile = "/proc/self/status") && (pat = /^Vm(\w+):\s+(\d+)/) =~ (data = File.binread(procfile)) + PROC_FILE = procfile + VM_PAT = pat + def self.read_status + IO.foreach(PROC_FILE, encoding: Encoding::ASCII_8BIT) do |l| + yield($1.downcase.intern, $2.to_i * 1024) if VM_PAT =~ l + end + end + + data.scan(pat) {|k, v| keys << k.downcase.intern} + + when /mswin|mingw/ =~ RUBY_PLATFORM + require 'fiddle/import' + require 'fiddle/types' + + module Win32 + extend Fiddle::Importer + dlload "kernel32.dll", "psapi.dll" + include Fiddle::Win32Types + typealias "SIZE_T", "size_t" + + PROCESS_MEMORY_COUNTERS = struct [ + "DWORD cb", + "DWORD PageFaultCount", + "SIZE_T PeakWorkingSetSize", + "SIZE_T WorkingSetSize", + "SIZE_T QuotaPeakPagedPoolUsage", + "SIZE_T QuotaPagedPoolUsage", + "SIZE_T QuotaPeakNonPagedPoolUsage", + "SIZE_T QuotaNonPagedPoolUsage", + "SIZE_T PagefileUsage", + "SIZE_T PeakPagefileUsage", + ] + + typealias "PPROCESS_MEMORY_COUNTERS", "PROCESS_MEMORY_COUNTERS*" + + extern "HANDLE GetCurrentProcess()", :stdcall + extern "BOOL GetProcessMemoryInfo(HANDLE, PPROCESS_MEMORY_COUNTERS, DWORD)", :stdcall + + module_function + def memory_info + size = PROCESS_MEMORY_COUNTERS.size + data = PROCESS_MEMORY_COUNTERS.malloc + data.cb = size + data if GetProcessMemoryInfo(GetCurrentProcess(), data, size) + end + end + + keys.push(:size, :rss, :peak) + def self.read_status + if info = Win32.memory_info + yield :size, info.PagefileUsage + yield :rss, info.WorkingSetSize + yield :peak, info.PeakWorkingSetSize + end + end + when (require_relative 'find_executable' + pat = /^\s*(\d+)\s+(\d+)$/ + pscmd = EnvUtil.find_executable("ps", "-ovsz=", "-orss=", "-p", $$.to_s) {|out| pat =~ out}) + pscmd.pop + PAT = pat + PSCMD = pscmd + + keys << :size << :rss + def self.read_status + if PAT =~ IO.popen(PSCMD + [$$.to_s], "r", err: [:child, :out], &:read) + yield :size, $1.to_i*1024 + yield :rss, $2.to_i*1024 + end + end + else + def self.read_status + raise NotImplementedError, "unsupported platform" + end + end + + if !keys.empty? + Status = Struct.new(*keys) + end +end unless defined?(Memory::Status) + +if defined?(Memory::Status) + class Memory::Status + def _update + Memory.read_status do |key, val| + self[key] = val + end + self + end unless method_defined?(:_update) + + Header = members.map {|k| k.to_s.upcase.rjust(6)}.join('') + Format = "%6d" + + def initialize + _update + end + + def to_s + status = each_pair.map {|n,v| + "#{n}:#{v}" + } + "{#{status.join(",")}}" + end + + def self.parse(str) + status = allocate + str.scan(/(?:\A\{|\G,)(#{members.join('|')}):(\d+)(?=,|\}\z)/) do + status[$1] = $2.to_i + end + status + end + end + + # On some platforms (e.g. Solaris), libc malloc does not return + # freed memory to OS because of efficiency, and linking with extra + # malloc library is needed to detect memory leaks. + # + case RUBY_PLATFORM + when /solaris2\.(?:9|[1-9][0-9])/i # Solaris 9, 10, 11,... + bits = [nil].pack('p').size == 8 ? 64 : 32 + if ENV['LD_PRELOAD'].to_s.empty? && + ENV["LD_PRELOAD_#{bits}"].to_s.empty? && + (ENV['UMEM_OPTIONS'].to_s.empty? || + ENV['UMEM_OPTIONS'] == 'backend=mmap') then + envs = { + 'LD_PRELOAD' => 'libumem.so', + 'UMEM_OPTIONS' => 'backend=mmap' + } + args = [ + envs, + "--disable=gems", + "-v", "-", + ] + _, err, status = EnvUtil.invoke_ruby(args, "exit(0)", true, true) + if status.exitstatus == 0 && err.to_s.empty? then + Memory::NO_MEMORY_LEAK_ENVS = envs + end + end + end #case RUBY_PLATFORM + +end diff --git a/tool/lib/profile_test_all.rb b/tool/lib/profile_test_all.rb new file mode 100644 index 0000000000..fb434e314d --- /dev/null +++ b/tool/lib/profile_test_all.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true +# +# purpose: +# Profile memory usage of each tests. +# +# usage: +# RUBY_TEST_ALL_PROFILE=[file] make test-all +# +# output: +# [file] specified by RUBY_TEST_ALL_PROFILE +# If [file] is 'true', then it is ./test_all_profile +# +# collected information: +# - ObjectSpace.memsize_of_all +# - GC.stat +# - /proc/meminfo (some fields, if exists) +# - /proc/self/status (some fields, if exists) +# - /proc/self/statm (if exists) +# + +require 'objspace' + +class Test::Unit::TestCase + alias orig_run run + + file = ENV['RUBY_TEST_ALL_PROFILE'] + file = 'test-all-profile-result' if file == 'true' + TEST_ALL_PROFILE_OUT = open(file, 'w') + TEST_ALL_PROFILE_GC_STAT_HASH = {} + TEST_ALL_PROFILE_BANNER = ['name'] + TEST_ALL_PROFILE_PROCS = [] + + def self.add *name, &b + TEST_ALL_PROFILE_BANNER.concat name + TEST_ALL_PROFILE_PROCS << b + end + + add 'failed?' do |result, tc| + result << (tc.passed? ? 0 : 1) + end + + add 'memsize_of_all' do |result, *| + result << ObjectSpace.memsize_of_all + end + + add(*GC.stat.keys) do |result, *| + GC.stat(TEST_ALL_PROFILE_GC_STAT_HASH) + result.concat TEST_ALL_PROFILE_GC_STAT_HASH.values + end + + def self.add_proc_meminfo file, fields + return unless FileTest.exist?(file) + regexp = /(#{fields.join("|")}):\s*(\d+) kB/ + # check = {}; fields.each{|e| check[e] = true} + add(*fields) do |result, *| + text = File.read(file) + text.scan(regexp){ + # check.delete $1 + result << $2 + '' + } + # raise check.inspect unless check.empty? + end + end + + add_proc_meminfo '/proc/meminfo', %w(MemTotal MemFree) + add_proc_meminfo '/proc/self/status', %w(VmPeak VmSize VmHWM VmRSS) + + if FileTest.exist?('/proc/self/statm') + add 'size', 'resident', 'share', 'text', 'lib', 'data', 'dt' do |result, *| + result.concat File.read('/proc/self/statm').split(/\s+/) + end + end + + def memprofile_test_all_result_result + result = ["#{self.class}\##{self.__name__.to_s.gsub(/\s+/, '')}"] + TEST_ALL_PROFILE_PROCS.each{|proc| + proc.call(result, self) + } + result.join("\t") + end + + def run runner + result = orig_run(runner) + TEST_ALL_PROFILE_OUT.puts memprofile_test_all_result_result + TEST_ALL_PROFILE_OUT.flush + result + end + + TEST_ALL_PROFILE_OUT.puts TEST_ALL_PROFILE_BANNER.join("\t") +end diff --git a/tool/lib/test/unit.rb b/tool/lib/test/unit.rb new file mode 100644 index 0000000000..3bb2692b43 --- /dev/null +++ b/tool/lib/test/unit.rb @@ -0,0 +1,1762 @@ +# frozen_string_literal: true + +require_relative '../envutil' +require_relative '../colorize' +require_relative '../leakchecker' +require_relative '../test/unit/testcase' +require 'optparse' + +# See Test::Unit +module Test + + class << self + ## + # Filter object for backtraces. + + attr_accessor :backtrace_filter + end + + class BacktraceFilter # :nodoc: + def filter bt + return ["No backtrace"] unless bt + + new_bt = [] + pattern = %r[/(?:lib\/test/|core_assertions\.rb:)] + + unless $DEBUG then + bt.each do |line| + break if pattern.match?(line) + new_bt << line + end + + new_bt = bt.reject { |line| pattern.match?(line) } if new_bt.empty? + new_bt = bt.dup if new_bt.empty? + else + new_bt = bt.dup + end + + new_bt + end + end + + self.backtrace_filter = BacktraceFilter.new + + def self.filter_backtrace bt # :nodoc: + backtrace_filter.filter bt + end + + ## + # Test::Unit is an implementation of the xUnit testing framework for Ruby. + module Unit + ## + # Assertion base class + + class AssertionFailedError < Exception; end + + ## + # Assertion raised when skipping a test + + class PendedError < AssertionFailedError; end + + module Order + class NoSort + def initialize(seed) + end + + def sort_by_name(list) + list + end + + alias sort_by_string sort_by_name + + def group(list) + list + end + end + + module JITFirst + def group(list) + # JIT first + jit, others = list.partition {|e| /test_jit/ =~ e} + jit + others + end + end + + class Alpha < NoSort + include JITFirst + + def sort_by_name(list) + list.sort_by(&:name) + end + + def sort_by_string(list) + list.sort + end + + end + + # shuffle test suites based on CRC32 of their names + Shuffle = Struct.new(:seed, :salt) do + include JITFirst + + def initialize(seed) + self.class::CRC_TBL ||= (0..255).map {|i| + (0..7).inject(i) {|c,| (c & 1 == 1) ? (0xEDB88320 ^ (c >> 1)) : (c >> 1) } + }.freeze + + salt = [seed].pack("V").unpack1("H*") + super(seed, "\n#{salt}".freeze).freeze + end + + def sort_by_name(list) + list.sort_by {|e| randomize_key(e.name)} + end + + def sort_by_string(list) + list.sort_by {|e| randomize_key(e)} + end + + private + + def crc32(str, crc32 = 0xffffffff) + crc_tbl = self.class::CRC_TBL + str.each_byte do |data| + crc32 = crc_tbl[(crc32 ^ data) & 0xff] ^ (crc32 >> 8) + end + crc32 + end + + def randomize_key(name) + crc32(salt, crc32(name)) ^ 0xffffffff + end + end + + Types = { + random: Shuffle, + alpha: Alpha, + sorted: Alpha, + nosort: NoSort, + } + Types.default_proc = proc {|_, order| + raise "Unknown test_order: #{order.inspect}" + } + end + + module RunCount # :nodoc: all + @@run_count = 0 + + def self.have_run? + @@run_count.nonzero? + end + + def run(*) + @@run_count += 1 + super + end + + def run_once + return if have_run? + return if $! # don't run if there was an exception + yield + end + module_function :run_once + end + + module Options # :nodoc: all + def initialize(*, &block) + @init_hook = block + @options = nil + super(&nil) + end + + def option_parser + @option_parser ||= OptionParser.new + end + + def process_args(args = []) + return @options if @options + orig_args = args.dup + options = {} + opts = option_parser + setup_options(opts, options) + opts.parse!(args) + orig_args -= args + args = @init_hook.call(args, options) if @init_hook + non_options(args, options) + @run_options = orig_args + + order = options[:test_order] + if seed = options[:seed] + order ||= :random + elsif (order ||= :random) == :random + seed = options[:seed] = rand(0x10000) + orig_args.unshift "--seed=#{seed}" + end + Test::Unit::TestCase.test_order = order if order + order = Test::Unit::TestCase.test_order + @order = Test::Unit::Order::Types[order].new(seed) + + @help = "\n" + orig_args.map { |s| + " " + (s =~ /[\s|&<>$()]/ ? s.inspect : s) + }.join("\n") + @options = options + end + + private + def setup_options(opts, options) + opts.separator 'test-unit options:' + + opts.on '-h', '--help', 'Display this help.' do + puts opts + exit + end + + opts.on '-s', '--seed SEED', Integer, "Sets random seed" do |m| + options[:seed] = m.to_i + end + + opts.on '-v', '--verbose', "Verbose. Show progress processing files." do + options[:verbose] = true + self.verbose = options[:verbose] + end + + opts.on '-n', '--name PATTERN', "Filter test method names on pattern: /REGEXP/, !/REGEXP/ or STRING" do |a| + (options[:filter] ||= []) << a + end + + orders = Test::Unit::Order::Types.keys + opts.on "--test-order=#{orders.join('|')}", orders do |a| + options[:test_order] = a + end + end + + def non_options(files, options) + filter = options[:filter] + if filter + pos_pat = /\A\/(.*)\/\z/ + neg_pat = /\A!\/(.*)\/\z/ + negative, positive = filter.partition {|s| neg_pat =~ s} + if positive.empty? + filter = nil + elsif negative.empty? and positive.size == 1 and pos_pat !~ positive[0] + filter = positive[0] + unless /\A[A-Z]\w*(?:::[A-Z]\w*)*#/ =~ filter + filter = /##{Regexp.quote(filter)}\z/ + end + else + filter = Regexp.union(*positive.map! {|s| Regexp.new(s[pos_pat, 1] || "\\A#{Regexp.quote(s)}\\z")}) + end + unless negative.empty? + negative = Regexp.union(*negative.map! {|s| Regexp.new(s[neg_pat, 1])}) + filter = /\A(?=.*#{filter})(?!.*#{negative})/ + end + options[:filter] = filter + end + true + end + end + + module Parallel # :nodoc: all + def process_args(args = []) + return @options if @options + options = super + if @options[:parallel] + @files = args + end + options + end + + def non_options(files, options) + @jobserver = nil + makeflags = ENV.delete("MAKEFLAGS") + if !options[:parallel] and + /(?:\A|\s)--jobserver-(?:auth|fds)=(\d+),(\d+)/ =~ makeflags + begin + r = IO.for_fd($1.to_i(10), "rb", autoclose: false) + w = IO.for_fd($2.to_i(10), "wb", autoclose: false) + rescue + r.close if r + nil + else + r.close_on_exec = true + w.close_on_exec = true + @jobserver = [r, w] + options[:parallel] ||= 1 + end + end + @worker_timeout = EnvUtil.apply_timeout_scale(options[:worker_timeout] || 180) + super + end + + def status(*args) + result = super + raise @interrupt if @interrupt + result + end + + private + def setup_options(opts, options) + super + + opts.separator "parallel test options:" + + options[:retry] = true + + opts.on '-j N', '--jobs N', /\A(t)?(\d+)\z/, "Allow run tests with N jobs at once" do |_, t, a| + options[:testing] = true & t # For testing + options[:parallel] = a.to_i + end + + opts.on '--worker-timeout=N', Integer, "Timeout workers not responding in N seconds" do |a| + options[:worker_timeout] = a + end + + opts.on '--separate', "Restart job process after one testcase has done" do + options[:parallel] ||= 1 + options[:separate] = true + end + + opts.on '--retry', "Retry running testcase when --jobs specified" do + options[:retry] = true + end + + opts.on '--no-retry', "Disable --retry" do + options[:retry] = false + end + + opts.on '--ruby VAL', "Path to ruby which is used at -j option" do |a| + options[:ruby] = a.split(/ /).reject(&:empty?) + end + + opts.on '--timetable-data=FILE', "Path to timetable data" do |a| + options[:timetable_data] = a + end + end + + class Worker + def self.launch(ruby,args=[]) + scale = EnvUtil.timeout_scale + io = IO.popen([*ruby, "-W1", + "#{__dir__}/unit/parallel.rb", + *("--timeout-scale=#{scale}" if scale), + *args], "rb+") + new(io, io.pid, :waiting) + end + + attr_reader :quit_called + attr_accessor :start_time + attr_accessor :response_at + attr_accessor :current + + @@worker_number = 0 + + def initialize(io, pid, status) + @num = (@@worker_number += 1) + @io = io + @pid = pid + @status = status + @file = nil + @real_file = nil + @loadpath = [] + @hooks = {} + @quit_called = false + @response_at = nil + end + + def name + "Worker #{@num}" + end + + def puts(*args) + @io.puts(*args) + end + + def run(task,type) + @file = File.basename(task, ".rb") + @real_file = task + begin + puts "loadpath #{[Marshal.dump($:-@loadpath)].pack("m0")}" + @loadpath = $:.dup + puts "run #{task} #{type}" + @status = :prepare + @start_time = Time.now + @response_at = @start_time + rescue Errno::EPIPE + died + rescue IOError + raise unless /stream closed|closed stream/ =~ $!.message + died + end + end + + def hook(id,&block) + @hooks[id] ||= [] + @hooks[id] << block + self + end + + def read + res = (@status == :quit) ? @io.read : @io.gets + @response_at = Time.now + res && res.chomp + end + + def close + @io.close unless @io.closed? + self + rescue IOError + end + + def quit + return if @io.closed? + @quit_called = true + @io.puts "quit" + rescue Errno::EPIPE => e + warn "#{@pid}:#{@status.to_s.ljust(7)}:#{@file}: #{e.message}" + end + + def kill + Process.kill(:KILL, @pid) + rescue Errno::ESRCH + end + + def died(*additional) + @status = :quit + @io.close + status = $? + if status and status.signaled? + additional[0] ||= SignalException.new(status.termsig) + end + + call_hook(:dead,*additional) + end + + def to_s + if @file and @status != :ready + "#{@pid}=#{@file}" + else + "#{@pid}:#{@status.to_s.ljust(7)}" + end + end + + attr_reader :io, :pid + attr_accessor :status, :file, :real_file, :loadpath + + private + + def call_hook(id,*additional) + @hooks[id] ||= [] + @hooks[id].each{|hook| hook[self,additional] } + self + end + + end + + def flush_job_tokens + if @jobserver + r, w = @jobserver.shift(2) + @jobserver = nil + w << @job_tokens.slice!(0..-1) + r.close + w.close + end + end + + def after_worker_down(worker, e=nil, c=false) + return unless @options[:parallel] + return if @interrupt + flush_job_tokens + warn e if e + real_file = worker.real_file and warn "running file: #{real_file}" + @need_quit = true + warn "" + warn "Some worker was crashed. It seems ruby interpreter's bug" + warn "or, a bug of test/unit/parallel.rb. try again without -j" + warn "option." + warn "" + if File.exist?('core') + require 'fileutils' + require 'time' + Dir.glob('/tmp/test-unit-core.*').each do |f| + if Time.now - File.mtime(f) > 7 * 24 * 60 * 60 # 7 days + warn "Deleting an old core file: #{f}" + FileUtils.rm(f) + end + end + core_path = "/tmp/test-unit-core.#{Time.now.utc.iso8601}" + warn "A core file is found. Saving it at: #{core_path.dump}" + FileUtils.mv('core', core_path) + cmd = ['gdb', RbConfig.ruby, '-c', core_path, '-ex', 'bt', '-batch'] + p cmd # debugging why it's not working + system(*cmd) + end + STDERR.flush + exit c + end + + def after_worker_quit(worker) + return unless @options[:parallel] + return if @interrupt + worker.close + if @jobserver and (token = @job_tokens.slice!(0)) + @jobserver[1] << token + end + @workers.delete(worker) + @dead_workers << worker + @ios = @workers.map(&:io) + end + + def launch_worker + begin + worker = Worker.launch(@options[:ruby], @run_options) + rescue => e + abort "ERROR: Failed to launch job process - #{e.class}: #{e.message}" + end + worker.hook(:dead) do |w,info| + after_worker_quit w + after_worker_down w, *info if !info.empty? && !worker.quit_called + end + @workers << worker + @ios << worker.io + @workers_hash[worker.io] = worker + worker + end + + def delete_worker(worker) + @workers_hash.delete worker.io + @workers.delete worker + @ios.delete worker.io + end + + def quit_workers(&cond) + return if @workers.empty? + closed = [] if cond + @workers.reject! do |worker| + next unless cond&.call(worker) + begin + Timeout.timeout(1) do + worker.quit + end + rescue Errno::EPIPE + rescue Timeout::Error + end + closed&.push worker + begin + Timeout.timeout(0.2) do + worker.close + end + rescue Timeout::Error + worker.kill + retry + end + @ios.delete worker.io + end + + return if (closed ||= @workers).empty? + pids = closed.map(&:pid) + begin + Timeout.timeout(0.2 * closed.size) do + Process.waitall + end + rescue Timeout::Error + if pids + Process.kill(:KILL, *pids) rescue nil + pids = nil + retry + end + end + @workers.clear unless cond + closed + end + + FakeClass = Struct.new(:name) + def fake_class(name) + (@fake_classes ||= {})[name] ||= FakeClass.new(name) + end + + def deal(io, type, result, rep, shutting_down = false) + worker = @workers_hash[io] + cmd = worker.read + cmd.sub!(/\A\.+/, '') if cmd # read may return nil + + case cmd + when '' + # just only dots, ignore + when /^okay$/ + worker.status = :running + when /^ready(!)?$/ + bang = $1 + worker.status = :ready + + unless task = @tasks.shift + worker.quit + return nil + end + if @options[:separate] and not bang + worker.quit + worker = launch_worker + end + worker.run(task, type) + @test_count += 1 + + jobs_status(worker) + when /^start (.+?)$/ + worker.current = Marshal.load($1.unpack1("m")) + when /^done (.+?)$/ + begin + r = Marshal.load($1.unpack1("m")) + rescue + print "unknown object: #{$1.unpack1("m").dump}" + return true + end + result << r[0..1] unless r[0..1] == [nil,nil] + rep << {file: worker.real_file, report: r[2], result: r[3], testcase: r[5]} + $:.push(*r[4]).uniq! + jobs_status(worker) if @options[:job_status] == :replace + + return true + when /^record (.+?)$/ + begin + r = Marshal.load($1.unpack1("m")) + + suite = r.first + key = [worker.name, suite] + if @records[key] + @records[key][1] = worker.start_time = Time.now + else + @records[key] = [worker.start_time, Time.now] + end + rescue => e + print "unknown record: #{e.message} #{$1.unpack1("m").dump}" + return true + end + record(fake_class(r[0]), *r[1..-1]) + when /^p (.+?)$/ + del_jobs_status + print $1.unpack1("m") + jobs_status(worker) if @options[:job_status] == :replace + when /^after (.+?)$/ + @warnings << Marshal.load($1.unpack1("m")) + when /^bye (.+?)$/ + after_worker_down worker, Marshal.load($1.unpack1("m")) + when /^bye$/, nil + if shutting_down || worker.quit_called + after_worker_quit worker + else + after_worker_down worker + end + else + print "unknown command: #{cmd.dump}\n" + end + return false + end + + def _run_parallel suites, type, result + @records = {} + + if @options[:parallel] < 1 + warn "Error: parameter of -j option should be greater than 0." + return + end + + # Require needed thing for parallel running + require 'timeout' + @tasks = @order.group(@order.sort_by_string(@files)) # Array of filenames. + + @need_quit = false + @dead_workers = [] # Array of dead workers. + @warnings = [] + @total_tests = @tasks.size.to_s(10) + rep = [] # FIXME: more good naming + + @workers = [] # Array of workers. + @workers_hash = {} # out-IO => worker + @ios = [] # Array of worker IOs + @job_tokens = String.new(encoding: Encoding::ASCII_8BIT) if @jobserver + begin + [@tasks.size, @options[:parallel]].min.times {launch_worker} + + while true + timeout = [(@workers.filter_map {|w| w.response_at}.min&.-(Time.now) || 0) + @worker_timeout, 1].max + + if !(_io = IO.select(@ios, nil, nil, timeout)) + timeout = Time.now - @worker_timeout + quit_workers {|w| w.response_at < timeout}&.map {|w| + rep << {file: w.real_file, result: nil, testcase: w.current[0], error: w.current} + } + elsif _io.first.any? {|io| + @need_quit or + (deal(io, type, result, rep).nil? and + !@workers.any? {|x| [:running, :prepare].include? x.status}) + } + break + end + break if @tasks.empty? and @workers.empty? + if @jobserver and @job_tokens and !@tasks.empty? and + ((newjobs = [@tasks.size, @options[:parallel]].min) > @workers.size or + !@workers.any? {|x| x.status == :ready}) + t = @jobserver[0].read_nonblock(newjobs, exception: false) + if String === t + @job_tokens << t + t.size.times {launch_worker} + end + end + end + rescue Interrupt => ex + @interrupt = ex + return result + ensure + if file = @options[:timetable_data] + open(file, 'w'){|f| + @records.each{|(worker, suite), (st, ed)| + f.puts '[' + [worker.dump, suite.dump, st.to_f * 1_000, ed.to_f * 1_000].join(", ") + '],' + } + } + end + + if @interrupt + @ios.select!{|x| @workers_hash[x].status == :running } + while !@ios.empty? && (__io = IO.select(@ios,[],[],10)) + __io[0].reject! {|io| deal(io, type, result, rep, true)} + end + end + + quit_workers + flush_job_tokens + + unless @interrupt || !@options[:retry] || @need_quit + parallel = @options[:parallel] + @options[:parallel] = false + suites, rep = rep.partition {|r| + r[:testcase] && r[:file] && + (!r.key?(:report) || r[:report].any? {|e| !e[2].is_a?(Test::Unit::PendedError)}) + } + suites.map {|r| File.realpath(r[:file])}.uniq.each {|file| require file} + del_status_line or puts + error, suites = suites.partition {|r| r[:error]} + unless suites.empty? + puts "\n""Retrying..." + @verbose = options[:verbose] + suites.map! {|r| ::Object.const_get(r[:testcase])} + _run_suites(suites, type) + end + unless error.empty? + puts "\n""Retrying hung up testcases..." + error.map! {|r| ::Object.const_get(r[:testcase])} + verbose = @verbose + job_status = options[:job_status] + options[:verbose] = @verbose = true + options[:job_status] = :normal + result.concat _run_suites(error, type) + options[:verbose] = @verbose = verbose + options[:job_status] = job_status + end + @options[:parallel] = parallel + end + unless @options[:retry] + del_status_line or puts + end + unless rep.empty? + rep.each do |r| + if r[:error] + puke(*r[:error], Timeout::Error) + next + end + r[:report]&.each do |f| + puke(*f) if f + end + end + if @options[:retry] + rep.each do |x| + (e, f, s = x[:result]) or next + @errors += e + @failures += f + @skips += s + end + end + end + unless @warnings.empty? + warn "" + @warnings.uniq! {|w| w[1].message} + @warnings.each do |w| + warn "#{w[0]}: #{w[1].message} (#{w[1].class})" + end + warn "" + end + end + end + + def _run_suites suites, type + _prepare_run(suites, type) + @interrupt = nil + result = [] + GC.start + if @options[:parallel] + _run_parallel suites, type, result + else + suites.each {|suite| + begin + result << _run_suite(suite, type) + rescue Interrupt => e + @interrupt = e + break + end + } + end + del_status_line + result + end + end + + module Skipping # :nodoc: all + def failed(s) + super if !s or @options[:hide_skip] + end + + private + def setup_options(opts, options) + super + + opts.separator "skipping options:" + + options[:hide_skip] = true + + opts.on '-q', '--hide-skip', 'Hide skipped tests' do + options[:hide_skip] = true + end + + opts.on '--show-skip', 'Show skipped tests' do + options[:hide_skip] = false + end + end + + def _run_suites(suites, type) + result = super + report.reject!{|r| r.start_with? "Skipped:" } if @options[:hide_skip] + report.sort_by!{|r| r.start_with?("Skipped:") ? 0 : \ + (r.start_with?("Failure:") ? 1 : 2) } + failed(nil) + result + end + end + + module Statistics + def update_list(list, rec, max) + if i = list.empty? ? 0 : list.bsearch_index {|*a| yield(*a)} + list[i, 0] = [rec] + list[max..-1] = [] if list.size >= max + end + end + + def record(suite, method, assertions, time, error) + if @options.values_at(:longest, :most_asserted).any? + @tops ||= {} + rec = [suite.name, method, assertions, time, error] + if max = @options[:longest] + update_list(@tops[:longest] ||= [], rec, max) {|_,_,_,t,_|t<time} + end + if max = @options[:most_asserted] + update_list(@tops[:most_asserted] ||= [], rec, max) {|_,_,a,_,_|a<assertions} + end + end + # (((@record ||= {})[suite] ||= {})[method]) = [assertions, time, error] + super + end + + def run(*args) + result = super + if @tops ||= nil + @tops.each do |t, list| + if list + puts "#{t.to_s.tr('_', ' ')} tests:" + list.each {|suite, method, assertions, time, error| + printf "%5.2fsec(%d): %s#%s\n", time, assertions, suite, method + } + end + end + end + result + end + + private + def setup_options(opts, options) + super + opts.separator "statistics options:" + opts.on '--longest=N', Integer, 'Show longest N tests' do |n| + options[:longest] = n + end + opts.on '--most-asserted=N', Integer, 'Show most asserted N tests' do |n| + options[:most_asserted] = n + end + end + end + + module StatusLine # :nodoc: all + def terminal_width + unless @terminal_width ||= nil + begin + require 'io/console' + width = $stdout.winsize[1] + rescue LoadError, NoMethodError, Errno::ENOTTY, Errno::EBADF, Errno::EINVAL + width = ENV["COLUMNS"].to_i.nonzero? || 80 + end + width -= 1 if /mswin|mingw/ =~ RUBY_PLATFORM + @terminal_width = width + end + @terminal_width + end + + def del_status_line(flush = true) + @status_line_size ||= 0 + if @options[:job_status] == :replace + $stdout.print "\r"+" "*@status_line_size+"\r" + else + $stdout.puts if @status_line_size > 0 + end + $stdout.flush if flush + @status_line_size = 0 + end + + def add_status(line) + @status_line_size ||= 0 + if @options[:job_status] == :replace + line = line[0...(terminal_width-@status_line_size)] + end + print line + @status_line_size += line.size + end + + def jobs_status(worker) + return if !@options[:job_status] or @verbose + if @options[:job_status] == :replace + status_line = @workers.map(&:to_s).join(" ") + else + status_line = worker.to_s + end + update_status(status_line) or (puts; nil) + end + + def del_jobs_status + return unless @options[:job_status] == :replace && @status_line_size.nonzero? + del_status_line + end + + def output + (@output ||= nil) || super + end + + def _prepare_run(suites, type) + options[:job_status] ||= :replace if @tty && !@verbose + case options[:color] + when :always + color = true + when :auto, nil + color = true if @tty || @options[:job_status] == :replace + else + color = false + end + @colorize = Colorize.new(color, colors_file: File.join(__dir__, "../../colors")) + if color or @options[:job_status] == :replace + @verbose = !options[:parallel] + end + @output = Output.new(self) unless @options[:testing] + filter = options[:filter] + type = "#{type}_methods" + total = if filter + suites.inject(0) {|n, suite| n + suite.send(type).grep(filter).size} + else + suites.inject(0) {|n, suite| n + suite.send(type).size} + end + @test_count = 0 + @total_tests = total.to_s(10) + end + + def new_test(s) + @test_count += 1 + update_status(s) + end + + def update_status(s) + count = @test_count.to_s(10).rjust(@total_tests.size) + del_status_line(false) + add_status(@colorize.pass("[#{count}/#{@total_tests}]")) + add_status(" #{s}") + $stdout.print "\r" if @options[:job_status] == :replace and !@verbose + $stdout.flush + end + + def _print(s); $stdout.print(s); end + def succeed; del_status_line; end + + def failed(s) + return if s and @options[:job_status] != :replace + sep = "\n" + @report_count ||= 0 + report.each do |msg| + if msg.start_with? "Skipped:" + if @options[:hide_skip] + del_status_line + next + end + color = :skip + else + color = :fail + end + first, msg = msg.split(/$/, 2) + first = sprintf("%3d) %s", @report_count += 1, first) + $stdout.print(sep, @colorize.decorate(first, color), msg, "\n") + sep = nil + end + report.clear + end + + def initialize + super + @tty = $stdout.tty? + end + + def run(*args) + result = super + puts "\nruby -v: #{RUBY_DESCRIPTION}" + result + end + + private + def setup_options(opts, options) + super + + opts.separator "status line options:" + + options[:job_status] = nil + + opts.on '--jobs-status [TYPE]', [:normal, :replace, :none], + "Show status of jobs every file; Disabled when --jobs isn't specified." do |type| + options[:job_status] = (type || :normal if type != :none) + end + + opts.on '--color[=WHEN]', + [:always, :never, :auto], + "colorize the output. WHEN defaults to 'always'", "or can be 'never' or 'auto'." do |c| + options[:color] = c || :always + end + + opts.on '--tty[=WHEN]', + [:yes, :no], + "force to output tty control. WHEN defaults to 'yes'", "or can be 'no'." do |c| + @tty = c != :no + end + end + + class Output < Struct.new(:runner) # :nodoc: all + def puts(*a) $stdout.puts(*a) unless a.empty? end + def respond_to_missing?(*a) $stdout.respond_to?(*a) end + def method_missing(*a, &b) $stdout.__send__(*a, &b) end + + def print(s) + case s + when /\A(.*\#.*) = \z/ + runner.new_test($1) + when /\A(.* s) = \z/ + runner.add_status(" = #$1") + when /\A\.+\z/ + runner.succeed + when /\A\.*[EFS][EFS.]*\z/ + runner.failed(s) + else + $stdout.print(s) + end + end + end + end + + module LoadPathOption # :nodoc: all + def non_options(files, options) + begin + require "rbconfig" + rescue LoadError + warn "#{caller(1, 1)[0]}: warning: Parallel running disabled because can't get path to ruby; run specify with --ruby argument" + options[:parallel] = nil + else + options[:ruby] ||= [RbConfig.ruby] + end + + super + end + + def setup_options(parser, options) + super + parser.separator "load path options:" + parser.on '-Idirectory', 'Add library load path' do |dirs| + dirs.split(':').each { |d| $LOAD_PATH.unshift d } + end + end + end + + module GlobOption # :nodoc: all + @@testfile_prefix = "test" + @@testfile_suffix = "test" + + def setup_options(parser, options) + super + parser.separator "globbing options:" + parser.on '-B', '--base-directory DIR', 'Base directory to glob.' do |dir| + raise OptionParser::InvalidArgument, "not a directory: #{dir}" unless File.directory?(dir) + options[:base_directory] = dir + end + parser.on '-x', '--exclude REGEXP', 'Exclude test files on pattern.' do |pattern| + (options[:reject] ||= []) << pattern + end + end + + def complement_test_name f, orig_f + basename = File.basename(f) + + if /\.rb\z/ !~ basename + return File.join(File.dirname(f), basename+'.rb') + elsif /\Atest_/ !~ basename + return File.join(File.dirname(f), 'test_'+basename) + end if f.end_with?(basename) # otherwise basename is dirname/ + + raise ArgumentError, "file not found: #{orig_f}" + end + + def non_options(files, options) + paths = [options.delete(:base_directory), nil].uniq + if reject = options.delete(:reject) + reject_pat = Regexp.union(reject.map {|r| %r"#{r}"}) + end + files.map! {|f| + f = f.tr(File::ALT_SEPARATOR, File::SEPARATOR) if File::ALT_SEPARATOR + orig_f = f + while true + ret = ((paths if /\A\.\.?(?:\z|\/)/ !~ f) || [nil]).any? do |prefix| + if prefix + path = f.empty? ? prefix : "#{prefix}/#{f}" + else + next if f.empty? + path = f + end + if f.end_with?(File::SEPARATOR) or !f.include?(File::SEPARATOR) or File.directory?(path) + match = (Dir["#{path}/**/#{@@testfile_prefix}_*.rb"] + Dir["#{path}/**/*_#{@@testfile_suffix}.rb"]).uniq + else + match = Dir[path] + end + if !match.empty? + if reject + match.reject! {|n| + n = n[(prefix.length+1)..-1] if prefix + reject_pat =~ n + } + end + break match + elsif !reject or reject_pat !~ f and File.exist? path + break path + end + end + if !ret + f = complement_test_name(f, orig_f) + else + break ret + end + end + } + files.flatten! + super(files, options) + end + end + + module GCOption # :nodoc: all + def setup_options(parser, options) + super + parser.separator "GC options:" + parser.on '--[no-]gc-stress', 'Set GC.stress as true' do |flag| + options[:gc_stress] = flag + end + parser.on '--[no-]gc-compact', 'GC.compact every time' do |flag| + options[:gc_compact] = flag + end + end + + def non_options(files, options) + if options.delete(:gc_stress) + Test::Unit::TestCase.class_eval do + oldrun = instance_method(:run) + define_method(:run) do |runner| + begin + gc_stress, GC.stress = GC.stress, true + oldrun.bind_call(self, runner) + ensure + GC.stress = gc_stress + end + end + end + end + if options.delete(:gc_compact) + Test::Unit::TestCase.class_eval do + oldrun = instance_method(:run) + define_method(:run) do |runner| + begin + oldrun.bind_call(self, runner) + ensure + GC.compact + end + end + end + end + super + end + end + + module RequireFiles # :nodoc: all + def non_options(files, options) + return false if !super + errors = {} + result = false + files.each {|f| + d = File.dirname(path = File.realpath(f)) + unless $:.include? d + $: << d + end + begin + require path unless options[:parallel] + result = true + rescue LoadError + next if errors[$!.message] + errors[$!.message] = true + puts "#{f}: #{$!}" + end + } + result + end + end + + module RepeatOption # :nodoc: all + def setup_options(parser, options) + super + options[:repeat_count] = nil + parser.separator "repeat options:" + parser.on '--repeat-count=NUM', "Number of times to repeat", Integer do |n| + options[:repeat_count] = n + end + end + + def _run_anything(type) + @repeat_count = @options[:repeat_count] + super + end + end + + module ExcludesOption # :nodoc: all + class ExcludedMethods < Struct.new(:excludes) + def exclude(name, reason) + excludes[name] = reason + end + + def exclude_from(klass) + excludes = self.excludes + pattern = excludes.keys.grep(Regexp).tap {|k| + break (Regexp.new(k.join('|')) unless k.empty?) + } + klass.class_eval do + public_instance_methods(false).each do |method| + if excludes[method] or (pattern and pattern =~ method) + remove_method(method) + end + end + public_instance_methods(true).each do |method| + if excludes[method] or (pattern and pattern =~ method) + undef_method(method) + end + end + end + end + + def self.load(dirs, name) + return unless dirs and name + instance = nil + dirs.each do |dir| + path = File.join(dir, name.gsub(/::/, '/') + ".rb") + begin + src = File.read(path) + rescue Errno::ENOENT + nil + else + instance ||= new({}) + instance.instance_eval(src, path) + end + end + instance + end + end + + def setup_options(parser, options) + super + if excludes = ENV["EXCLUDES"] + excludes = excludes.split(File::PATH_SEPARATOR) + end + options[:excludes] = excludes || [] + parser.separator "excludes options:" + parser.on '-X', '--excludes-dir DIRECTORY', "Directory name of exclude files" do |d| + options[:excludes].concat d.split(File::PATH_SEPARATOR) + end + end + + def _run_suite(suite, type) + if ex = ExcludedMethods.load(@options[:excludes], suite.name) + ex.exclude_from(suite) + end + super + end + end + + module TimeoutOption + def setup_options(parser, options) + super + parser.separator "timeout options:" + parser.on '--timeout-scale NUM', '--subprocess-timeout-scale NUM', "Scale timeout", Float do |scale| + raise OptionParser::InvalidArgument, "timeout scale must be positive" unless scale > 0 + options[:timeout_scale] = scale + end + end + + def non_options(files, options) + if scale = options[:timeout_scale] or + (scale = ENV["RUBY_TEST_TIMEOUT_SCALE"] || ENV["RUBY_TEST_SUBPROCESS_TIMEOUT_SCALE"] and + (scale = scale.to_f) > 0) + EnvUtil.timeout_scale = scale + end + super + end + end + + class Runner # :nodoc: all + + attr_accessor :report, :failures, :errors, :skips # :nodoc: + attr_accessor :assertion_count # :nodoc: + attr_writer :test_count # :nodoc: + attr_accessor :start_time # :nodoc: + attr_accessor :help # :nodoc: + attr_accessor :verbose # :nodoc: + attr_writer :options # :nodoc: + + ## + # :attr: + # + # if true, installs an "INFO" signal handler (only available to BSD and + # OS X users) which prints diagnostic information about the test run. + # + # This is auto-detected by default but may be overridden by custom + # runners. + + attr_accessor :info_signal + + ## + # Lazy accessor for options. + + def options + @options ||= {seed: 42} + end + + @@installed_at_exit ||= false + @@out = $stdout + @@after_tests = [] + @@current_repeat_count = 0 + + ## + # A simple hook allowing you to run a block of code after _all_ of + # the tests are done. Eg: + # + # Test::Unit::Runner.after_tests { p $debugging_info } + + def self.after_tests &block + @@after_tests << block + end + + ## + # Returns the stream to use for output. + + def self.output + @@out + end + + ## + # Sets Test::Unit::Runner to write output to +stream+. $stdout is the default + # output + + def self.output= stream + @@out = stream + end + + ## + # Tells Test::Unit::Runner to delegate to +runner+, an instance of a + # Test::Unit::Runner subclass, when Test::Unit::Runner#run is called. + + def self.runner= runner + @@runner = runner + end + + ## + # Returns the Test::Unit::Runner subclass instance that will be used + # to run the tests. A Test::Unit::Runner instance is the default + # runner. + + def self.runner + @@runner ||= self.new + end + + ## + # Return all plugins' run methods (methods that start with "run_"). + + def self.plugins + @@plugins ||= (["run_tests"] + + public_instance_methods(false). + grep(/^run_/).map { |s| s.to_s }).uniq + end + + ## + # Return the IO for output. + + def output + self.class.output + end + + def puts *a # :nodoc: + output.puts(*a) + end + + def print *a # :nodoc: + output.print(*a) + end + + def test_count # :nodoc: + @test_count ||= 0 + end + + ## + # Runner for a given +type+ (eg, test vs bench). + + def self.current_repeat_count + @@current_repeat_count + end + + def _run_anything type + suites = Test::Unit::TestCase.send "#{type}_suites" + return if suites.empty? + + suites = @order.sort_by_name(suites) + + puts + puts "# Running #{type}s:" + puts + + @test_count, @assertion_count = 0, 0 + test_count = assertion_count = 0 + sync = output.respond_to? :"sync=" # stupid emacs + old_sync, output.sync = output.sync, true if sync + + @@current_repeat_count = 0 + begin + start = Time.now + + results = _run_suites suites, type + + @test_count = results.inject(0) { |sum, (tc, _)| sum + tc } + @assertion_count = results.inject(0) { |sum, (_, ac)| sum + ac } + test_count += @test_count + assertion_count += @assertion_count + t = Time.now - start + @@current_repeat_count += 1 + unless @repeat_count + puts + puts + end + puts "Finished%s %ss in %.6fs, %.4f tests/s, %.4f assertions/s.\n" % + [(@repeat_count ? "(#{@@current_repeat_count}/#{@repeat_count}) " : ""), type, + t, @test_count.fdiv(t), @assertion_count.fdiv(t)] + end while @repeat_count && @@current_repeat_count < @repeat_count && + report.empty? && failures.zero? && errors.zero? + + output.sync = old_sync if sync + + report.each_with_index do |msg, i| + puts "\n%3d) %s" % [i + 1, msg] + end + + puts + @test_count = test_count + @assertion_count = assertion_count + + status + end + + ## + # Run a single +suite+ for a given +type+. + + def _run_suite suite, type + header = "#{type}_suite_header" + puts send(header, suite) if respond_to? header + + filter = options[:filter] + + all_test_methods = suite.send "#{type}_methods" + if filter + all_test_methods.select! {|method| + filter === "#{suite}##{method}" + } + end + all_test_methods = @order.sort_by_name(all_test_methods) + + leakchecker = LeakChecker.new + if ENV["LEAK_CHECKER_TRACE_OBJECT_ALLOCATION"] + require "objspace" + trace = true + end + + assertions = all_test_methods.map { |method| + + inst = suite.new method + _start_method(inst) + inst._assertions = 0 + + print "#{suite}##{method} = " if @verbose + + start_time = Time.now if @verbose + result = + if trace + ObjectSpace.trace_object_allocations {inst.run self} + else + inst.run self + end + + print "%.2f s = " % (Time.now - start_time) if @verbose + print result + puts if @verbose + $stdout.flush + + unless defined?(RubyVM::MJIT) && RubyVM::MJIT.enabled? # compiler process is wrongly considered as leak + leakchecker.check("#{inst.class}\##{inst.__name__}") + end + + _end_method(inst) + + inst._assertions + } + return assertions.size, assertions.inject(0) { |sum, n| sum + n } + end + + def _start_method(inst) + end + def _end_method(inst) + end + + ## + # Record the result of a single test. Makes it very easy to gather + # information. Eg: + # + # class StatisticsRecorder < Test::Unit::Runner + # def record suite, method, assertions, time, error + # # ... record the results somewhere ... + # end + # end + # + # Test::Unit::Runner.runner = StatisticsRecorder.new + # + # NOTE: record might be sent more than once per test. It will be + # sent once with the results from the test itself. If there is a + # failure or error in teardown, it will be sent again with the + # error or failure. + + def record suite, method, assertions, time, error + end + + def location e # :nodoc: + last_before_assertion = "" + + return '<empty>' unless e.backtrace # SystemStackError can return nil. + + e.backtrace.reverse_each do |s| + break if s =~ /in .(assert|refute|flunk|pass|fail|raise|must|wont)/ + last_before_assertion = s + end + last_before_assertion.sub(/:in .*$/, '') + end + + ## + # Writes status for failed test +meth+ in +klass+ which finished with + # exception +e+ + + def initialize # :nodoc: + @report = [] + @errors = @failures = @skips = 0 + @verbose = false + @mutex = Thread::Mutex.new + @info_signal = Signal.list['INFO'] + @repeat_count = nil + end + + def synchronize # :nodoc: + if @mutex then + @mutex.synchronize { yield } + else + yield + end + end + + def inspect + "#<#{self.class.name}: " << + instance_variables.filter_map do |var| + next if var == :@option_parser # too big + "#{var}=#{instance_variable_get(var).inspect}" + end.join(", ") << ">" + end + + ## + # Top level driver, controls all output and filtering. + + def _run args = [] + args = process_args args # ARGH!! blame test/unit process_args + self.options.merge! args + + puts "Run options: #{help}" + + self.class.plugins.each do |plugin| + send plugin + break unless report.empty? + end + + return failures + errors if self.test_count > 0 # or return nil... + rescue Interrupt + abort 'Interrupted' + end + + ## + # Runs test suites matching +filter+. + + def run_tests + _run_anything :test + end + + ## + # Writes status to +io+ + + def status io = self.output + format = "%d tests, %d assertions, %d failures, %d errors, %d skips" + io.puts format % [test_count, assertion_count, failures, errors, skips] + end + + prepend Test::Unit::Options + prepend Test::Unit::StatusLine + prepend Test::Unit::Parallel + prepend Test::Unit::Statistics + prepend Test::Unit::Skipping + prepend Test::Unit::GlobOption + prepend Test::Unit::RepeatOption + prepend Test::Unit::LoadPathOption + prepend Test::Unit::GCOption + prepend Test::Unit::ExcludesOption + prepend Test::Unit::TimeoutOption + prepend Test::Unit::RunCount + + ## + # Begins the full test run. Delegates to +runner+'s #_run method. + + def run(argv = []) + self.class.runner._run(argv) + rescue NoMemoryError + system("cat /proc/meminfo") if File.exist?("/proc/meminfo") + system("ps x -opid,args,%cpu,%mem,nlwp,rss,vsz,wchan,stat,start,time,etime,blocked,caught,ignored,pending,f") if File.exist?("/bin/ps") + raise + end + + @@stop_auto_run = false + def self.autorun + at_exit { + Test::Unit::RunCount.run_once { + exit(Test::Unit::Runner.new.run(ARGV) || true) + } unless @@stop_auto_run + } unless @@installed_at_exit + @@installed_at_exit = true + end + + alias orig_run_suite _run_suite + + # Overriding of Test::Unit::Runner#puke + def puke klass, meth, e + n = report.size + e = case e + when Test::Unit::PendedError then + @skips += 1 + return "S" unless @verbose + "Skipped:\n#{klass}##{meth} [#{location e}]:\n#{e.message}\n" + when Test::Unit::AssertionFailedError then + @failures += 1 + "Failure:\n#{klass}##{meth} [#{location e}]:\n#{e.message}\n" + else + @errors += 1 + bt = Test::filter_backtrace(e.backtrace).join "\n " + "Error:\n#{klass}##{meth}:\n#{e.class}: #{e.message.b}\n #{bt}\n" + end + @report << e + rep = e[0, 1] + if Test::Unit::PendedError === e and /no message given\z/ =~ e.message + report.slice!(n..-1) + rep = "." + end + rep + end + end + + class AutoRunner # :nodoc: all + class Runner < Test::Unit::Runner + include Test::Unit::RequireFiles + end + + attr_accessor :to_run, :options + + def initialize(force_standalone = false, default_dir = nil, argv = ARGV) + @force_standalone = force_standalone + @runner = Runner.new do |files, options| + base = options[:base_directory] ||= default_dir + files << default_dir if files.empty? and default_dir + @to_run = files + yield self if block_given? + $LOAD_PATH.unshift base if base + files + end + Runner.runner = @runner + @options = @runner.option_parser + if @force_standalone + @options.banner.sub!(/\[options\]/, '\& tests...') + end + @argv = argv + end + + def process_args(*args) + @runner.process_args(*args) + !@to_run.empty? + end + + def run + if @force_standalone and not process_args(@argv) + abort @options.banner + end + @runner.run(@argv) || true + end + + def self.run(*args) + new(*args).run + end + end + + class ProxyError < StandardError # :nodoc: all + def initialize(ex) + @message = ex.message + @backtrace = ex.backtrace + end + + attr_accessor :message, :backtrace + end + end +end + +Test::Unit::Runner.autorun diff --git a/tool/lib/test/unit/assertions.rb b/tool/lib/test/unit/assertions.rb new file mode 100644 index 0000000000..dcf3e6fcb9 --- /dev/null +++ b/tool/lib/test/unit/assertions.rb @@ -0,0 +1,839 @@ +# frozen_string_literal: true +require 'pp' + +module Test + module Unit + module Assertions + + ## + # Returns the diff command to use in #diff. Tries to intelligently + # figure out what diff to use. + + def self.diff + unless defined? @diff + exe = RbConfig::CONFIG['EXEEXT'] + @diff = %W"gdiff#{exe} diff#{exe}".find do |diff| + if system(diff, "-u", __FILE__, __FILE__) + break "#{diff} -u" + end + end + end + + @diff + end + + ## + # Set the diff command to use in #diff. + + def self.diff= o + @diff = o + end + + ## + # Returns a diff between +exp+ and +act+. If there is no known + # diff command or if it doesn't make sense to diff the output + # (single line, short output), then it simply returns a basic + # comparison between the two. + + def diff exp, act + require "tempfile" + + expect = mu_pp_for_diff exp + butwas = mu_pp_for_diff act + result = nil + + need_to_diff = + self.class.diff && + (expect.include?("\n") || + butwas.include?("\n") || + expect.size > 30 || + butwas.size > 30 || + expect == butwas) + + return "Expected: #{mu_pp exp}\n Actual: #{mu_pp act}" unless + need_to_diff + + tempfile_a = nil + tempfile_b = nil + + Tempfile.open("expect") do |a| + tempfile_a = a + a.puts expect + a.flush + + Tempfile.open("butwas") do |b| + tempfile_b = b + b.puts butwas + b.flush + + result = `#{self.class.diff} #{a.path} #{b.path}` + result.sub!(/^\-\-\- .+/, "--- expected") + result.sub!(/^\+\+\+ .+/, "+++ actual") + + if result.empty? then + klass = exp.class + result = [ + "No visible difference in the #{klass}#inspect output.\n", + "You should look at the implementation of #== on ", + "#{klass} or its members.\n", + expect, + ].join + end + end + end + + result + ensure + tempfile_a.close! if tempfile_a + tempfile_b.close! if tempfile_b + end + + ## + # This returns a diff-able human-readable version of +obj+. This + # differs from the regular mu_pp because it expands escaped + # newlines and makes hex-values generic (like object_ids). This + # uses mu_pp to do the first pass and then cleans it up. + + def mu_pp_for_diff obj + mu_pp(obj).gsub(/(?<!\\)(?:\\\\)*\K\\n/, "\n").gsub(/:0x[a-fA-F0-9]{4,}/m, ':0xXXXXXX') + end + + ## + # Fails unless +test+ is a true value. + + def assert test, msg = nil + msg ||= "Failed assertion, no message given." + self._assertions += 1 + unless test then + msg = msg.call if Proc === msg + raise Test::Unit::AssertionFailedError, msg + end + true + end + + ## + # Fails unless +obj+ is empty. + + def assert_empty obj, msg = nil + msg = message(msg) { "Expected #{mu_pp(obj)} to be empty" } + assert_respond_to obj, :empty? + assert obj.empty?, msg + end + + ## + # For comparing Floats. Fails unless +exp+ and +act+ are within +delta+ + # of each other. + # + # assert_in_delta Math::PI, (22.0 / 7.0), 0.01 + + def assert_in_delta exp, act, delta = 0.001, msg = nil + n = (exp - act).abs + msg = message(msg) { + "Expected |#{exp} - #{act}| (#{n}) to be <= #{delta}" + } + assert delta >= n, msg + end + + ## + # For comparing Floats. Fails unless +exp+ and +act+ have a relative + # error less than +epsilon+. + + def assert_in_epsilon a, b, epsilon = 0.001, msg = nil + assert_in_delta a, b, [a.abs, b.abs].min * epsilon, msg + end + + ## + # Fails unless +collection+ includes +obj+. + + def assert_includes collection, obj, msg = nil + msg = message(msg) { + "Expected #{mu_pp(collection)} to include #{mu_pp(obj)}" + } + assert_respond_to collection, :include? + assert collection.include?(obj), msg + end + + ## + # Fails unless +obj+ is an instance of +cls+. + + def assert_instance_of cls, obj, msg = nil + msg = message(msg) { + "Expected #{mu_pp(obj)} to be an instance of #{cls}, not #{obj.class}" + } + + assert obj.instance_of?(cls), msg + end + + ## + # Fails unless +obj+ is a kind of +cls+. + + def assert_kind_of cls, obj, msg = nil # TODO: merge with instance_of + msg = message(msg) { + "Expected #{mu_pp(obj)} to be a kind of #{cls}, not #{obj.class}" } + + assert obj.kind_of?(cls), msg + end + + ## + # Fails unless +matcher+ <tt>=~</tt> +obj+. + + def assert_match matcher, obj, msg = nil + msg = message(msg) { "Expected #{mu_pp matcher} to match #{mu_pp obj}" } + assert_respond_to matcher, :"=~" + matcher = Regexp.new Regexp.escape matcher if String === matcher + assert matcher =~ obj, msg + end + + ## + # Fails unless +obj+ is nil + + def assert_nil obj, msg = nil + msg = message(msg) { "Expected #{mu_pp(obj)} to be nil" } + assert obj.nil?, msg + end + + ## + # Fails unless +obj+ is true + + def assert_true obj, msg = nil + msg = message(msg) { "Expected #{mu_pp(obj)} to be true" } + assert obj == true, msg + end + + ## + # Fails unless +obj+ is false + + def assert_false obj, msg = nil + msg = message(msg) { "Expected #{mu_pp(obj)} to be false" } + assert obj == false, msg + end + + ## + # For testing with binary operators. + # + # assert_operator 5, :<=, 4 + + def assert_operator o1, op, o2 = (predicate = true; nil), msg = nil + return assert_predicate o1, op, msg if predicate + msg = message(msg) { "Expected #{mu_pp(o1)} to be #{op} #{mu_pp(o2)}" } + assert o1.__send__(op, o2), msg + end + + ## + # Fails if stdout or stderr do not output the expected results. + # Pass in nil if you don't care about that streams output. Pass in + # "" if you require it to be silent. Pass in a regexp if you want + # to pattern match. + # + # NOTE: this uses #capture_io, not #capture_subprocess_io. + # + # See also: #assert_silent + + def assert_output stdout = nil, stderr = nil + out, err = capture_output do + yield + end + + err_msg = Regexp === stderr ? :assert_match : :assert_equal if stderr + out_msg = Regexp === stdout ? :assert_match : :assert_equal if stdout + + y = send err_msg, stderr, err, "In stderr" if err_msg + x = send out_msg, stdout, out, "In stdout" if out_msg + + (!stdout || x) && (!stderr || y) + end + + ## + # For testing with predicates. + # + # assert_predicate str, :empty? + # + # This is really meant for specs and is front-ended by assert_operator: + # + # str.must_be :empty? + + def assert_predicate o1, op, msg = nil + msg = message(msg) { "Expected #{mu_pp(o1)} to be #{op}" } + assert o1.__send__(op), msg + end + + ## + # Fails unless +obj+ responds to +meth+. + + def assert_respond_to obj, meth, msg = nil + msg = message(msg) { + "Expected #{mu_pp(obj)} (#{obj.class}) to respond to ##{meth}" + } + assert obj.respond_to?(meth), msg + end + + ## + # Fails unless +exp+ and +act+ are #equal? + + def assert_same exp, act, msg = nil + msg = message(msg) { + data = [mu_pp(act), act.object_id, mu_pp(exp), exp.object_id] + "Expected %s (oid=%d) to be the same as %s (oid=%d)" % data + } + assert exp.equal?(act), msg + end + + ## + # Fails if the block outputs anything to stderr or stdout. + # + # See also: #assert_output + + def assert_silent + assert_output "", "" do + yield + end + end + + ## + # Fails unless the block throws +sym+ + + def assert_throws sym, msg = nil + default = "Expected #{mu_pp(sym)} to have been thrown" + caught = true + catch(sym) do + begin + yield + rescue ThreadError => e # wtf?!? 1.8 + threads == suck + default += ", not \:#{e.message[/uncaught throw \`(\w+?)\'/, 1]}" + rescue ArgumentError => e # 1.9 exception + default += ", not #{e.message.split(/ /).last}" + rescue NameError => e # 1.8 exception + default += ", not #{e.name.inspect}" + end + caught = false + end + + assert caught, message(msg) { default } + end + + def assert_path_exists(path, msg = nil) + msg = message(msg) { "Expected path '#{path}' to exist" } + assert File.exist?(path), msg + end + alias assert_path_exist assert_path_exists + alias refute_path_not_exist assert_path_exists + + def refute_path_exists(path, msg = nil) + msg = message(msg) { "Expected path '#{path}' to not exist" } + refute File.exist?(path), msg + end + alias refute_path_exist refute_path_exists + alias assert_path_not_exist refute_path_exists + + ## + # Captures $stdout and $stderr into strings: + # + # out, err = capture_output do + # puts "Some info" + # warn "You did a bad thing" + # end + # + # assert_match %r%info%, out + # assert_match %r%bad%, err + + def capture_output + require 'stringio' + + captured_stdout, captured_stderr = StringIO.new, StringIO.new + + synchronize do + orig_stdout, orig_stderr = $stdout, $stderr + $stdout, $stderr = captured_stdout, captured_stderr + + begin + yield + ensure + $stdout = orig_stdout + $stderr = orig_stderr + end + end + + return captured_stdout.string, captured_stderr.string + end + + def capture_io + raise NoMethodError, "use capture_output" + end + + ## + # Fails with +msg+ + + def flunk msg = nil + msg ||= "Epic Fail!" + assert false, msg + end + + ## + # used for counting assertions + + def pass msg = nil + assert true + end + + ## + # Fails if +test+ is a true value + + def refute test, msg = nil + msg ||= "Failed refutation, no message given" + not assert(! test, msg) + end + + ## + # Fails if +obj+ is empty. + + def refute_empty obj, msg = nil + msg = message(msg) { "Expected #{mu_pp(obj)} to not be empty" } + assert_respond_to obj, :empty? + refute obj.empty?, msg + end + + ## + # Fails if <tt>exp == act</tt>. + # + # For floats use refute_in_delta. + + def refute_equal exp, act, msg = nil + msg = message(msg) { + "Expected #{mu_pp(act)} to not be equal to #{mu_pp(exp)}" + } + refute exp == act, msg + end + + ## + # For comparing Floats. Fails if +exp+ is within +delta+ of +act+. + # + # refute_in_delta Math::PI, (22.0 / 7.0) + + def refute_in_delta exp, act, delta = 0.001, msg = nil + n = (exp - act).abs + msg = message(msg) { + "Expected |#{exp} - #{act}| (#{n}) to not be <= #{delta}" + } + refute delta >= n, msg + end + + ## + # For comparing Floats. Fails if +exp+ and +act+ have a relative error + # less than +epsilon+. + + def refute_in_epsilon a, b, epsilon = 0.001, msg = nil + refute_in_delta a, b, a * epsilon, msg + end + + ## + # Fails if +collection+ includes +obj+. + + def refute_includes collection, obj, msg = nil + msg = message(msg) { + "Expected #{mu_pp(collection)} to not include #{mu_pp(obj)}" + } + assert_respond_to collection, :include? + refute collection.include?(obj), msg + end + + ## + # Fails if +obj+ is an instance of +cls+. + + def refute_instance_of cls, obj, msg = nil + msg = message(msg) { + "Expected #{mu_pp(obj)} to not be an instance of #{cls}" + } + refute obj.instance_of?(cls), msg + end + + ## + # Fails if +obj+ is a kind of +cls+. + + def refute_kind_of cls, obj, msg = nil # TODO: merge with instance_of + msg = message(msg) { "Expected #{mu_pp(obj)} to not be a kind of #{cls}" } + refute obj.kind_of?(cls), msg + end + + ## + # Fails if +matcher+ <tt>=~</tt> +obj+. + + def refute_match matcher, obj, msg = nil + msg = message(msg) {"Expected #{mu_pp matcher} to not match #{mu_pp obj}"} + assert_respond_to matcher, :"=~" + matcher = Regexp.new Regexp.escape matcher if String === matcher + refute matcher =~ obj, msg + end + + ## + # Fails if +obj+ is nil. + + def refute_nil obj, msg = nil + msg = message(msg) { "Expected #{mu_pp(obj)} to not be nil" } + refute obj.nil?, msg + end + + ## + # Fails if +o1+ is not +op+ +o2+. Eg: + # + # refute_operator 1, :>, 2 #=> pass + # refute_operator 1, :<, 2 #=> fail + + def refute_operator o1, op, o2 = (predicate = true; nil), msg = nil + return refute_predicate o1, op, msg if predicate + msg = message(msg) { "Expected #{mu_pp(o1)} to not be #{op} #{mu_pp(o2)}"} + refute o1.__send__(op, o2), msg + end + + ## + # For testing with predicates. + # + # refute_predicate str, :empty? + # + # This is really meant for specs and is front-ended by refute_operator: + # + # str.wont_be :empty? + + def refute_predicate o1, op, msg = nil + msg = message(msg) { "Expected #{mu_pp(o1)} to not be #{op}" } + refute o1.__send__(op), msg + end + + ## + # Fails if +obj+ responds to the message +meth+. + + def refute_respond_to obj, meth, msg = nil + msg = message(msg) { "Expected #{mu_pp(obj)} to not respond to #{meth}" } + + refute obj.respond_to?(meth), msg + end + + ## + # Fails if +exp+ is the same (by object identity) as +act+. + + def refute_same exp, act, msg = nil + msg = message(msg) { + data = [mu_pp(act), act.object_id, mu_pp(exp), exp.object_id] + "Expected %s (oid=%d) to not be the same as %s (oid=%d)" % data + } + refute exp.equal?(act), msg + end + + ## + # Skips the current test. Gets listed at the end of the run but + # doesn't cause a failure exit code. + + def pend msg = nil, bt = caller + msg ||= "Skipped, no message given" + @skip = true + raise Test::Unit::PendedError, msg, bt + end + alias omit pend + + # TODO: Removed this and enabled to raise NoMethodError with skip + alias skip pend + # def skip(msg = nil, bt = caller) + # raise NoMethodError, "use omit or pend", caller + # end + + ## + # Was this testcase skipped? Meant for #teardown. + + def skipped? + defined?(@skip) and @skip + end + + ## + # Takes a block and wraps it with the runner's shared mutex. + + def synchronize + Test::Unit::Runner.runner.synchronize do + yield + end + end + + # :call-seq: + # assert_block( failure_message = nil ) + # + #Tests the result of the given block. If the block does not return true, + #the assertion will fail. The optional +failure_message+ argument is the same as in + #Assertions#assert. + # + # assert_block do + # [1, 2, 3].any? { |num| num < 1 } + # end + def assert_block(*msgs) + assert yield, *msgs + end + + def assert_raises(*exp, &b) + raise NoMethodError, "use assert_raise", caller + end + + # :call-seq: + # assert_nothing_thrown( failure_message = nil, &block ) + # + #Fails if the given block uses a call to Kernel#throw, and + #returns the result of the block otherwise. + # + #An optional failure message may be provided as the final argument. + # + # assert_nothing_thrown "Something was thrown!" do + # throw :problem? + # end + def assert_nothing_thrown(msg=nil) + begin + ret = yield + rescue ArgumentError => error + raise error if /\Auncaught throw (.+)\z/m !~ error.message + msg = message(msg) { "<#{$1}> was thrown when nothing was expected" } + flunk(msg) + end + assert(true, "Expected nothing to be thrown") + ret + end + + # :call-seq: + # assert_equal( expected, actual, failure_message = nil ) + # + #Tests if +expected+ is equal to +actual+. + # + #An optional failure message may be provided as the final argument. + def assert_equal(exp, act, msg = nil) + msg = message(msg) { + exp_str = mu_pp(exp) + act_str = mu_pp(act) + exp_comment = '' + act_comment = '' + if exp_str == act_str + if (exp.is_a?(String) && act.is_a?(String)) || + (exp.is_a?(Regexp) && act.is_a?(Regexp)) + exp_comment = " (#{exp.encoding})" + act_comment = " (#{act.encoding})" + elsif exp.is_a?(Float) && act.is_a?(Float) + exp_str = "%\#.#{Float::DIG+2}g" % exp + act_str = "%\#.#{Float::DIG+2}g" % act + elsif exp.is_a?(Time) && act.is_a?(Time) + if exp.subsec * 1000_000_000 == exp.nsec + exp_comment = " (#{exp.nsec}[ns])" + else + exp_comment = " (subsec=#{exp.subsec})" + end + if act.subsec * 1000_000_000 == act.nsec + act_comment = " (#{act.nsec}[ns])" + else + act_comment = " (subsec=#{act.subsec})" + end + elsif exp.class != act.class + # a subclass of Range, for example. + exp_comment = " (#{exp.class})" + act_comment = " (#{act.class})" + end + elsif !Encoding.compatible?(exp_str, act_str) + if exp.is_a?(String) && act.is_a?(String) + exp_str = exp.dump + act_str = act.dump + exp_comment = " (#{exp.encoding})" + act_comment = " (#{act.encoding})" + else + exp_str = exp_str.dump + act_str = act_str.dump + end + end + "<#{exp_str}>#{exp_comment} expected but was\n<#{act_str}>#{act_comment}" + } + assert(exp == act, msg) + end + + # :call-seq: + # assert_not_nil( expression, failure_message = nil ) + # + #Tests if +expression+ is not nil. + # + #An optional failure message may be provided as the final argument. + def assert_not_nil(exp, msg=nil) + msg = message(msg) { "<#{mu_pp(exp)}> expected to not be nil" } + assert(!exp.nil?, msg) + end + + # :call-seq: + # assert_not_equal( expected, actual, failure_message = nil ) + # + #Tests if +expected+ is not equal to +actual+. + # + #An optional failure message may be provided as the final argument. + def assert_not_equal(exp, act, msg=nil) + msg = message(msg) { "<#{mu_pp(exp)}> expected to be != to\n<#{mu_pp(act)}>" } + assert(exp != act, msg) + end + + # :call-seq: + # assert_no_match( regexp, string, failure_message = nil ) + # + #Tests if the given Regexp does not match a given String. + # + #An optional failure message may be provided as the final argument. + def assert_no_match(regexp, string, msg=nil) + assert_instance_of(Regexp, regexp, "The first argument to assert_no_match should be a Regexp.") + self._assertions -= 1 + msg = message(msg) { "<#{mu_pp(regexp)}> expected to not match\n<#{mu_pp(string)}>" } + assert(regexp !~ string, msg) + end + + # :call-seq: + # assert_not_same( expected, actual, failure_message = nil ) + # + #Tests if +expected+ is not the same object as +actual+. + #This test uses Object#equal? to test equality. + # + #An optional failure message may be provided as the final argument. + # + # assert_not_same("x", "x") #Succeeds + def assert_not_same(expected, actual, message="") + msg = message(msg) { build_message(message, <<EOT, expected, expected.__id__, actual, actual.__id__) } +<?> +with id <?> expected to not be equal\\? to +<?> +with id <?>. +EOT + assert(!actual.equal?(expected), msg) + end + + # :call-seq: + # assert_send( +send_array+, failure_message = nil ) + # + # Passes if the method send returns a true value. + # + # +send_array+ is composed of: + # * A receiver + # * A method + # * Arguments to the method + # + # Example: + # assert_send(["Hello world", :include?, "Hello"]) # -> pass + # assert_send(["Hello world", :include?, "Goodbye"]) # -> fail + def assert_send send_ary, m = nil + recv, msg, *args = send_ary + m = message(m) { + if args.empty? + argsstr = "" + else + (argsstr = mu_pp(args)).sub!(/\A\[(.*)\]\z/m, '(\1)') + end + "Expected #{mu_pp(recv)}.#{msg}#{argsstr} to return true" + } + assert recv.__send__(msg, *args), m + end + + # :call-seq: + # assert_not_send( +send_array+, failure_message = nil ) + # + # Passes if the method send doesn't return a true value. + # + # +send_array+ is composed of: + # * A receiver + # * A method + # * Arguments to the method + # + # Example: + # assert_not_send([[1, 2], :member?, 1]) # -> fail + # assert_not_send([[1, 2], :member?, 4]) # -> pass + def assert_not_send send_ary, m = nil + recv, msg, *args = send_ary + m = message(m) { + if args.empty? + argsstr = "" + else + (argsstr = mu_pp(args)).sub!(/\A\[(.*)\]\z/m, '(\1)') + end + "Expected #{mu_pp(recv)}.#{msg}#{argsstr} to return false" + } + assert !recv.__send__(msg, *args), m + end + + ms = instance_methods(true).map {|sym| sym.to_s } + ms.grep(/\Arefute_/) do |m| + mname = ('assert_not_'.dup << m.to_s[/.*?_(.*)/, 1]) + alias_method(mname, m) unless ms.include? mname + end + alias assert_include assert_includes + alias assert_not_include assert_not_includes + + def assert_not_all?(obj, m = nil, &blk) + failed = [] + obj.each do |*a, &b| + if blk.call(*a, &b) + failed << (a.size > 1 ? a : a[0]) + end + end + assert(failed.empty?, message(m) {failed.pretty_inspect}) + end + + def assert_syntax_error(code, error, *args, **opt) + prepare_syntax_check(code, *args, **opt) do |src, fname, line, mesg| + yield if defined?(yield) + e = assert_raise(SyntaxError, mesg) do + syntax_check(src, fname, line) + end + assert_match(error, e.message, mesg) + e + end + end + + def assert_no_warning(pat, msg = nil) + result = nil + stderr = EnvUtil.verbose_warning { + EnvUtil.with_default_internal(pat.encoding) { + result = yield + } + } + msg = message(msg) {diff pat, stderr} + refute(pat === stderr, msg) + result + end + + # kernel resolution can limit the minimum time we can measure + # [ruby-core:81540] + MIN_HZ = /mswin|mingw/ =~ RUBY_PLATFORM ? 67 : 100 + MIN_MEASURABLE = 1.0 / MIN_HZ + + def assert_cpu_usage_low(msg = nil, pct: 0.05, wait: 1.0, stop: nil) + require 'benchmark' + + wait = EnvUtil.apply_timeout_scale(wait) + if wait < 0.1 # TIME_QUANTUM_USEC in thread_pthread.c + warn "test #{msg || 'assert_cpu_usage_low'} too short to be accurate" + end + tms = Benchmark.measure(msg || '') do + if stop + th = Thread.start {sleep wait; stop.call} + yield + th.join + else + begin + Timeout.timeout(wait) {yield} + rescue Timeout::Error + end + end + end + + max = pct * tms.real + min_measurable = MIN_MEASURABLE + min_measurable *= 1.30 # add a little (30%) to account for misc. overheads + if max < min_measurable + max = min_measurable + end + + assert_operator tms.total, :<=, max, msg + end + + def assert_is_minus_zero(f) + assert(1.0/f == -Float::INFINITY, "#{f} is not -0.0") + end + + def build_message(head, template=nil, *arguments) #:nodoc: + template &&= template.chomp + template.gsub(/\G((?:[^\\]|\\.)*?)(\\)?\?/) { $1 + ($2 ? "?" : mu_pp(arguments.shift)) } + end + end + end +end diff --git a/tool/lib/test/unit/parallel.rb b/tool/lib/test/unit/parallel.rb new file mode 100644 index 0000000000..b3a8957f26 --- /dev/null +++ b/tool/lib/test/unit/parallel.rb @@ -0,0 +1,212 @@ +# frozen_string_literal: true +$LOAD_PATH.unshift "#{__dir__}/../.." +require_relative '../../test/unit' + +require_relative '../../profile_test_all' if ENV.key?('RUBY_TEST_ALL_PROFILE') +require_relative '../../tracepointchecker' +require_relative '../../zombie_hunter' +require_relative '../../iseq_loader_checker' +require_relative '../../gc_checker' + +module Test + module Unit + class Worker < Runner # :nodoc: + class << self + undef autorun + end + + undef _run_suite + undef _run_suites + undef run + + def increment_io(orig) # :nodoc: + *rest, io = 32.times.inject([orig.dup]){|ios, | ios << ios.last.dup } + rest.each(&:close) + io + end + + def _run_suites(suites, type) # :nodoc: + suites.map do |suite| + _run_suite(suite, type) + end + end + + def _start_method(inst) + _report "start", Marshal.dump([inst.class.name, inst.__name__]) + end + + def _run_suite(suite, type) # :nodoc: + @partial_report = [] + orig_testout = Test::Unit::Runner.output + i,o = IO.pipe + + Test::Unit::Runner.output = o + orig_stdin, orig_stdout = $stdin, $stdout + + th = Thread.new do + begin + while buf = (self.verbose ? i.gets : i.readpartial(1024)) + _report "p", buf or break + end + rescue IOError + end + end + + e, f, s = @errors, @failures, @skips + + begin + result = orig_run_suite(suite, type) + rescue Interrupt + @need_exit = true + result = [nil,nil] + end + + Test::Unit::Runner.output = orig_testout + $stdin = orig_stdin + $stdout = orig_stdout + + o.close + begin + th.join + rescue IOError + raise unless /stream closed|closed stream/ =~ $!.message + end + i.close + + result << @partial_report + @partial_report = nil + result << [@errors-e,@failures-f,@skips-s] + result << ($: - @old_loadpath) + result << suite.name + + _report "done", Marshal.dump(result) + return result + ensure + Test::Unit::Runner.output = orig_stdout + $stdin = orig_stdin if orig_stdin + $stdout = orig_stdout if orig_stdout + o.close if o && !o.closed? + i.close if i && !i.closed? + end + + def run(args = []) # :nodoc: + process_args args + @@stop_auto_run = true + @opts = @options.dup + @need_exit = false + + @old_loadpath = [] + begin + begin + @stdout = increment_io(STDOUT) + @stdin = increment_io(STDIN) + rescue + exit 2 + end + exit 2 unless @stdout && @stdin + + @stdout.sync = true + _report "ready!" + while buf = @stdin.gets + case buf.chomp + when /^loadpath (.+?)$/ + @old_loadpath = $:.dup + $:.push(*Marshal.load($1.unpack1("m").force_encoding("ASCII-8BIT"))).uniq! + when /^run (.+?) (.+?)$/ + _report "okay" + + @options = @opts.dup + suites = Test::Unit::TestCase.test_suites + + begin + require File.realpath($1) + rescue LoadError + _report "after", Marshal.dump([$1, ProxyError.new($!)]) + _report "ready" + next + end + _run_suites Test::Unit::TestCase.test_suites-suites, $2.to_sym + + if @need_exit + _report "bye" + exit + else + _report "ready" + end + when /^quit$/ + _report "bye" + exit + end + end + rescue Exception => e + trace = e.backtrace || ['unknown method'] + err = ["#{trace.shift}: #{e.message} (#{e.class})"] + trace.map{|t| "\t" + t } + + if @stdout + _report "bye", Marshal.dump(err.join("\n")) + else + raise "failed to report a failure due to lack of @stdout" + end + exit + ensure + @stdin.close if @stdin + @stdout.close if @stdout + end + end + + def _report(res, *args) # :nodoc: + @stdout.write(args.empty? ? "#{res}\n" : "#{res} #{args.pack("m0")}\n") + true + rescue Errno::EPIPE + rescue TypeError => e + abort("#{e.inspect} in _report(#{res.inspect}, #{args.inspect})\n#{e.backtrace.join("\n")}") + end + + def puke(klass, meth, e) # :nodoc: + if e.is_a?(Test::Unit::PendedError) + new_e = Test::Unit::PendedError.new(e.message) + new_e.set_backtrace(e.backtrace) + e = new_e + end + @partial_report << [klass.name, meth, e.is_a?(Test::Unit::AssertionFailedError) ? e : ProxyError.new(e)] + super + end + + def record(suite, method, assertions, time, error) # :nodoc: + case error + when nil + when Test::Unit::AssertionFailedError, Test::Unit::PendedError + case error.cause + when nil, Test::Unit::AssertionFailedError, Test::Unit::PendedError + else + bt = error.backtrace + error = error.class.new(error.message) + error.set_backtrace(bt) + end + else + error = ProxyError.new(error) + end + _report "record", Marshal.dump([suite.name, method, assertions, time, error]) + super + end + end + end +end + +if $0 == __FILE__ + module Test + module Unit + class TestCase # :nodoc: all + undef on_parallel_worker? + def on_parallel_worker? + true + end + def self.on_parallel_worker? + true + end + end + end + end + require 'rubygems' + Test::Unit::Worker.new.run(ARGV) +end diff --git a/tool/lib/test/unit/testcase.rb b/tool/lib/test/unit/testcase.rb new file mode 100644 index 0000000000..44d9ba7fdb --- /dev/null +++ b/tool/lib/test/unit/testcase.rb @@ -0,0 +1,296 @@ +# frozen_string_literal: true +require_relative 'assertions' +require_relative '../../core_assertions' + +module Test + module Unit + + ## + # Provides a simple set of guards that you can use in your tests + # to skip execution if it is not applicable. These methods are + # mixed into TestCase as both instance and class methods so you + # can use them inside or outside of the test methods. + # + # def test_something_for_mri + # skip "bug 1234" if jruby? + # # ... + # end + # + # if windows? then + # # ... lots of test methods ... + # end + + module Guard + + ## + # Is this running on jruby? + + def jruby? platform = RUBY_PLATFORM + "java" == platform + end + + ## + # Is this running on mri? + + def mri? platform = RUBY_DESCRIPTION + /^ruby/ =~ platform + end + + ## + # Is this running on windows? + + def windows? platform = RUBY_PLATFORM + /mswin|mingw/ =~ platform + end + + ## + # Is this running on mingw? + + def mingw? platform = RUBY_PLATFORM + /mingw/ =~ platform + end + + end + + ## + # Provides before/after hooks for setup and teardown. These are + # meant for library writers, NOT for regular test authors. See + # #before_setup for an example. + + module LifecycleHooks + ## + # Runs before every test, after setup. This hook is meant for + # libraries to extend Test::Unit. It is not meant to be used by + # test developers. + # + # See #before_setup for an example. + + def after_setup; end + + ## + # Runs before every test, before setup. This hook is meant for + # libraries to extend Test::Unit. It is not meant to be used by + # test developers. + # + # As a simplistic example: + # + # module MyTestUnitPlugin + # def before_setup + # super + # # ... stuff to do before setup is run + # end + # + # def after_setup + # # ... stuff to do after setup is run + # super + # end + # + # def before_teardown + # super + # # ... stuff to do before teardown is run + # end + # + # def after_teardown + # # ... stuff to do after teardown is run + # super + # end + # end + # + # class Test::Unit::Runner::TestCase + # include MyTestUnitPlugin + # end + + def before_setup; end + + ## + # Runs after every test, before teardown. This hook is meant for + # libraries to extend Test::Unit. It is not meant to be used by + # test developers. + # + # See #before_setup for an example. + + def before_teardown; end + + ## + # Runs after every test, after teardown. This hook is meant for + # libraries to extend Test::Unit. It is not meant to be used by + # test developers. + # + # See #before_setup for an example. + + def after_teardown; end + end + + ## + # Subclass TestCase to create your own tests. Typically you'll want a + # TestCase subclass per implementation class. + # + # See <code>Test::Unit::AssertionFailedError</code>s + + class TestCase + include Assertions + include CoreAssertions + + include LifecycleHooks + include Guard + extend Guard + + attr_reader :__name__ # :nodoc: + + PASSTHROUGH_EXCEPTIONS = [NoMemoryError, SignalException, + Interrupt, SystemExit] # :nodoc: + + ## + # Runs the tests reporting the status to +runner+ + + def run runner + @options = runner.options + + trap "INFO" do + runner.report.each_with_index do |msg, i| + warn "\n%3d) %s" % [i + 1, msg] + end + warn '' + time = runner.start_time ? Time.now - runner.start_time : 0 + warn "Current Test: %s#%s %.2fs" % [self.class, self.__name__, time] + runner.status $stderr + end if runner.info_signal + + start_time = Time.now + + result = "" + + begin + @passed = nil + self.before_setup + self.setup + self.after_setup + self.run_test self.__name__ + result = "." unless io? + time = Time.now - start_time + runner.record self.class, self.__name__, self._assertions, time, nil + @passed = true + rescue *PASSTHROUGH_EXCEPTIONS + raise + rescue Exception => e + @passed = Test::Unit::PendedError === e + time = Time.now - start_time + runner.record self.class, self.__name__, self._assertions, time, e + result = runner.puke self.class, self.__name__, e + ensure + %w{ before_teardown teardown after_teardown }.each do |hook| + begin + self.send hook + rescue *PASSTHROUGH_EXCEPTIONS + raise + rescue Exception => e + @passed = false + runner.record self.class, self.__name__, self._assertions, time, e + result = runner.puke self.class, self.__name__, e + end + end + trap 'INFO', 'DEFAULT' if runner.info_signal + end + result + end + + RUN_TEST_TRACE = "#{__FILE__}:#{__LINE__+3}:in `run_test'".freeze + def run_test(name) + progname, $0 = $0, "#{$0}: #{self.class}##{name}" + self.__send__(name) + ensure + $@.delete(RUN_TEST_TRACE) if $@ + $0 = progname + end + + def initialize name # :nodoc: + @__name__ = name + @__io__ = nil + @passed = nil + @@current = self # FIX: make thread local + end + + def self.current # :nodoc: + @@current # FIX: make thread local + end + + ## + # Return the output IO object + + def io + @__io__ = true + Test::Unit::Runner.output + end + + ## + # Have we hooked up the IO yet? + + def io? + @__io__ + end + + def self.reset # :nodoc: + @@test_suites = {} + @@test_suites[self] = true + end + + reset + + def self.inherited klass # :nodoc: + @@test_suites[klass] = true + super + end + + @test_order = :sorted + + class << self + attr_writer :test_order + end + + def self.test_order + defined?(@test_order) ? @test_order : superclass.test_order + end + + def self.test_suites # :nodoc: + @@test_suites.keys + end + + def self.test_methods # :nodoc: + public_instance_methods(true).grep(/^test/) + end + + ## + # Returns true if the test passed. + + def passed? + @passed + end + + ## + # Runs before every test. Use this to set up before each test + # run. + + def setup; end + + ## + # Runs after every test. Use this to clean up after each test + # run. + + def teardown; end + + def on_parallel_worker? + false + end + + def self.method_added(name) + super + return unless name.to_s.start_with?("test_") + @test_methods ||= {} + if @test_methods[name] + raise AssertionFailedError, "test/unit: method #{ self }##{ name } is redefined" + end + @test_methods[name] = true + end + end + end +end diff --git a/tool/lib/tracepointchecker.rb b/tool/lib/tracepointchecker.rb new file mode 100644 index 0000000000..3254e59357 --- /dev/null +++ b/tool/lib/tracepointchecker.rb @@ -0,0 +1,126 @@ +# frozen_string_literal: true +module TracePointChecker + STATE = { + count: 0, + running: false, + } + + module ZombieTraceHunter + def tracepoint_capture_stat_get + TracePoint.stat.map{|k, (activated, deleted)| + deleted = 0 unless @tracepoint_captured_singlethread + [k, activated, deleted] + } + end + + def before_setup + @tracepoint_captured_singlethread = (Thread.list.size == 1) + @tracepoint_captured_stat = tracepoint_capture_stat_get() + super + end + + def after_teardown + super + + # detect zombie traces. + assert_equal( + @tracepoint_captured_stat, + tracepoint_capture_stat_get(), + "The number of active/deleted trace events was changed" + ) + # puts "TracePoint - deleted: #{deleted}" if deleted > 0 + + TracePointChecker.check if STATE[:running] + end + end + + MAIN_THREAD = Thread.current + TRACES = [] + + def self.prefix event + case event + when :call, :return + :n + when :c_call, :c_return + :c + when :b_call, :b_return + :b + end + end + + def self.clear_call_stack + Thread.current[:call_stack] = [] + end + + def self.call_stack + stack = Thread.current[:call_stack] + stack = clear_call_stack unless stack + stack + end + + def self.verbose_out label, method + puts label => call_stack, :count => STATE[:count], :method => method + end + + def self.method_label tp + "#{prefix(tp.event)}##{tp.method_id}" + end + + def self.start verbose: false, stop_at_failure: false + call_events = %i(a_call) + return_events = %i(a_return) + clear_call_stack + + STATE[:running] = true + + TRACES << TracePoint.new(*call_events){|tp| + next if Thread.current != MAIN_THREAD + + method = method_label(tp) + call_stack.push method + STATE[:count] += 1 + + verbose_out :push, method if verbose + } + + TRACES << TracePoint.new(*return_events){|tp| + next if Thread.current != MAIN_THREAD + STATE[:count] += 1 + + method = "#{prefix(tp.event)}##{tp.method_id}" + verbose_out :pop1, method if verbose + + stored_method = call_stack.pop + next if stored_method.nil? + + verbose_out :pop2, method if verbose + + if stored_method != method + stop if stop_at_failure + RubyVM::SDR() if defined? RubyVM::SDR() + call_stack.clear + raise "#{stored_method} is expected, but #{method} (count: #{STATE[:count]})" + end + } + + TRACES.each{|trace| trace.enable} + end + + def self.stop + STATE[:running] = true + TRACES.each{|trace| trace.disable} + TRACES.clear + end + + def self.check + TRACES.each{|trace| + raise "trace #{trace} should not be deactivated" unless trace.enabled? + } + end +end if defined?(TracePoint.stat) + +class ::Test::Unit::TestCase + include TracePointChecker::ZombieTraceHunter +end if defined?(TracePointChecker) + +# TracePointChecker.start verbose: false diff --git a/tool/lib/vcs.rb b/tool/lib/vcs.rb new file mode 100644 index 0000000000..c41276e3b4 --- /dev/null +++ b/tool/lib/vcs.rb @@ -0,0 +1,733 @@ +# vcs +require 'fileutils' +require 'optparse' + +# This library is used by several other tools/ scripts to detect the current +# VCS in use (e.g. SVN, Git) or to interact with that VCS. + +ENV.delete('PWD') + +class VCS + DEBUG_OUT = STDERR.dup +end + +unless File.respond_to? :realpath + require 'pathname' + def File.realpath(arg) + Pathname(arg).realpath.to_s + end +end + +def IO.pread(*args) + VCS::DEBUG_OUT.puts(args.inspect) if $DEBUG + popen(*args) {|f|f.read} +end + +module DebugPOpen + refine IO.singleton_class do + def popen(*args) + VCS::DEBUG_OUT.puts args.inspect if $DEBUG + super + end + end +end +using DebugPOpen +module DebugSystem + def system(*args) + VCS::DEBUG_OUT.puts args.inspect if $DEBUG + exception = false + opts = Hash.try_convert(args[-1]) + if RUBY_VERSION >= "2.6" + unless opts + opts = {} + args << opts + end + exception = opts.fetch(:exception) {opts[:exception] = true} + elsif opts + exception = opts.delete(:exception) {true} + args.pop if opts.empty? + end + ret = super(*args) + raise "Command failed with status (#$?): #{args[0]}" if exception and !ret + ret + end +end + +class VCS + prepend(DebugSystem) if defined?(DebugSystem) + class NotFoundError < RuntimeError; end + + @@dirs = [] + def self.register(dir, &pred) + @@dirs << [dir, self, pred] + end + + def self.detect(path = '.', options = {}, parser = nil, **opts) + options.update(opts) + uplevel_limit = options.fetch(:uplevel_limit, 0) + curr = path + begin + @@dirs.each do |dir, klass, pred| + if pred ? pred[curr, dir] : File.directory?(File.join(curr, dir)) + vcs = klass.new(curr) + vcs.define_options(parser) if parser + vcs.set_options(options) + return vcs + end + end + if uplevel_limit + break if uplevel_limit.zero? + uplevel_limit -= 1 + end + prev, curr = curr, File.realpath(File.join(curr, '..')) + end until curr == prev # stop at the root directory + raise VCS::NotFoundError, "does not seem to be under a vcs: #{path}" + end + + def self.local_path?(path) + String === path or path.respond_to?(:to_path) + end + + def self.define_options(parser, opts = {}) + parser.separator(" VCS common options:") + parser.define("--[no-]dryrun") {|v| opts[:dryrun] = v} + parser.define("--[no-]debug") {|v| opts[:debug] = v} + opts + end + + attr_reader :srcdir + + def initialize(path) + @srcdir = path + super() + end + + def chdir(path) + @srcdir = path + end + + def define_options(parser) + end + + def set_options(opts) + @debug = opts.fetch(:debug) {$DEBUG} + @dryrun = opts.fetch(:dryrun) {@debug} + end + + attr_reader :dryrun, :debug + alias dryrun? dryrun + alias debug? debug + + NullDevice = defined?(IO::NULL) ? IO::NULL : + %w[/dev/null NUL NIL: NL:].find {|dev| File.exist?(dev)} + + # returns + # * the last revision of the current branch + # * the last revision in which +path+ was modified + # * the last modified time of +path+ + # * the last commit title since the latest upstream + def get_revisions(path) + if self.class.local_path?(path) + path = relative_to(path) + end + last, changed, modified, *rest = ( + begin + if NullDevice + save_stderr = STDERR.dup + STDERR.reopen NullDevice, 'w' + end + _get_revisions(path, @srcdir) + rescue Errno::ENOENT => e + raise VCS::NotFoundError, e.message + ensure + if save_stderr + STDERR.reopen save_stderr + save_stderr.close + end + end + ) + last or raise VCS::NotFoundError, "last revision not found" + changed or raise VCS::NotFoundError, "changed revision not found" + if modified + /\A(\d+)-(\d+)-(\d+)\D(\d+):(\d+):(\d+(?:\.\d+)?)\s*(?:Z|([-+]\d\d)(\d\d))\z/ =~ modified or + raise "unknown time format - #{modified}" + match = $~[1..6].map { |x| x.to_i } + off = $7 ? "#{$7}:#{$8}" : "+00:00" + match << off + begin + modified = Time.new(*match) + rescue ArgumentError + modified = Time.utc(*$~[1..6]) + $7.to_i * 3600 + $8.to_i * 60 + end + end + return last, changed, modified, *rest + end + + def modified(path) + _, _, modified, * = get_revisions(path) + modified + end + + def relative_to(path) + if path + srcdir = File.realpath(@srcdir) + path = File.realdirpath(path) + list1 = srcdir.split(%r{/}) + list2 = path.split(%r{/}) + while !list1.empty? && !list2.empty? && list1.first == list2.first + list1.shift + list2.shift + end + if list1.empty? && list2.empty? + "." + else + ([".."] * list1.length + list2).join("/") + end + else + '.' + end + end + + def after_export(dir) + FileUtils.rm_rf(Dir.glob("#{dir}/.git*")) + end + + def revision_handler(rev) + self.class + end + + def revision_name(rev) + revision_handler(rev).revision_name(rev) + end + + def short_revision(rev) + revision_handler(rev).short_revision(rev) + end + + class SVN < self + register(".svn") + COMMAND = ENV['SVN'] || 'svn' + + def self.revision_name(rev) + "r#{rev}" + end + + def self.short_revision(rev) + rev + end + + def _get_revisions(path, srcdir = nil) + if srcdir and self.class.local_path?(path) + path = File.join(srcdir, path) + end + if srcdir + info_xml = IO.pread(%W"#{COMMAND} info --xml #{srcdir}") + info_xml = nil unless info_xml[/<url>(.*)<\/url>/, 1] == path.to_s + end + info_xml ||= IO.pread(%W"#{COMMAND} info --xml #{path}") + _, last, _, changed, _ = info_xml.split(/revision="(\d+)"/) + modified = info_xml[/<date>([^<>]*)/, 1] + branch = info_xml[%r'<relative-url>\^/(?:branches/|tags/)?([^<>]+)', 1] + [Integer(last), Integer(changed), modified, branch] + end + + def self.search_root(path) + return unless local_path?(path) + parent = File.realpath(path) + begin + parent = File.dirname(wkdir = parent) + return wkdir if File.directory?(wkdir + "/.svn") + end until parent == wkdir + end + + def get_info + @info ||= IO.pread(%W"#{COMMAND} info --xml #{@srcdir}") + end + + def url + @url ||= begin + url = get_info[/<root>(.*)<\/root>/, 1] + @url = URI.parse(url+"/") if url + end + end + + def wcroot + @wcroot ||= begin + info = get_info + @wcroot = info[/<wcroot-abspath>(.*)<\/wcroot-abspath>/, 1] + @wcroot ||= self.class.search_root(@srcdir) + end + end + + def branch(name) + return trunk if name == "trunk" + url + "branches/#{name}" + end + + def tag(name) + url + "tags/#{name}" + end + + def trunk + url + "trunk" + end + alias master trunk + + def branch_list(pat) + IO.popen(%W"#{COMMAND} ls #{branch('')}") do |f| + f.each do |line| + line.chomp! + line.chomp!('/') + yield(line) if File.fnmatch?(pat, line) + end + end + end + + def grep(pat, tag, *files, &block) + cmd = %W"#{COMMAND} cat" + files.map! {|n| File.join(tag, n)} if tag + set = block.binding.eval("proc {|match| $~ = match}") + IO.popen([cmd, *files]) do |f| + f.grep(pat) do |s| + set[$~] + yield s + end + end + end + + def export(revision, url, dir, keep_temp = false) + if @srcdir and (rootdir = wcroot) + srcdir = File.realpath(@srcdir) + rootdir << "/" + if srcdir.start_with?(rootdir) + subdir = srcdir[rootdir.size..-1] + subdir = nil if subdir.empty? + FileUtils.mkdir_p(svndir = dir+"/.svn") + FileUtils.ln_s(Dir.glob(rootdir+"/.svn/*"), svndir) + system(COMMAND, "-q", "revert", "-R", subdir || ".", :chdir => dir) or return false + FileUtils.rm_rf(svndir) unless keep_temp + if subdir + tmpdir = Dir.mktmpdir("tmp-co.", "#{dir}/#{subdir}") + File.rename(tmpdir, tmpdir = "#{dir}/#{File.basename(tmpdir)}") + FileUtils.mv(Dir.glob("#{dir}/#{subdir}/{.[^.]*,..?*,*}"), tmpdir) + begin + Dir.rmdir("#{dir}/#{subdir}") + end until (subdir = File.dirname(subdir)) == '.' + FileUtils.mv(Dir.glob("#{tmpdir}/#{subdir}/{.[^.]*,..?*,*}"), dir) + Dir.rmdir(tmpdir) + end + return self + end + end + IO.popen(%W"#{COMMAND} export -r #{revision} #{url} #{dir}") do |pipe| + pipe.each {|line| /^A/ =~ line or yield line} + end + self if $?.success? + end + + def after_export(dir) + super + FileUtils.rm_rf(dir+"/.svn") + end + + def branch_beginning(url) + # `--limit` of svn-log is useless in this case, because it is + # applied before `--search`. + rev = IO.pread(%W[ #{COMMAND} log --xml + --search=matz --search-and=has\ started + -- #{url}/version.h])[/<logentry\s+revision="(\d+)"/m, 1] + rev.to_i if rev + end + + def export_changelog(url = '.', from = nil, to = nil, _path = nil, path: _path) + range = [to || 'HEAD', (from ? from+1 : branch_beginning(url))].compact.join(':') + IO.popen({'TZ' => 'JST-9', 'LANG' => 'C', 'LC_ALL' => 'C'}, + %W"#{COMMAND} log -r#{range} #{url}") do |r| + IO.copy_stream(r, path) + end + end + + def commit + args = %W"#{COMMAND} commit" + if dryrun? + VCS::DEBUG_OUT.puts(args.inspect) + return true + end + system(*args) + end + end + + class GIT < self + register(".git") {|path, dir| File.exist?(File.join(path, dir))} + COMMAND = ENV["GIT"] || 'git' + + def cmd_args(cmds, srcdir = nil) + (opts = cmds.last).kind_of?(Hash) or cmds << (opts = {}) + opts[:external_encoding] ||= "UTF-8" + if srcdir + opts[:chdir] ||= srcdir + end + VCS::DEBUG_OUT.puts cmds.inspect if debug? + cmds + end + + def cmd_pipe_at(srcdir, cmds, &block) + without_gitconfig { IO.popen(*cmd_args(cmds, srcdir), &block) } + end + + def cmd_read_at(srcdir, cmds) + result = without_gitconfig { IO.pread(*cmd_args(cmds, srcdir)) } + VCS::DEBUG_OUT.puts result.inspect if debug? + result + end + + def cmd_pipe(*cmds, &block) + cmd_pipe_at(@srcdir, cmds, &block) + end + + def cmd_read(*cmds) + cmd_read_at(@srcdir, cmds) + end + + def svn_revision(log) + if /^ *git-svn-id: .*@(\d+) .*\n+\z/ =~ log + $1.to_i + end + end + + def _get_revisions(path, srcdir = nil) + ref = Branch === path ? path.to_str : 'HEAD' + gitcmd = [COMMAND] + last = cmd_read_at(srcdir, [[*gitcmd, 'rev-parse', ref]]).rstrip + log = cmd_read_at(srcdir, [[*gitcmd, 'log', '-n1', '--date=iso', '--pretty=fuller', *path]]) + changed = log[/\Acommit (\h+)/, 1] + modified = log[/^CommitDate:\s+(.*)/, 1] + if rev = svn_revision(log) + if changed == last + last = rev + else + svn_rev = svn_revision(cmd_read_at(srcdir, [[*gitcmd, 'log', '-n1', '--format=%B', last]])) + last = svn_rev if svn_rev + end + changed = rev + end + branch = cmd_read_at(srcdir, [gitcmd + %W[symbolic-ref --short #{ref}]]) + if branch.empty? + branch = cmd_read_at(srcdir, [gitcmd + %W[describe --contains #{ref}]]).strip + end + if branch.empty? + branch_list = cmd_read_at(srcdir, [gitcmd + %W[branch --list --contains #{ref}]]).lines.to_a + branch, = branch_list.grep(/\A\*/) + case branch + when /\A\* *\(\S+ detached at (.*)\)\Z/ + branch = $1 + branch = nil if last.start_with?(branch) + when /\A\* (\S+)\Z/ + branch = $1 + else + branch = nil + end + unless branch + branch_list.each {|b| b.strip!} + branch_list.delete_if {|b| / / =~ b} + branch = branch_list.min_by(&:length) || "" + end + end + branch.chomp! + branch = ":detached:" if branch.empty? + upstream = cmd_read_at(srcdir, [gitcmd + %W[branch --list --format=%(upstream:short) #{branch}]]) + upstream.chomp! + title = cmd_read_at(srcdir, [gitcmd + %W[log --format=%s -n1 #{upstream}..#{ref}]]) + title = nil if title.empty? + [last, changed, modified, branch, title] + end + + def self.revision_name(rev) + short_revision(rev) + end + + def self.short_revision(rev) + rev[0, 10] + end + + def revision_handler(rev) + case rev + when Integer + SVN + else + super + end + end + + def without_gitconfig + home = ENV.delete('HOME') + yield + ensure + ENV['HOME'] = home if home + end + + def initialize(*) + super + @srcdir = File.realpath(@srcdir) + VCS::DEBUG_OUT.puts @srcdir.inspect if debug? + self + end + + Branch = Struct.new(:to_str) + + def branch(name) + Branch.new(name) + end + + alias tag branch + + def master + branch("master") + end + alias trunk master + + def stable + cmd = %W"#{COMMAND} for-each-ref --format=\%(refname:short) refs/heads/ruby_[0-9]*" + branch(cmd_read(cmd)[/.*^(ruby_\d+_\d+)$/m, 1]) + end + + def branch_list(pat) + cmd = %W"#{COMMAND} for-each-ref --format=\%(refname:short) refs/heads/#{pat}" + cmd_pipe(cmd) {|f| + f.each {|line| + line.chomp! + yield line + } + } + end + + def grep(pat, tag, *files, &block) + cmd = %W[#{COMMAND} grep -h --perl-regexp #{tag} --] + set = block.binding.eval("proc {|match| $~ = match}") + cmd_pipe(cmd+files) do |f| + f.grep(pat) do |s| + set[$~] + yield s + end + end + end + + def export(revision, url, dir, keep_temp = false) + system(COMMAND, "clone", "-c", "advice.detachedHead=false", "-s", (@srcdir || '.').to_s, "-b", url, dir) or return + (Integer === revision ? GITSVN : GIT).new(File.expand_path(dir)) + end + + def branch_beginning(url) + cmd_read(%W[ #{COMMAND} log -n1 --format=format:%H + --author=matz --committer=matz --grep=started\\.$ + #{url.to_str} -- version.h include/ruby/version.h]) + end + + def export_changelog(url = '@', from = nil, to = nil, _path = nil, path: _path, base_url: nil) + svn = nil + from, to = [from, to].map do |rev| + rev or next + if Integer === rev + svn = true + rev = cmd_read({'LANG' => 'C', 'LC_ALL' => 'C'}, + %W"#{COMMAND} log -n1 --format=format:%H" << + "--grep=^ *git-svn-id: .*@#{rev} ") + end + rev unless rev.empty? + end + unless (from && /./.match(from)) or ((from = branch_beginning(url)) && /./.match(from)) + warn "no starting commit found", uplevel: 1 + from = nil + end + if svn or system(*%W"#{COMMAND} fetch origin refs/notes/commits:refs/notes/commits", + chdir: @srcdir, exception: false) + system(*%W"#{COMMAND} fetch origin refs/notes/log-fix:refs/notes/log-fix", + chdir: @srcdir, exception: false) + else + warn "Could not fetch notes/commits tree", uplevel: 1 + end + to ||= url.to_str + if from + arg = ["#{from}^..#{to}"] + else + arg = ["--since=25 Dec 00:00:00", to] + end + writer = + if svn + format_changelog_as_svn(path, arg) + else + if base_url == true + remote, = upstream + if remote &&= cmd_read(env, %W[#{COMMAND} remote get-url --no-push #{remote}]) + remote.chomp! + # hack to redirect git.r-l.o to github + remote.sub!(/\Agit@git\.ruby-lang\.org:/, 'git@github.com:ruby/') + remote.sub!(/\Agit@(.*?):(.*?)(?:\.git)?\z/, 'https://\1/\2/commit/') + end + base_url = remote + end + format_changelog(path, arg, base_url) + end + if !path or path == '-' + writer[$stdout] + else + File.open(path, 'wb', &writer) + end + end + + LOG_FIX_REGEXP_SEPARATORS = '/!:;|,#%&' + + def format_changelog(path, arg, base_url = nil) + env = {'TZ' => 'JST-9', 'LANG' => 'C', 'LC_ALL' => 'C'} + cmd = %W"#{COMMAND} log --format=fuller --notes=commits --notes=log-fix --topo-order --no-merges" + date = "--date=iso-local" + unless system(env, *cmd, date, chdir: @srcdir, out: NullDevice, exception: false) + date = "--date=iso" + end + cmd << date + cmd.concat(arg) + proc do |w| + w.print "-*- coding: utf-8 -*-\n\n" + w.print "base-url = #{base_url}\n\n" if base_url + cmd_pipe(env, cmd, chdir: @srcdir) do |r| + while s = r.gets("\ncommit ") + h, s = s.split(/^$/, 2) + h.gsub!(/^(?:(?:Author|Commit)(?:Date)?|Date): /, ' \&') + if s.sub!(/\nNotes \(log-fix\):\n((?: +.*\n)+)/, '') + fix = $1 + s = s.lines + fix.each_line do |x| + case x + when %r[^ +(\d+)s([#{LOG_FIX_REGEXP_SEPARATORS}])(.+)\2(.*)\2]o + n = $1.to_i + wrong = $3 + correct = $4 + begin + s[n][wrong] = correct + rescue IndexError + message = ["format_changelog failed to replace #{wrong.dump} with #{correct.dump} at #$1\n"] + from = [1, n-2].max + to = [s.size-1, n+2].min + s.each_with_index do |e, i| + next if i < from + break if to < i + message << "#{i}:#{e}" + end + raise message.join('') + end + when %r[^( +)(\d+)i([#{LOG_FIX_REGEXP_SEPARATORS}])(.*)\3]o + s[$2.to_i, 0] = "#{$1}#{$4}\n" + when %r[^ +(\d+)(?:,(\d+))?d] + n = $1.to_i + e = $2 + s[n..(e ? e.to_i : n)] = [] + end + end + s = s.join('') + end + + if %r[^ +(https://github\.com/[^/]+/[^/]+/)commit/\h+\n(?=(?: +\n(?i: +Co-authored-by: .*\n)+)?(?:\n|\Z))] =~ s + issue = "#{$1}pull/" + s.gsub!(/\b[Ff]ix(?:e[sd])? \K#(?=\d+)/) {issue} + end + + s.gsub!(/ +\n/, "\n") + s.sub!(/^Notes:/, ' \&') + w.print h, s + end + end + end + end + + def format_changelog_as_svn(path, arg) + cmd = %W"#{COMMAND} log --topo-order --no-notes -z --format=%an%n%at%n%B" + cmd.concat(arg) + proc do |w| + sep = "-"*72 + "\n" + w.print sep + cmd_pipe(cmd) do |r| + while s = r.gets("\0") + s.chomp!("\0") + author, time, s = s.split("\n", 3) + s.sub!(/\n\ngit-svn-id: .*@(\d+) .*\n\Z/, '') + rev = $1 + time = Time.at(time.to_i).getlocal("+09:00").strftime("%F %T %z (%a, %d %b %Y)") + lines = s.count("\n") + 1 + lines = "#{lines} line#{lines == 1 ? '' : 's'}" + w.print "r#{rev} | #{author} | #{time} | #{lines}\n\n", s, "\n", sep + end + end + end + end + + def upstream + (branch = cmd_read(%W"#{COMMAND} symbolic-ref --short HEAD")).chomp! + (upstream = cmd_read(%W"#{COMMAND} branch --list --format=%(upstream) #{branch}")).chomp! + while ref = upstream[%r"\Arefs/heads/(.*)", 1] + upstream = cmd_read(%W"#{COMMAND} branch --list --format=%(upstream) #{ref}") + end + unless %r"\Arefs/remotes/([^/]+)/(.*)" =~ upstream + raise "Upstream not found" + end + [$1, $2] + end + + def commit(opts = {}) + args = [COMMAND, "push"] + args << "-n" if dryrun + remote, branch = upstream + args << remote + branches = %W[refs/notes/commits:refs/notes/commits HEAD:#{branch}] + if dryrun? + branches.each do |b| + VCS::DEBUG_OUT.puts((args + [b]).inspect) + end + return true + end + branches.each do |b| + system(*(args + [b])) or return false + end + true + end + end + + class GITSVN < GIT + def self.revision_name(rev) + SVN.revision_name(rev) + end + + def last_changed_revision + rev = cmd_read(%W"#{COMMAND} svn info"+[STDERR=>[:child, :out]])[/^Last Changed Rev: (\d+)/, 1] + com = cmd_read(%W"#{COMMAND} svn find-rev r#{rev}").chomp + return rev, com + end + + def commit(opts = {}) + rev, com = last_changed_revision + head = cmd_read(%W"#{COMMAND} symbolic-ref --short HEAD").chomp + + commits = cmd_read([COMMAND, "log", "--reverse", "--format=%H %ae %ce", "#{com}..@"], "rb").split("\n") + commits.each_with_index do |l, i| + r, a, c = l.split(' ') + dcommit = [COMMAND, "svn", "dcommit"] + dcommit.insert(-2, "-n") if dryrun + dcommit << "--add-author-from" unless a == c + dcommit << r + system(*dcommit) or return false + system(COMMAND, "checkout", head) or return false + system(COMMAND, "rebase") or return false + end + + if rev + old = [cmd_read(%W"#{COMMAND} log -1 --format=%H").chomp] + old << cmd_read(%W"#{COMMAND} svn reset -r#{rev}")[/^r#{rev} = (\h+)/, 1] + 3.times do + sleep 2 + system(*%W"#{COMMAND} pull --no-edit --rebase") + break unless old.include?(cmd_read(%W"#{COMMAND} log -1 --format=%H").chomp) + end + end + true + end + end +end diff --git a/tool/lib/vpath.rb b/tool/lib/vpath.rb new file mode 100644 index 0000000000..48ab148405 --- /dev/null +++ b/tool/lib/vpath.rb @@ -0,0 +1,87 @@ +# -*- coding: us-ascii -*- + +class VPath + attr_accessor :separator + + def initialize(*list) + @list = list + @additional = [] + @separator = nil + end + + def inspect + list.inspect + end + + def search(meth, base, *rest) + begin + meth.call(base, *rest) + rescue Errno::ENOENT => error + list.each do |dir| + return meth.call(File.join(dir, base), *rest) rescue nil + end + raise error + end + end + + def process(*args, &block) + search(File.method(__callee__), *args, &block) + end + + alias stat process + alias lstat process + + def open(*args) + f = search(File.method(:open), *args) + if block_given? + begin + yield f + ensure + f.close unless f.closed? + end + else + f + end + end + + def read(*args) + open(*args) {|f| f.read} + end + + def foreach(file, *args, &block) + open(file) {|f| f.each(*args, &block)} + end + + def def_options(opt) + opt.on("-I", "--srcdir=DIR", "add a directory to search path") {|dir| + @additional << dir + } + opt.on("-L", "--vpath=PATH LIST", "add directories to search path") {|dirs| + @additional << [dirs] + } + opt.on("--path-separator=SEP", /\A(?:\W\z|\.(\W).+)/, "separator for vpath") {|sep, vsep| + # hack for msys make. + @separator = vsep || sep + } + end + + def list + @additional.reject! do |dirs| + case dirs + when String + @list << dirs + when Array + raise "--path-separator option is needed for vpath list" unless @separator + # @separator ||= (require 'rbconfig'; RbConfig::CONFIG["PATH_SEPARATOR"]) + @list.concat(dirs[0].split(@separator)) + end + true + end + @list + end + + def strip(path) + prefix = list.map {|dir| Regexp.quote(dir)} + path.sub(/\A#{prefix.join('|')}(?:\/|\z)/, '') + end +end diff --git a/tool/lib/webrick.rb b/tool/lib/webrick.rb new file mode 100644 index 0000000000..b854b68db4 --- /dev/null +++ b/tool/lib/webrick.rb @@ -0,0 +1,232 @@ +# frozen_string_literal: false +## +# = WEB server toolkit. +# +# WEBrick is an HTTP server toolkit that can be configured as an HTTPS server, +# a proxy server, and a virtual-host server. WEBrick features complete +# logging of both server operations and HTTP access. WEBrick supports both +# basic and digest authentication in addition to algorithms not in RFC 2617. +# +# A WEBrick server can be composed of multiple WEBrick servers or servlets to +# provide differing behavior on a per-host or per-path basis. WEBrick +# includes servlets for handling CGI scripts, ERB pages, Ruby blocks and +# directory listings. +# +# WEBrick also includes tools for daemonizing a process and starting a process +# at a higher privilege level and dropping permissions. +# +# == Security +# +# *Warning:* WEBrick is not recommended for production. It only implements +# basic security checks. +# +# == Starting an HTTP server +# +# To create a new WEBrick::HTTPServer that will listen to connections on port +# 8000 and serve documents from the current user's public_html folder: +# +# require 'webrick' +# +# root = File.expand_path '~/public_html' +# server = WEBrick::HTTPServer.new :Port => 8000, :DocumentRoot => root +# +# To run the server you will need to provide a suitable shutdown hook as +# starting the server blocks the current thread: +# +# trap 'INT' do server.shutdown end +# +# server.start +# +# == Custom Behavior +# +# The easiest way to have a server perform custom operations is through +# WEBrick::HTTPServer#mount_proc. The block given will be called with a +# WEBrick::HTTPRequest with request info and a WEBrick::HTTPResponse which +# must be filled in appropriately: +# +# server.mount_proc '/' do |req, res| +# res.body = 'Hello, world!' +# end +# +# Remember that +server.mount_proc+ must precede +server.start+. +# +# == Servlets +# +# Advanced custom behavior can be obtained through mounting a subclass of +# WEBrick::HTTPServlet::AbstractServlet. Servlets provide more modularity +# when writing an HTTP server than mount_proc allows. Here is a simple +# servlet: +# +# class Simple < WEBrick::HTTPServlet::AbstractServlet +# def do_GET request, response +# status, content_type, body = do_stuff_with request +# +# response.status = 200 +# response['Content-Type'] = 'text/plain' +# response.body = 'Hello, World!' +# end +# end +# +# To initialize the servlet you mount it on the server: +# +# server.mount '/simple', Simple +# +# See WEBrick::HTTPServlet::AbstractServlet for more details. +# +# == Virtual Hosts +# +# A server can act as a virtual host for multiple host names. After creating +# the listening host, additional hosts that do not listen can be created and +# attached as virtual hosts: +# +# server = WEBrick::HTTPServer.new # ... +# +# vhost = WEBrick::HTTPServer.new :ServerName => 'vhost.example', +# :DoNotListen => true, # ... +# vhost.mount '/', ... +# +# server.virtual_host vhost +# +# If no +:DocumentRoot+ is provided and no servlets or procs are mounted on the +# main server it will return 404 for all URLs. +# +# == HTTPS +# +# To create an HTTPS server you only need to enable SSL and provide an SSL +# certificate name: +# +# require 'webrick' +# require 'webrick/https' +# +# cert_name = [ +# %w[CN localhost], +# ] +# +# server = WEBrick::HTTPServer.new(:Port => 8000, +# :SSLEnable => true, +# :SSLCertName => cert_name) +# +# This will start the server with a self-generated self-signed certificate. +# The certificate will be changed every time the server is restarted. +# +# To create a server with a pre-determined key and certificate you can provide +# them: +# +# require 'webrick' +# require 'webrick/https' +# require 'openssl' +# +# cert = OpenSSL::X509::Certificate.new File.read '/path/to/cert.pem' +# pkey = OpenSSL::PKey::RSA.new File.read '/path/to/pkey.pem' +# +# server = WEBrick::HTTPServer.new(:Port => 8000, +# :SSLEnable => true, +# :SSLCertificate => cert, +# :SSLPrivateKey => pkey) +# +# == Proxy Server +# +# WEBrick can act as a proxy server: +# +# require 'webrick' +# require 'webrick/httpproxy' +# +# proxy = WEBrick::HTTPProxyServer.new :Port => 8000 +# +# trap 'INT' do proxy.shutdown end +# +# See WEBrick::HTTPProxy for further details including modifying proxied +# responses. +# +# == Basic and Digest authentication +# +# WEBrick provides both Basic and Digest authentication for regular and proxy +# servers. See WEBrick::HTTPAuth, WEBrick::HTTPAuth::BasicAuth and +# WEBrick::HTTPAuth::DigestAuth. +# +# == WEBrick as a daemonized Web Server +# +# WEBrick can be run as a daemonized server for small loads. +# +# === Daemonizing +# +# To start a WEBrick server as a daemon simple run WEBrick::Daemon.start +# before starting the server. +# +# === Dropping Permissions +# +# WEBrick can be started as one user to gain permission to bind to port 80 or +# 443 for serving HTTP or HTTPS traffic then can drop these permissions for +# regular operation. To listen on all interfaces for HTTP traffic: +# +# sockets = WEBrick::Utils.create_listeners nil, 80 +# +# Then drop privileges: +# +# WEBrick::Utils.su 'www' +# +# Then create a server that does not listen by default: +# +# server = WEBrick::HTTPServer.new :DoNotListen => true, # ... +# +# Then overwrite the listening sockets with the port 80 sockets: +# +# server.listeners.replace sockets +# +# === Logging +# +# WEBrick can separately log server operations and end-user access. For +# server operations: +# +# log_file = File.open '/var/log/webrick.log', 'a+' +# log = WEBrick::Log.new log_file +# +# For user access logging: +# +# access_log = [ +# [log_file, WEBrick::AccessLog::COMBINED_LOG_FORMAT], +# ] +# +# server = WEBrick::HTTPServer.new :Logger => log, :AccessLog => access_log +# +# See WEBrick::AccessLog for further log formats. +# +# === Log Rotation +# +# To rotate logs in WEBrick on a HUP signal (like syslogd can send), open the +# log file in 'a+' mode (as above) and trap 'HUP' to reopen the log file: +# +# trap 'HUP' do log_file.reopen '/path/to/webrick.log', 'a+' +# +# == Copyright +# +# Author: IPR -- Internet Programming with Ruby -- writers +# +# Copyright (c) 2000 TAKAHASHI Masayoshi, GOTOU YUUZOU +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +#-- +# $IPR: webrick.rb,v 1.12 2002/10/01 17:16:31 gotoyuzo Exp $ + +module WEBrick +end + +require 'webrick/compat.rb' + +require 'webrick/version.rb' +require 'webrick/config.rb' +require 'webrick/log.rb' +require 'webrick/server.rb' +require_relative 'webrick/utils.rb' +require 'webrick/accesslog' + +require 'webrick/htmlutils.rb' +require 'webrick/httputils.rb' +require 'webrick/cookie.rb' +require 'webrick/httpversion.rb' +require 'webrick/httpstatus.rb' +require 'webrick/httprequest.rb' +require 'webrick/httpresponse.rb' +require 'webrick/httpserver.rb' +require 'webrick/httpservlet.rb' +require 'webrick/httpauth.rb' diff --git a/tool/lib/webrick/.document b/tool/lib/webrick/.document new file mode 100644 index 0000000000..c62f89083b --- /dev/null +++ b/tool/lib/webrick/.document @@ -0,0 +1,6 @@ +# Add files to this as they become documented + +*.rb + +httpauth +httpservlet diff --git a/tool/lib/webrick/accesslog.rb b/tool/lib/webrick/accesslog.rb new file mode 100644 index 0000000000..e4849637f3 --- /dev/null +++ b/tool/lib/webrick/accesslog.rb @@ -0,0 +1,157 @@ +# frozen_string_literal: false +#-- +# accesslog.rb -- Access log handling utilities +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2002 keita yamaguchi +# Copyright (c) 2002 Internet Programming with Ruby writers +# +# $IPR: accesslog.rb,v 1.1 2002/10/01 17:16:32 gotoyuzo Exp $ + +module WEBrick + + ## + # AccessLog provides logging to various files in various formats. + # + # Multiple logs may be written to at the same time: + # + # access_log = [ + # [$stderr, WEBrick::AccessLog::COMMON_LOG_FORMAT], + # [$stderr, WEBrick::AccessLog::REFERER_LOG_FORMAT], + # ] + # + # server = WEBrick::HTTPServer.new :AccessLog => access_log + # + # Custom log formats may be defined. WEBrick::AccessLog provides a subset + # of the formatting from Apache's mod_log_config + # http://httpd.apache.org/docs/mod/mod_log_config.html#formats. See + # AccessLog::setup_params for a list of supported options + + module AccessLog + + ## + # Raised if a parameter such as %e, %i, %o or %n is used without fetching + # a specific field. + + class AccessLogError < StandardError; end + + ## + # The Common Log Format's time format + + CLF_TIME_FORMAT = "[%d/%b/%Y:%H:%M:%S %Z]" + + ## + # Common Log Format + + COMMON_LOG_FORMAT = "%h %l %u %t \"%r\" %s %b" + + ## + # Short alias for Common Log Format + + CLF = COMMON_LOG_FORMAT + + ## + # Referer Log Format + + REFERER_LOG_FORMAT = "%{Referer}i -> %U" + + ## + # User-Agent Log Format + + AGENT_LOG_FORMAT = "%{User-Agent}i" + + ## + # Combined Log Format + + COMBINED_LOG_FORMAT = "#{CLF} \"%{Referer}i\" \"%{User-agent}i\"" + + module_function + + # This format specification is a subset of mod_log_config of Apache: + # + # %a:: Remote IP address + # %b:: Total response size + # %e{variable}:: Given variable in ENV + # %f:: Response filename + # %h:: Remote host name + # %{header}i:: Given request header + # %l:: Remote logname, always "-" + # %m:: Request method + # %{attr}n:: Given request attribute from <tt>req.attributes</tt> + # %{header}o:: Given response header + # %p:: Server's request port + # %{format}p:: The canonical port of the server serving the request or the + # actual port or the client's actual port. Valid formats are + # canonical, local or remote. + # %q:: Request query string + # %r:: First line of the request + # %s:: Request status + # %t:: Time the request was received + # %T:: Time taken to process the request + # %u:: Remote user from auth + # %U:: Unparsed URI + # %%:: Literal % + + def setup_params(config, req, res) + params = Hash.new("") + params["a"] = req.peeraddr[3] + params["b"] = res.sent_size + params["e"] = ENV + params["f"] = res.filename || "" + params["h"] = req.peeraddr[2] + params["i"] = req + params["l"] = "-" + params["m"] = req.request_method + params["n"] = req.attributes + params["o"] = res + params["p"] = req.port + params["q"] = req.query_string + params["r"] = req.request_line.sub(/\x0d?\x0a\z/o, '') + params["s"] = res.status # won't support "%>s" + params["t"] = req.request_time + params["T"] = Time.now - req.request_time + params["u"] = req.user || "-" + params["U"] = req.unparsed_uri + params["v"] = config[:ServerName] + params + end + + ## + # Formats +params+ according to +format_string+ which is described in + # setup_params. + + def format(format_string, params) + format_string.gsub(/\%(?:\{(.*?)\})?>?([a-zA-Z%])/){ + param, spec = $1, $2 + case spec[0] + when ?e, ?i, ?n, ?o + raise AccessLogError, + "parameter is required for \"#{spec}\"" unless param + (param = params[spec][param]) ? escape(param) : "-" + when ?t + params[spec].strftime(param || CLF_TIME_FORMAT) + when ?p + case param + when 'remote' + escape(params["i"].peeraddr[1].to_s) + else + escape(params["p"].to_s) + end + when ?% + "%" + else + escape(params[spec].to_s) + end + } + end + + ## + # Escapes control characters in +data+ + + def escape(data) + data = data.gsub(/[[:cntrl:]\\]+/) {$&.dump[1...-1]} + data.untaint if RUBY_VERSION < '2.7' + data + end + end +end diff --git a/tool/lib/webrick/cgi.rb b/tool/lib/webrick/cgi.rb new file mode 100644 index 0000000000..bb0ae2fc84 --- /dev/null +++ b/tool/lib/webrick/cgi.rb @@ -0,0 +1,313 @@ +# frozen_string_literal: false +# +# cgi.rb -- Yet another CGI library +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2003 Internet Programming with Ruby writers. All rights +# reserved. +# +# $Id$ + +require_relative "httprequest" +require_relative "httpresponse" +require_relative "config" +require "stringio" + +module WEBrick + + # A CGI library using WEBrick requests and responses. + # + # Example: + # + # class MyCGI < WEBrick::CGI + # def do_GET req, res + # res.body = 'it worked!' + # res.status = 200 + # end + # end + # + # MyCGI.new.start + + class CGI + + # The CGI error exception class + + CGIError = Class.new(StandardError) + + ## + # The CGI configuration. This is based on WEBrick::Config::HTTP + + attr_reader :config + + ## + # The CGI logger + + attr_reader :logger + + ## + # Creates a new CGI interface. + # + # The first argument in +args+ is a configuration hash which would update + # WEBrick::Config::HTTP. + # + # Any remaining arguments are stored in the <code>@options</code> instance + # variable for use by a subclass. + + def initialize(*args) + if defined?(MOD_RUBY) + unless ENV.has_key?("GATEWAY_INTERFACE") + Apache.request.setup_cgi_env + end + end + if %r{HTTP/(\d+\.\d+)} =~ ENV["SERVER_PROTOCOL"] + httpv = $1 + end + @config = WEBrick::Config::HTTP.dup.update( + :ServerSoftware => ENV["SERVER_SOFTWARE"] || "null", + :HTTPVersion => HTTPVersion.new(httpv || "1.0"), + :RunOnCGI => true, # to detect if it runs on CGI. + :NPH => false # set true to run as NPH script. + ) + if config = args.shift + @config.update(config) + end + @config[:Logger] ||= WEBrick::BasicLog.new($stderr) + @logger = @config[:Logger] + @options = args + end + + ## + # Reads +key+ from the configuration + + def [](key) + @config[key] + end + + ## + # Starts the CGI process with the given environment +env+ and standard + # input and output +stdin+ and +stdout+. + + def start(env=ENV, stdin=$stdin, stdout=$stdout) + sock = WEBrick::CGI::Socket.new(@config, env, stdin, stdout) + req = HTTPRequest.new(@config) + res = HTTPResponse.new(@config) + unless @config[:NPH] or defined?(MOD_RUBY) + def res.setup_header + unless @header["status"] + phrase = HTTPStatus::reason_phrase(@status) + @header["status"] = "#{@status} #{phrase}" + end + super + end + def res.status_line + "" + end + end + + begin + req.parse(sock) + req.script_name = (env["SCRIPT_NAME"] || File.expand_path($0)).dup + req.path_info = (env["PATH_INFO"] || "").dup + req.query_string = env["QUERY_STRING"] + req.user = env["REMOTE_USER"] + res.request_method = req.request_method + res.request_uri = req.request_uri + res.request_http_version = req.http_version + res.keep_alive = req.keep_alive? + self.service(req, res) + rescue HTTPStatus::Error => ex + res.set_error(ex) + rescue HTTPStatus::Status => ex + res.status = ex.code + rescue Exception => ex + @logger.error(ex) + res.set_error(ex, true) + ensure + req.fixup + if defined?(MOD_RUBY) + res.setup_header + Apache.request.status_line = "#{res.status} #{res.reason_phrase}" + Apache.request.status = res.status + table = Apache.request.headers_out + res.header.each{|key, val| + case key + when /^content-encoding$/i + Apache::request.content_encoding = val + when /^content-type$/i + Apache::request.content_type = val + else + table[key] = val.to_s + end + } + res.cookies.each{|cookie| + table.add("Set-Cookie", cookie.to_s) + } + Apache.request.send_http_header + res.send_body(sock) + else + res.send_response(sock) + end + end + end + + ## + # Services the request +req+ which will fill in the response +res+. See + # WEBrick::HTTPServlet::AbstractServlet#service for details. + + def service(req, res) + method_name = "do_" + req.request_method.gsub(/-/, "_") + if respond_to?(method_name) + __send__(method_name, req, res) + else + raise HTTPStatus::MethodNotAllowed, + "unsupported method `#{req.request_method}'." + end + end + + ## + # Provides HTTP socket emulation from the CGI environment + + class Socket # :nodoc: + include Enumerable + + private + + def initialize(config, env, stdin, stdout) + @config = config + @env = env + @header_part = StringIO.new + @body_part = stdin + @out_port = stdout + @out_port.binmode + + @server_addr = @env["SERVER_ADDR"] || "0.0.0.0" + @server_name = @env["SERVER_NAME"] + @server_port = @env["SERVER_PORT"] + @remote_addr = @env["REMOTE_ADDR"] + @remote_host = @env["REMOTE_HOST"] || @remote_addr + @remote_port = @env["REMOTE_PORT"] || 0 + + begin + @header_part << request_line << CRLF + setup_header + @header_part << CRLF + @header_part.rewind + rescue Exception + raise CGIError, "invalid CGI environment" + end + end + + def request_line + meth = @env["REQUEST_METHOD"] || "GET" + unless url = @env["REQUEST_URI"] + url = (@env["SCRIPT_NAME"] || File.expand_path($0)).dup + url << @env["PATH_INFO"].to_s + url = WEBrick::HTTPUtils.escape_path(url) + if query_string = @env["QUERY_STRING"] + unless query_string.empty? + url << "?" << query_string + end + end + end + # we cannot get real HTTP version of client ;) + httpv = @config[:HTTPVersion] + return "#{meth} #{url} HTTP/#{httpv}" + end + + def setup_header + @env.each{|key, value| + case key + when "CONTENT_TYPE", "CONTENT_LENGTH" + add_header(key.gsub(/_/, "-"), value) + when /^HTTP_(.*)/ + add_header($1.gsub(/_/, "-"), value) + end + } + end + + def add_header(hdrname, value) + unless value.empty? + @header_part << hdrname << ": " << value << CRLF + end + end + + def input + @header_part.eof? ? @body_part : @header_part + end + + public + + def peeraddr + [nil, @remote_port, @remote_host, @remote_addr] + end + + def addr + [nil, @server_port, @server_name, @server_addr] + end + + def gets(eol=LF, size=nil) + input.gets(eol, size) + end + + def read(size=nil) + input.read(size) + end + + def each + input.each{|line| yield(line) } + end + + def eof? + input.eof? + end + + def <<(data) + @out_port << data + end + + def write(data) + @out_port.write(data) + end + + def cert + return nil unless defined?(OpenSSL) + if pem = @env["SSL_SERVER_CERT"] + OpenSSL::X509::Certificate.new(pem) unless pem.empty? + end + end + + def peer_cert + return nil unless defined?(OpenSSL) + if pem = @env["SSL_CLIENT_CERT"] + OpenSSL::X509::Certificate.new(pem) unless pem.empty? + end + end + + def peer_cert_chain + return nil unless defined?(OpenSSL) + if @env["SSL_CLIENT_CERT_CHAIN_0"] + keys = @env.keys + certs = keys.sort.collect{|k| + if /^SSL_CLIENT_CERT_CHAIN_\d+$/ =~ k + if pem = @env[k] + OpenSSL::X509::Certificate.new(pem) unless pem.empty? + end + end + } + certs.compact + end + end + + def cipher + return nil unless defined?(OpenSSL) + if cipher = @env["SSL_CIPHER"] + ret = [ cipher ] + ret << @env["SSL_PROTOCOL"] + ret << @env["SSL_CIPHER_USEKEYSIZE"] + ret << @env["SSL_CIPHER_ALGKEYSIZE"] + ret + end + end + end + end +end diff --git a/tool/lib/webrick/compat.rb b/tool/lib/webrick/compat.rb new file mode 100644 index 0000000000..c497a1933c --- /dev/null +++ b/tool/lib/webrick/compat.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: false +# +# compat.rb -- cross platform compatibility +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2002 GOTOU Yuuzou +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: compat.rb,v 1.6 2002/10/01 17:16:32 gotoyuzo Exp $ + +## +# System call error module used by webrick for cross platform compatibility. +# +# EPROTO:: protocol error +# ECONNRESET:: remote host reset the connection request +# ECONNABORTED:: Client sent TCP reset (RST) before server has accepted the +# connection requested by client. +# +module Errno + ## + # Protocol error. + + class EPROTO < SystemCallError; end + + ## + # Remote host reset the connection request. + + class ECONNRESET < SystemCallError; end + + ## + # Client sent TCP reset (RST) before server has accepted the connection + # requested by client. + + class ECONNABORTED < SystemCallError; end +end diff --git a/tool/lib/webrick/config.rb b/tool/lib/webrick/config.rb new file mode 100644 index 0000000000..9f2ab44f49 --- /dev/null +++ b/tool/lib/webrick/config.rb @@ -0,0 +1,158 @@ +# frozen_string_literal: false +# +# config.rb -- Default configurations. +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou +# Copyright (c) 2003 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: config.rb,v 1.52 2003/07/22 19:20:42 gotoyuzo Exp $ + +require_relative 'version' +require_relative 'httpversion' +require_relative 'httputils' +require_relative 'utils' +require_relative 'log' + +module WEBrick + module Config + LIBDIR = File::dirname(__FILE__) # :nodoc: + + # for GenericServer + General = Hash.new { |hash, key| + case key + when :ServerName + hash[key] = Utils.getservername + else + nil + end + }.update( + :BindAddress => nil, # "0.0.0.0" or "::" or nil + :Port => nil, # users MUST specify this!! + :MaxClients => 100, # maximum number of the concurrent connections + :ServerType => nil, # default: WEBrick::SimpleServer + :Logger => nil, # default: WEBrick::Log.new + :ServerSoftware => "WEBrick/#{WEBrick::VERSION} " + + "(Ruby/#{RUBY_VERSION}/#{RUBY_RELEASE_DATE})", + :TempDir => ENV['TMPDIR']||ENV['TMP']||ENV['TEMP']||'/tmp', + :DoNotListen => false, + :StartCallback => nil, + :StopCallback => nil, + :AcceptCallback => nil, + :DoNotReverseLookup => true, + :ShutdownSocketWithoutClose => false, + ) + + # for HTTPServer, HTTPRequest, HTTPResponse ... + HTTP = General.dup.update( + :Port => 80, + :RequestTimeout => 30, + :HTTPVersion => HTTPVersion.new("1.1"), + :AccessLog => nil, + :MimeTypes => HTTPUtils::DefaultMimeTypes, + :DirectoryIndex => ["index.html","index.htm","index.cgi","index.rhtml"], + :DocumentRoot => nil, + :DocumentRootOptions => { :FancyIndexing => true }, + :RequestCallback => nil, + :ServerAlias => nil, + :InputBufferSize => 65536, # input buffer size in reading request body + :OutputBufferSize => 65536, # output buffer size in sending File or IO + + # for HTTPProxyServer + :ProxyAuthProc => nil, + :ProxyContentHandler => nil, + :ProxyVia => true, + :ProxyTimeout => true, + :ProxyURI => nil, + + :CGIInterpreter => nil, + :CGIPathEnv => nil, + + # workaround: if Request-URIs contain 8bit chars, + # they should be escaped before calling of URI::parse(). + :Escape8bitURI => false + ) + + ## + # Default configuration for WEBrick::HTTPServlet::FileHandler + # + # :AcceptableLanguages:: + # Array of languages allowed for accept-language. There is no default + # :DirectoryCallback:: + # Allows preprocessing of directory requests. There is no default + # callback. + # :FancyIndexing:: + # If true, show an index for directories. The default is true. + # :FileCallback:: + # Allows preprocessing of file requests. There is no default callback. + # :HandlerCallback:: + # Allows preprocessing of requests. There is no default callback. + # :HandlerTable:: + # Maps file suffixes to file handlers. DefaultFileHandler is used by + # default but any servlet can be used. + # :NondisclosureName:: + # Do not show files matching this array of globs. .ht* and *~ are + # excluded by default. + # :UserDir:: + # Directory inside ~user to serve content from for /~user requests. + # Only works if mounted on /. Disabled by default. + + FileHandler = { + :NondisclosureName => [".ht*", "*~"], + :FancyIndexing => false, + :HandlerTable => {}, + :HandlerCallback => nil, + :DirectoryCallback => nil, + :FileCallback => nil, + :UserDir => nil, # e.g. "public_html" + :AcceptableLanguages => [] # ["en", "ja", ... ] + } + + ## + # Default configuration for WEBrick::HTTPAuth::BasicAuth + # + # :AutoReloadUserDB:: Reload the user database provided by :UserDB + # automatically? + + BasicAuth = { + :AutoReloadUserDB => true, + } + + ## + # Default configuration for WEBrick::HTTPAuth::DigestAuth. + # + # :Algorithm:: MD5, MD5-sess (default), SHA1, SHA1-sess + # :Domain:: An Array of URIs that define the protected space + # :Qop:: 'auth' for authentication, 'auth-int' for integrity protection or + # both + # :UseOpaque:: Should the server send opaque values to the client? This + # helps prevent replay attacks. + # :CheckNc:: Should the server check the nonce count? This helps the + # server detect replay attacks. + # :UseAuthenticationInfoHeader:: Should the server send an + # AuthenticationInfo header? + # :AutoReloadUserDB:: Reload the user database provided by :UserDB + # automatically? + # :NonceExpirePeriod:: How long should we store used nonces? Default is + # 30 minutes. + # :NonceExpireDelta:: How long is a nonce valid? Default is 1 minute + # :InternetExplorerHack:: Hack which allows Internet Explorer to work. + # :OperaHack:: Hack which allows Opera to work. + + DigestAuth = { + :Algorithm => 'MD5-sess', # or 'MD5' + :Domain => nil, # an array includes domain names. + :Qop => [ 'auth' ], # 'auth' or 'auth-int' or both. + :UseOpaque => true, + :UseNextNonce => false, + :CheckNc => false, + :UseAuthenticationInfoHeader => true, + :AutoReloadUserDB => true, + :NonceExpirePeriod => 30*60, + :NonceExpireDelta => 60, + :InternetExplorerHack => true, + :OperaHack => true, + } + end +end diff --git a/tool/lib/webrick/cookie.rb b/tool/lib/webrick/cookie.rb new file mode 100644 index 0000000000..5fd3bfb228 --- /dev/null +++ b/tool/lib/webrick/cookie.rb @@ -0,0 +1,172 @@ +# frozen_string_literal: false +# +# cookie.rb -- Cookie class +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: cookie.rb,v 1.16 2002/09/21 12:23:35 gotoyuzo Exp $ + +require 'time' +require_relative 'httputils' + +module WEBrick + + ## + # Processes HTTP cookies + + class Cookie + + ## + # The cookie name + + attr_reader :name + + ## + # The cookie value + + attr_accessor :value + + ## + # The cookie version + + attr_accessor :version + + ## + # The cookie domain + attr_accessor :domain + + ## + # The cookie path + + attr_accessor :path + + ## + # Is this a secure cookie? + + attr_accessor :secure + + ## + # The cookie comment + + attr_accessor :comment + + ## + # The maximum age of the cookie + + attr_accessor :max_age + + #attr_accessor :comment_url, :discard, :port + + ## + # Creates a new cookie with the given +name+ and +value+ + + def initialize(name, value) + @name = name + @value = value + @version = 0 # Netscape Cookie + + @domain = @path = @secure = @comment = @max_age = + @expires = @comment_url = @discard = @port = nil + end + + ## + # Sets the cookie expiration to the time +t+. The expiration time may be + # a false value to disable expiration or a Time or HTTP format time string + # to set the expiration date. + + def expires=(t) + @expires = t && (t.is_a?(Time) ? t.httpdate : t.to_s) + end + + ## + # Retrieves the expiration time as a Time + + def expires + @expires && Time.parse(@expires) + end + + ## + # The cookie string suitable for use in an HTTP header + + def to_s + ret = "" + ret << @name << "=" << @value + ret << "; " << "Version=" << @version.to_s if @version > 0 + ret << "; " << "Domain=" << @domain if @domain + ret << "; " << "Expires=" << @expires if @expires + ret << "; " << "Max-Age=" << @max_age.to_s if @max_age + ret << "; " << "Comment=" << @comment if @comment + ret << "; " << "Path=" << @path if @path + ret << "; " << "Secure" if @secure + ret + end + + ## + # Parses a Cookie field sent from the user-agent. Returns an array of + # cookies. + + def self.parse(str) + if str + ret = [] + cookie = nil + ver = 0 + str.split(/;\s+/).each{|x| + key, val = x.split(/=/,2) + val = val ? HTTPUtils::dequote(val) : "" + case key + when "$Version"; ver = val.to_i + when "$Path"; cookie.path = val + when "$Domain"; cookie.domain = val + when "$Port"; cookie.port = val + else + ret << cookie if cookie + cookie = self.new(key, val) + cookie.version = ver + end + } + ret << cookie if cookie + ret + end + end + + ## + # Parses the cookie in +str+ + + def self.parse_set_cookie(str) + cookie_elem = str.split(/;/) + first_elem = cookie_elem.shift + first_elem.strip! + key, value = first_elem.split(/=/, 2) + cookie = new(key, HTTPUtils.dequote(value)) + cookie_elem.each{|pair| + pair.strip! + key, value = pair.split(/=/, 2) + if value + value = HTTPUtils.dequote(value.strip) + end + case key.downcase + when "domain" then cookie.domain = value + when "path" then cookie.path = value + when "expires" then cookie.expires = value + when "max-age" then cookie.max_age = Integer(value) + when "comment" then cookie.comment = value + when "version" then cookie.version = Integer(value) + when "secure" then cookie.secure = true + end + } + return cookie + end + + ## + # Parses the cookies in +str+ + + def self.parse_set_cookies(str) + return str.split(/,(?=[^;,]*=)|,$/).collect{|c| + parse_set_cookie(c) + } + end + end +end diff --git a/tool/lib/webrick/htmlutils.rb b/tool/lib/webrick/htmlutils.rb new file mode 100644 index 0000000000..ed9f4ac0d3 --- /dev/null +++ b/tool/lib/webrick/htmlutils.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: false +#-- +# htmlutils.rb -- HTMLUtils Module +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: htmlutils.rb,v 1.7 2002/09/21 12:23:35 gotoyuzo Exp $ + +module WEBrick + module HTMLUtils + + ## + # Escapes &, ", > and < in +string+ + + def escape(string) + return "" unless string + str = string.b + str.gsub!(/&/n, '&') + str.gsub!(/\"/n, '"') + str.gsub!(/>/n, '>') + str.gsub!(/</n, '<') + str.force_encoding(string.encoding) + end + module_function :escape + + end +end diff --git a/tool/lib/webrick/httpauth.rb b/tool/lib/webrick/httpauth.rb new file mode 100644 index 0000000000..f8bf09a6f1 --- /dev/null +++ b/tool/lib/webrick/httpauth.rb @@ -0,0 +1,96 @@ +# frozen_string_literal: false +# +# httpauth.rb -- HTTP access authentication +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: httpauth.rb,v 1.14 2003/07/22 19:20:42 gotoyuzo Exp $ + +require_relative 'httpauth/basicauth' +require_relative 'httpauth/digestauth' +require_relative 'httpauth/htpasswd' +require_relative 'httpauth/htdigest' +require_relative 'httpauth/htgroup' + +module WEBrick + + ## + # HTTPAuth provides both basic and digest authentication. + # + # To enable authentication for requests in WEBrick you will need a user + # database and an authenticator. To start, here's an Htpasswd database for + # use with a DigestAuth authenticator: + # + # config = { :Realm => 'DigestAuth example realm' } + # + # htpasswd = WEBrick::HTTPAuth::Htpasswd.new 'my_password_file' + # htpasswd.auth_type = WEBrick::HTTPAuth::DigestAuth + # htpasswd.set_passwd config[:Realm], 'username', 'password' + # htpasswd.flush + # + # The +:Realm+ is used to provide different access to different groups + # across several resources on a server. Typically you'll need only one + # realm for a server. + # + # This database can be used to create an authenticator: + # + # config[:UserDB] = htpasswd + # + # digest_auth = WEBrick::HTTPAuth::DigestAuth.new config + # + # To authenticate a request call #authenticate with a request and response + # object in a servlet: + # + # def do_GET req, res + # @authenticator.authenticate req, res + # end + # + # For digest authentication the authenticator must not be created every + # request, it must be passed in as an option via WEBrick::HTTPServer#mount. + + module HTTPAuth + module_function + + def _basic_auth(req, res, realm, req_field, res_field, err_type, + block) # :nodoc: + user = pass = nil + if /^Basic\s+(.*)/o =~ req[req_field] + userpass = $1 + user, pass = userpass.unpack("m*")[0].split(":", 2) + end + if block.call(user, pass) + req.user = user + return + end + res[res_field] = "Basic realm=\"#{realm}\"" + raise err_type + end + + ## + # Simple wrapper for providing basic authentication for a request. When + # called with a request +req+, response +res+, authentication +realm+ and + # +block+ the block will be called with a +username+ and +password+. If + # the block returns true the request is allowed to continue, otherwise an + # HTTPStatus::Unauthorized error is raised. + + def basic_auth(req, res, realm, &block) # :yield: username, password + _basic_auth(req, res, realm, "Authorization", "WWW-Authenticate", + HTTPStatus::Unauthorized, block) + end + + ## + # Simple wrapper for providing basic authentication for a proxied request. + # When called with a request +req+, response +res+, authentication +realm+ + # and +block+ the block will be called with a +username+ and +password+. + # If the block returns true the request is allowed to continue, otherwise + # an HTTPStatus::ProxyAuthenticationRequired error is raised. + + def proxy_basic_auth(req, res, realm, &block) # :yield: username, password + _basic_auth(req, res, realm, "Proxy-Authorization", "Proxy-Authenticate", + HTTPStatus::ProxyAuthenticationRequired, block) + end + end +end diff --git a/tool/lib/webrick/httpauth/authenticator.rb b/tool/lib/webrick/httpauth/authenticator.rb new file mode 100644 index 0000000000..8f0eaa3aca --- /dev/null +++ b/tool/lib/webrick/httpauth/authenticator.rb @@ -0,0 +1,117 @@ +# frozen_string_literal: false +#-- +# httpauth/authenticator.rb -- Authenticator mix-in module. +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2003 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: authenticator.rb,v 1.3 2003/02/20 07:15:47 gotoyuzo Exp $ + +module WEBrick + module HTTPAuth + + ## + # Module providing generic support for both Digest and Basic + # authentication schemes. + + module Authenticator + + RequestField = "Authorization" # :nodoc: + ResponseField = "WWW-Authenticate" # :nodoc: + ResponseInfoField = "Authentication-Info" # :nodoc: + AuthException = HTTPStatus::Unauthorized # :nodoc: + + ## + # Method of authentication, must be overridden by the including class + + AuthScheme = nil + + ## + # The realm this authenticator covers + + attr_reader :realm + + ## + # The user database for this authenticator + + attr_reader :userdb + + ## + # The logger for this authenticator + + attr_reader :logger + + private + + # :stopdoc: + + ## + # Initializes the authenticator from +config+ + + def check_init(config) + [:UserDB, :Realm].each{|sym| + unless config[sym] + raise ArgumentError, "Argument #{sym.inspect} missing." + end + } + @realm = config[:Realm] + @userdb = config[:UserDB] + @logger = config[:Logger] || Log::new($stderr) + @reload_db = config[:AutoReloadUserDB] + @request_field = self::class::RequestField + @response_field = self::class::ResponseField + @resp_info_field = self::class::ResponseInfoField + @auth_exception = self::class::AuthException + @auth_scheme = self::class::AuthScheme + end + + ## + # Ensures +req+ has credentials that can be authenticated. + + def check_scheme(req) + unless credentials = req[@request_field] + error("no credentials in the request.") + return nil + end + unless match = /^#{@auth_scheme}\s+/i.match(credentials) + error("invalid scheme in %s.", credentials) + info("%s: %s", @request_field, credentials) if $DEBUG + return nil + end + return match.post_match + end + + def log(meth, fmt, *args) + msg = format("%s %s: ", @auth_scheme, @realm) + msg << fmt % args + @logger.__send__(meth, msg) + end + + def error(fmt, *args) + if @logger.error? + log(:error, fmt, *args) + end + end + + def info(fmt, *args) + if @logger.info? + log(:info, fmt, *args) + end + end + + # :startdoc: + end + + ## + # Module providing generic support for both Digest and Basic + # authentication schemes for proxies. + + module ProxyAuthenticator + RequestField = "Proxy-Authorization" # :nodoc: + ResponseField = "Proxy-Authenticate" # :nodoc: + InfoField = "Proxy-Authentication-Info" # :nodoc: + AuthException = HTTPStatus::ProxyAuthenticationRequired # :nodoc: + end + end +end diff --git a/tool/lib/webrick/httpauth/basicauth.rb b/tool/lib/webrick/httpauth/basicauth.rb new file mode 100644 index 0000000000..7d0a9cfc8f --- /dev/null +++ b/tool/lib/webrick/httpauth/basicauth.rb @@ -0,0 +1,116 @@ +# frozen_string_literal: false +# +# httpauth/basicauth.rb -- HTTP basic access authentication +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2003 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: basicauth.rb,v 1.5 2003/02/20 07:15:47 gotoyuzo Exp $ + +require_relative '../config' +require_relative '../httpstatus' +require_relative 'authenticator' + +module WEBrick + module HTTPAuth + + ## + # Basic Authentication for WEBrick + # + # Use this class to add basic authentication to a WEBrick servlet. + # + # Here is an example of how to set up a BasicAuth: + # + # config = { :Realm => 'BasicAuth example realm' } + # + # htpasswd = WEBrick::HTTPAuth::Htpasswd.new 'my_password_file', password_hash: :bcrypt + # htpasswd.set_passwd config[:Realm], 'username', 'password' + # htpasswd.flush + # + # config[:UserDB] = htpasswd + # + # basic_auth = WEBrick::HTTPAuth::BasicAuth.new config + + class BasicAuth + include Authenticator + + AuthScheme = "Basic" # :nodoc: + + ## + # Used by UserDB to create a basic password entry + + def self.make_passwd(realm, user, pass) + pass ||= "" + pass.crypt(Utils::random_string(2)) + end + + attr_reader :realm, :userdb, :logger + + ## + # Creates a new BasicAuth instance. + # + # See WEBrick::Config::BasicAuth for default configuration entries + # + # You must supply the following configuration entries: + # + # :Realm:: The name of the realm being protected. + # :UserDB:: A database of usernames and passwords. + # A WEBrick::HTTPAuth::Htpasswd instance should be used. + + def initialize(config, default=Config::BasicAuth) + check_init(config) + @config = default.dup.update(config) + end + + ## + # Authenticates a +req+ and returns a 401 Unauthorized using +res+ if + # the authentication was not correct. + + def authenticate(req, res) + unless basic_credentials = check_scheme(req) + challenge(req, res) + end + userid, password = basic_credentials.unpack("m*")[0].split(":", 2) + password ||= "" + if userid.empty? + error("user id was not given.") + challenge(req, res) + end + unless encpass = @userdb.get_passwd(@realm, userid, @reload_db) + error("%s: the user is not allowed.", userid) + challenge(req, res) + end + + case encpass + when /\A\$2[aby]\$/ + password_matches = BCrypt::Password.new(encpass.sub(/\A\$2[aby]\$/, '$2a$')) == password + else + password_matches = password.crypt(encpass) == encpass + end + + unless password_matches + error("%s: password unmatch.", userid) + challenge(req, res) + end + info("%s: authentication succeeded.", userid) + req.user = userid + end + + ## + # Returns a challenge response which asks for authentication information + + def challenge(req, res) + res[@response_field] = "#{@auth_scheme} realm=\"#{@realm}\"" + raise @auth_exception + end + end + + ## + # Basic authentication for proxy servers. See BasicAuth for details. + + class ProxyBasicAuth < BasicAuth + include ProxyAuthenticator + end + end +end diff --git a/tool/lib/webrick/httpauth/digestauth.rb b/tool/lib/webrick/httpauth/digestauth.rb new file mode 100644 index 0000000000..3cf12899d2 --- /dev/null +++ b/tool/lib/webrick/httpauth/digestauth.rb @@ -0,0 +1,395 @@ +# frozen_string_literal: false +# +# httpauth/digestauth.rb -- HTTP digest access authentication +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2003 Internet Programming with Ruby writers. +# Copyright (c) 2003 H.M. +# +# The original implementation is provided by H.M. +# URL: http://rwiki.jin.gr.jp/cgi-bin/rw-cgi.rb?cmd=view;name= +# %C7%A7%BE%DA%B5%A1%C7%BD%A4%F2%B2%FE%C2%A4%A4%B7%A4%C6%A4%DF%A4%EB +# +# $IPR: digestauth.rb,v 1.5 2003/02/20 07:15:47 gotoyuzo Exp $ + +require_relative '../config' +require_relative '../httpstatus' +require_relative 'authenticator' +require 'digest/md5' +require 'digest/sha1' + +module WEBrick + module HTTPAuth + + ## + # RFC 2617 Digest Access Authentication for WEBrick + # + # Use this class to add digest authentication to a WEBrick servlet. + # + # Here is an example of how to set up DigestAuth: + # + # config = { :Realm => 'DigestAuth example realm' } + # + # htdigest = WEBrick::HTTPAuth::Htdigest.new 'my_password_file' + # htdigest.set_passwd config[:Realm], 'username', 'password' + # htdigest.flush + # + # config[:UserDB] = htdigest + # + # digest_auth = WEBrick::HTTPAuth::DigestAuth.new config + # + # When using this as with a servlet be sure not to create a new DigestAuth + # object in the servlet's #initialize. By default WEBrick creates a new + # servlet instance for every request and the DigestAuth object must be + # used across requests. + + class DigestAuth + include Authenticator + + AuthScheme = "Digest" # :nodoc: + + ## + # Struct containing the opaque portion of the digest authentication + + OpaqueInfo = Struct.new(:time, :nonce, :nc) # :nodoc: + + ## + # Digest authentication algorithm + + attr_reader :algorithm + + ## + # Quality of protection. RFC 2617 defines "auth" and "auth-int" + + attr_reader :qop + + ## + # Used by UserDB to create a digest password entry + + def self.make_passwd(realm, user, pass) + pass ||= "" + Digest::MD5::hexdigest([user, realm, pass].join(":")) + end + + ## + # Creates a new DigestAuth instance. Be sure to use the same DigestAuth + # instance for multiple requests as it saves state between requests in + # order to perform authentication. + # + # See WEBrick::Config::DigestAuth for default configuration entries + # + # You must supply the following configuration entries: + # + # :Realm:: The name of the realm being protected. + # :UserDB:: A database of usernames and passwords. + # A WEBrick::HTTPAuth::Htdigest instance should be used. + + def initialize(config, default=Config::DigestAuth) + check_init(config) + @config = default.dup.update(config) + @algorithm = @config[:Algorithm] + @domain = @config[:Domain] + @qop = @config[:Qop] + @use_opaque = @config[:UseOpaque] + @use_next_nonce = @config[:UseNextNonce] + @check_nc = @config[:CheckNc] + @use_auth_info_header = @config[:UseAuthenticationInfoHeader] + @nonce_expire_period = @config[:NonceExpirePeriod] + @nonce_expire_delta = @config[:NonceExpireDelta] + @internet_explorer_hack = @config[:InternetExplorerHack] + + case @algorithm + when 'MD5','MD5-sess' + @h = Digest::MD5 + when 'SHA1','SHA1-sess' # it is a bonus feature :-) + @h = Digest::SHA1 + else + msg = format('Algorithm "%s" is not supported.', @algorithm) + raise ArgumentError.new(msg) + end + + @instance_key = hexdigest(self.__id__, Time.now.to_i, Process.pid) + @opaques = {} + @last_nonce_expire = Time.now + @mutex = Thread::Mutex.new + end + + ## + # Authenticates a +req+ and returns a 401 Unauthorized using +res+ if + # the authentication was not correct. + + def authenticate(req, res) + unless result = @mutex.synchronize{ _authenticate(req, res) } + challenge(req, res) + end + if result == :nonce_is_stale + challenge(req, res, true) + end + return true + end + + ## + # Returns a challenge response which asks for authentication information + + def challenge(req, res, stale=false) + nonce = generate_next_nonce(req) + if @use_opaque + opaque = generate_opaque(req) + @opaques[opaque].nonce = nonce + end + + param = Hash.new + param["realm"] = HTTPUtils::quote(@realm) + param["domain"] = HTTPUtils::quote(@domain.to_a.join(" ")) if @domain + param["nonce"] = HTTPUtils::quote(nonce) + param["opaque"] = HTTPUtils::quote(opaque) if opaque + param["stale"] = stale.to_s + param["algorithm"] = @algorithm + param["qop"] = HTTPUtils::quote(@qop.to_a.join(",")) if @qop + + res[@response_field] = + "#{@auth_scheme} " + param.map{|k,v| "#{k}=#{v}" }.join(", ") + info("%s: %s", @response_field, res[@response_field]) if $DEBUG + raise @auth_exception + end + + private + + # :stopdoc: + + MustParams = ['username','realm','nonce','uri','response'] + MustParamsAuth = ['cnonce','nc'] + + def _authenticate(req, res) + unless digest_credentials = check_scheme(req) + return false + end + + auth_req = split_param_value(digest_credentials) + if auth_req['qop'] == "auth" || auth_req['qop'] == "auth-int" + req_params = MustParams + MustParamsAuth + else + req_params = MustParams + end + req_params.each{|key| + unless auth_req.has_key?(key) + error('%s: parameter missing. "%s"', auth_req['username'], key) + raise HTTPStatus::BadRequest + end + } + + if !check_uri(req, auth_req) + raise HTTPStatus::BadRequest + end + + if auth_req['realm'] != @realm + error('%s: realm unmatch. "%s" for "%s"', + auth_req['username'], auth_req['realm'], @realm) + return false + end + + auth_req['algorithm'] ||= 'MD5' + if auth_req['algorithm'].upcase != @algorithm.upcase + error('%s: algorithm unmatch. "%s" for "%s"', + auth_req['username'], auth_req['algorithm'], @algorithm) + return false + end + + if (@qop.nil? && auth_req.has_key?('qop')) || + (@qop && (! @qop.member?(auth_req['qop']))) + error('%s: the qop is not allowed. "%s"', + auth_req['username'], auth_req['qop']) + return false + end + + password = @userdb.get_passwd(@realm, auth_req['username'], @reload_db) + unless password + error('%s: the user is not allowed.', auth_req['username']) + return false + end + + nonce_is_invalid = false + if @use_opaque + info("@opaque = %s", @opaque.inspect) if $DEBUG + if !(opaque = auth_req['opaque']) + error('%s: opaque is not given.', auth_req['username']) + nonce_is_invalid = true + elsif !(opaque_struct = @opaques[opaque]) + error('%s: invalid opaque is given.', auth_req['username']) + nonce_is_invalid = true + elsif !check_opaque(opaque_struct, req, auth_req) + @opaques.delete(auth_req['opaque']) + nonce_is_invalid = true + end + elsif !check_nonce(req, auth_req) + nonce_is_invalid = true + end + + if /-sess$/i =~ auth_req['algorithm'] + ha1 = hexdigest(password, auth_req['nonce'], auth_req['cnonce']) + else + ha1 = password + end + + if auth_req['qop'] == "auth" || auth_req['qop'] == nil + ha2 = hexdigest(req.request_method, auth_req['uri']) + ha2_res = hexdigest("", auth_req['uri']) + elsif auth_req['qop'] == "auth-int" + body_digest = @h.new + req.body { |chunk| body_digest.update(chunk) } + body_digest = body_digest.hexdigest + ha2 = hexdigest(req.request_method, auth_req['uri'], body_digest) + ha2_res = hexdigest("", auth_req['uri'], body_digest) + end + + if auth_req['qop'] == "auth" || auth_req['qop'] == "auth-int" + param2 = ['nonce', 'nc', 'cnonce', 'qop'].map{|key| + auth_req[key] + }.join(':') + digest = hexdigest(ha1, param2, ha2) + digest_res = hexdigest(ha1, param2, ha2_res) + else + digest = hexdigest(ha1, auth_req['nonce'], ha2) + digest_res = hexdigest(ha1, auth_req['nonce'], ha2_res) + end + + if digest != auth_req['response'] + error("%s: digest unmatch.", auth_req['username']) + return false + elsif nonce_is_invalid + error('%s: digest is valid, but nonce is not valid.', + auth_req['username']) + return :nonce_is_stale + elsif @use_auth_info_header + auth_info = { + 'nextnonce' => generate_next_nonce(req), + 'rspauth' => digest_res + } + if @use_opaque + opaque_struct.time = req.request_time + opaque_struct.nonce = auth_info['nextnonce'] + opaque_struct.nc = "%08x" % (auth_req['nc'].hex + 1) + end + if auth_req['qop'] == "auth" || auth_req['qop'] == "auth-int" + ['qop','cnonce','nc'].each{|key| + auth_info[key] = auth_req[key] + } + end + res[@resp_info_field] = auth_info.keys.map{|key| + if key == 'nc' + key + '=' + auth_info[key] + else + key + "=" + HTTPUtils::quote(auth_info[key]) + end + }.join(', ') + end + info('%s: authentication succeeded.', auth_req['username']) + req.user = auth_req['username'] + return true + end + + def split_param_value(string) + ret = {} + string.scan(/\G\s*([\w\-.*%!]+)=\s*(?:\"((?>\\.|[^\"])*)\"|([^,\"]*))\s*,?/) do + ret[$1] = $3 || $2.gsub(/\\(.)/, "\\1") + end + ret + end + + def generate_next_nonce(req) + now = "%012d" % req.request_time.to_i + pk = hexdigest(now, @instance_key)[0,32] + nonce = [now + ":" + pk].pack("m0") # it has 60 length of chars. + nonce + end + + def check_nonce(req, auth_req) + username = auth_req['username'] + nonce = auth_req['nonce'] + + pub_time, pk = nonce.unpack("m*")[0].split(":", 2) + if (!pub_time || !pk) + error("%s: empty nonce is given", username) + return false + elsif (hexdigest(pub_time, @instance_key)[0,32] != pk) + error("%s: invalid private-key: %s for %s", + username, hexdigest(pub_time, @instance_key)[0,32], pk) + return false + end + + diff_time = req.request_time.to_i - pub_time.to_i + if (diff_time < 0) + error("%s: difference of time-stamp is negative.", username) + return false + elsif diff_time > @nonce_expire_period + error("%s: nonce is expired.", username) + return false + end + + return true + end + + def generate_opaque(req) + @mutex.synchronize{ + now = req.request_time + if now - @last_nonce_expire > @nonce_expire_delta + @opaques.delete_if{|key,val| + (now - val.time) > @nonce_expire_period + } + @last_nonce_expire = now + end + begin + opaque = Utils::random_string(16) + end while @opaques[opaque] + @opaques[opaque] = OpaqueInfo.new(now, nil, '00000001') + opaque + } + end + + def check_opaque(opaque_struct, req, auth_req) + if (@use_next_nonce && auth_req['nonce'] != opaque_struct.nonce) + error('%s: nonce unmatched. "%s" for "%s"', + auth_req['username'], auth_req['nonce'], opaque_struct.nonce) + return false + elsif !check_nonce(req, auth_req) + return false + end + if (@check_nc && auth_req['nc'] != opaque_struct.nc) + error('%s: nc unmatched."%s" for "%s"', + auth_req['username'], auth_req['nc'], opaque_struct.nc) + return false + end + true + end + + def check_uri(req, auth_req) + uri = auth_req['uri'] + if uri != req.request_uri.to_s && uri != req.unparsed_uri && + (@internet_explorer_hack && uri != req.path) + error('%s: uri unmatch. "%s" for "%s"', auth_req['username'], + auth_req['uri'], req.request_uri.to_s) + return false + end + true + end + + def hexdigest(*args) + @h.hexdigest(args.join(":")) + end + + # :startdoc: + end + + ## + # Digest authentication for proxy servers. See DigestAuth for details. + + class ProxyDigestAuth < DigestAuth + include ProxyAuthenticator + + private + def check_uri(req, auth_req) # :nodoc: + return true + end + end + end +end diff --git a/tool/lib/webrick/httpauth/htdigest.rb b/tool/lib/webrick/httpauth/htdigest.rb new file mode 100644 index 0000000000..93b18e2c75 --- /dev/null +++ b/tool/lib/webrick/httpauth/htdigest.rb @@ -0,0 +1,132 @@ +# frozen_string_literal: false +# +# httpauth/htdigest.rb -- Apache compatible htdigest file +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2003 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: htdigest.rb,v 1.4 2003/07/22 19:20:45 gotoyuzo Exp $ + +require_relative 'userdb' +require_relative 'digestauth' +require 'tempfile' + +module WEBrick + module HTTPAuth + + ## + # Htdigest accesses apache-compatible digest password files. Passwords are + # matched to a realm where they are valid. For security, the path for a + # digest password database should be stored outside of the paths available + # to the HTTP server. + # + # Htdigest is intended for use with WEBrick::HTTPAuth::DigestAuth and + # stores passwords using cryptographic hashes. + # + # htpasswd = WEBrick::HTTPAuth::Htdigest.new 'my_password_file' + # htpasswd.set_passwd 'my realm', 'username', 'password' + # htpasswd.flush + + class Htdigest + include UserDB + + ## + # Open a digest password database at +path+ + + def initialize(path) + @path = path + @mtime = Time.at(0) + @digest = Hash.new + @mutex = Thread::Mutex::new + @auth_type = DigestAuth + File.open(@path,"a").close unless File.exist?(@path) + reload + end + + ## + # Reloads passwords from the database + + def reload + mtime = File::mtime(@path) + if mtime > @mtime + @digest.clear + File.open(@path){|io| + while line = io.gets + line.chomp! + user, realm, pass = line.split(/:/, 3) + unless @digest[realm] + @digest[realm] = Hash.new + end + @digest[realm][user] = pass + end + } + @mtime = mtime + end + end + + ## + # Flush the password database. If +output+ is given the database will + # be written there instead of to the original path. + + def flush(output=nil) + output ||= @path + tmp = Tempfile.create("htpasswd", File::dirname(output)) + renamed = false + begin + each{|item| tmp.puts(item.join(":")) } + tmp.close + File::rename(tmp.path, output) + renamed = true + ensure + tmp.close + File.unlink(tmp.path) if !renamed + end + end + + ## + # Retrieves a password from the database for +user+ in +realm+. If + # +reload_db+ is true the database will be reloaded first. + + def get_passwd(realm, user, reload_db) + reload() if reload_db + if hash = @digest[realm] + hash[user] + end + end + + ## + # Sets a password in the database for +user+ in +realm+ to +pass+. + + def set_passwd(realm, user, pass) + @mutex.synchronize{ + unless @digest[realm] + @digest[realm] = Hash.new + end + @digest[realm][user] = make_passwd(realm, user, pass) + } + end + + ## + # Removes a password from the database for +user+ in +realm+. + + def delete_passwd(realm, user) + if hash = @digest[realm] + hash.delete(user) + end + end + + ## + # Iterate passwords in the database. + + def each # :yields: [user, realm, password_hash] + @digest.keys.sort.each{|realm| + hash = @digest[realm] + hash.keys.sort.each{|user| + yield([user, realm, hash[user]]) + } + } + end + end + end +end diff --git a/tool/lib/webrick/httpauth/htgroup.rb b/tool/lib/webrick/httpauth/htgroup.rb new file mode 100644 index 0000000000..e06c441b18 --- /dev/null +++ b/tool/lib/webrick/httpauth/htgroup.rb @@ -0,0 +1,97 @@ +# frozen_string_literal: false +# +# httpauth/htgroup.rb -- Apache compatible htgroup file +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2003 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: htgroup.rb,v 1.1 2003/02/16 22:22:56 gotoyuzo Exp $ + +require 'tempfile' + +module WEBrick + module HTTPAuth + + ## + # Htgroup accesses apache-compatible group files. Htgroup can be used to + # provide group-based authentication for users. Currently Htgroup is not + # directly integrated with any authenticators in WEBrick. For security, + # the path for a digest password database should be stored outside of the + # paths available to the HTTP server. + # + # Example: + # + # htgroup = WEBrick::HTTPAuth::Htgroup.new 'my_group_file' + # htgroup.add 'superheroes', %w[spiderman batman] + # + # htgroup.members('superheroes').include? 'magneto' # => false + + class Htgroup + + ## + # Open a group database at +path+ + + def initialize(path) + @path = path + @mtime = Time.at(0) + @group = Hash.new + File.open(@path,"a").close unless File.exist?(@path) + reload + end + + ## + # Reload groups from the database + + def reload + if (mtime = File::mtime(@path)) > @mtime + @group.clear + File.open(@path){|io| + while line = io.gets + line.chomp! + group, members = line.split(/:\s*/) + @group[group] = members.split(/\s+/) + end + } + @mtime = mtime + end + end + + ## + # Flush the group database. If +output+ is given the database will be + # written there instead of to the original path. + + def flush(output=nil) + output ||= @path + tmp = Tempfile.create("htgroup", File::dirname(output)) + begin + @group.keys.sort.each{|group| + tmp.puts(format("%s: %s", group, self.members(group).join(" "))) + } + ensure + tmp.close + if $! + File.unlink(tmp.path) + else + return File.rename(tmp.path, output) + end + end + end + + ## + # Retrieve the list of members from +group+ + + def members(group) + reload + @group[group] || [] + end + + ## + # Add an Array of +members+ to +group+ + + def add(group, members) + @group[group] = members(group) | members + end + end + end +end diff --git a/tool/lib/webrick/httpauth/htpasswd.rb b/tool/lib/webrick/httpauth/htpasswd.rb new file mode 100644 index 0000000000..abca30532e --- /dev/null +++ b/tool/lib/webrick/httpauth/htpasswd.rb @@ -0,0 +1,158 @@ +# frozen_string_literal: false +# +# httpauth/htpasswd -- Apache compatible htpasswd file +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2003 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: htpasswd.rb,v 1.4 2003/07/22 19:20:45 gotoyuzo Exp $ + +require_relative 'userdb' +require_relative 'basicauth' +require 'tempfile' + +module WEBrick + module HTTPAuth + + ## + # Htpasswd accesses apache-compatible password files. Passwords are + # matched to a realm where they are valid. For security, the path for a + # password database should be stored outside of the paths available to the + # HTTP server. + # + # Htpasswd is intended for use with WEBrick::HTTPAuth::BasicAuth. + # + # To create an Htpasswd database with a single user: + # + # htpasswd = WEBrick::HTTPAuth::Htpasswd.new 'my_password_file' + # htpasswd.set_passwd 'my realm', 'username', 'password' + # htpasswd.flush + + class Htpasswd + include UserDB + + ## + # Open a password database at +path+ + + def initialize(path, password_hash: nil) + @path = path + @mtime = Time.at(0) + @passwd = Hash.new + @auth_type = BasicAuth + @password_hash = password_hash + + case @password_hash + when nil + # begin + # require "string/crypt" + # rescue LoadError + # warn("Unable to load string/crypt, proceeding with deprecated use of String#crypt, consider using password_hash: :bcrypt") + # end + @password_hash = :crypt + when :crypt + # require "string/crypt" + when :bcrypt + require "bcrypt" + else + raise ArgumentError, "only :crypt and :bcrypt are supported for password_hash keyword argument" + end + + File.open(@path,"a").close unless File.exist?(@path) + reload + end + + ## + # Reload passwords from the database + + def reload + mtime = File::mtime(@path) + if mtime > @mtime + @passwd.clear + File.open(@path){|io| + while line = io.gets + line.chomp! + case line + when %r!\A[^:]+:[a-zA-Z0-9./]{13}\z! + if @password_hash == :bcrypt + raise StandardError, ".htpasswd file contains crypt password, only bcrypt passwords supported" + end + user, pass = line.split(":") + when %r!\A[^:]+:\$2[aby]\$\d{2}\$.{53}\z! + if @password_hash == :crypt + raise StandardError, ".htpasswd file contains bcrypt password, only crypt passwords supported" + end + user, pass = line.split(":") + when /:\$/, /:{SHA}/ + raise NotImplementedError, + 'MD5, SHA1 .htpasswd file not supported' + else + raise StandardError, 'bad .htpasswd file' + end + @passwd[user] = pass + end + } + @mtime = mtime + end + end + + ## + # Flush the password database. If +output+ is given the database will + # be written there instead of to the original path. + + def flush(output=nil) + output ||= @path + tmp = Tempfile.create("htpasswd", File::dirname(output)) + renamed = false + begin + each{|item| tmp.puts(item.join(":")) } + tmp.close + File::rename(tmp.path, output) + renamed = true + ensure + tmp.close + File.unlink(tmp.path) if !renamed + end + end + + ## + # Retrieves a password from the database for +user+ in +realm+. If + # +reload_db+ is true the database will be reloaded first. + + def get_passwd(realm, user, reload_db) + reload() if reload_db + @passwd[user] + end + + ## + # Sets a password in the database for +user+ in +realm+ to +pass+. + + def set_passwd(realm, user, pass) + if @password_hash == :bcrypt + # Cost of 5 to match Apache default, and because the + # bcrypt default of 10 will introduce significant delays + # for every request. + @passwd[user] = BCrypt::Password.create(pass, :cost=>5) + else + @passwd[user] = make_passwd(realm, user, pass) + end + end + + ## + # Removes a password from the database for +user+ in +realm+. + + def delete_passwd(realm, user) + @passwd.delete(user) + end + + ## + # Iterate passwords in the database. + + def each # :yields: [user, password] + @passwd.keys.sort.each{|user| + yield([user, @passwd[user]]) + } + end + end + end +end diff --git a/tool/lib/webrick/httpauth/userdb.rb b/tool/lib/webrick/httpauth/userdb.rb new file mode 100644 index 0000000000..7a17715cdf --- /dev/null +++ b/tool/lib/webrick/httpauth/userdb.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: false +#-- +# httpauth/userdb.rb -- UserDB mix-in module. +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2003 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: userdb.rb,v 1.2 2003/02/20 07:15:48 gotoyuzo Exp $ + +module WEBrick + module HTTPAuth + + ## + # User database mixin for HTTPAuth. This mixin dispatches user record + # access to the underlying auth_type for this database. + + module UserDB + + ## + # The authentication type. + # + # WEBrick::HTTPAuth::BasicAuth or WEBrick::HTTPAuth::DigestAuth are + # built-in. + + attr_accessor :auth_type + + ## + # Creates an obscured password in +realm+ with +user+ and +password+ + # using the auth_type of this database. + + def make_passwd(realm, user, pass) + @auth_type::make_passwd(realm, user, pass) + end + + ## + # Sets a password in +realm+ with +user+ and +password+ for the + # auth_type of this database. + + def set_passwd(realm, user, pass) + self[user] = pass + end + + ## + # Retrieves a password in +realm+ for +user+ for the auth_type of this + # database. +reload_db+ is a dummy value. + + def get_passwd(realm, user, reload_db=false) + make_passwd(realm, user, self[user]) + end + end + end +end diff --git a/tool/lib/webrick/httpproxy.rb b/tool/lib/webrick/httpproxy.rb new file mode 100644 index 0000000000..7607c3df88 --- /dev/null +++ b/tool/lib/webrick/httpproxy.rb @@ -0,0 +1,354 @@ +# frozen_string_literal: false +# +# httpproxy.rb -- HTTPProxy Class +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2002 GOTO Kentaro +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: httpproxy.rb,v 1.18 2003/03/08 18:58:10 gotoyuzo Exp $ +# $kNotwork: straw.rb,v 1.3 2002/02/12 15:13:07 gotoken Exp $ + +require_relative "httpserver" +require "net/http" + +module WEBrick + + NullReader = Object.new # :nodoc: + class << NullReader # :nodoc: + def read(*args) + nil + end + alias gets read + end + + FakeProxyURI = Object.new # :nodoc: + class << FakeProxyURI # :nodoc: + def method_missing(meth, *args) + if %w(scheme host port path query userinfo).member?(meth.to_s) + return nil + end + super + end + end + + # :startdoc: + + ## + # An HTTP Proxy server which proxies GET, HEAD and POST requests. + # + # To create a simple proxy server: + # + # require 'webrick' + # require 'webrick/httpproxy' + # + # proxy = WEBrick::HTTPProxyServer.new Port: 8000 + # + # trap 'INT' do proxy.shutdown end + # trap 'TERM' do proxy.shutdown end + # + # proxy.start + # + # See ::new for proxy-specific configuration items. + # + # == Modifying proxied responses + # + # To modify content the proxy server returns use the +:ProxyContentHandler+ + # option: + # + # handler = proc do |req, res| + # if res['content-type'] == 'text/plain' then + # res.body << "\nThis content was proxied!\n" + # end + # end + # + # proxy = + # WEBrick::HTTPProxyServer.new Port: 8000, ProxyContentHandler: handler + + class HTTPProxyServer < HTTPServer + + ## + # Proxy server configurations. The proxy server handles the following + # configuration items in addition to those supported by HTTPServer: + # + # :ProxyAuthProc:: Called with a request and response to authorize a + # request + # :ProxyVia:: Appended to the via header + # :ProxyURI:: The proxy server's URI + # :ProxyContentHandler:: Called with a request and response and allows + # modification of the response + # :ProxyTimeout:: Sets the proxy timeouts to 30 seconds for open and 60 + # seconds for read operations + + def initialize(config={}, default=Config::HTTP) + super(config, default) + c = @config + @via = "#{c[:HTTPVersion]} #{c[:ServerName]}:#{c[:Port]}" + end + + # :stopdoc: + def service(req, res) + if req.request_method == "CONNECT" + do_CONNECT(req, res) + elsif req.unparsed_uri =~ %r!^http://! + proxy_service(req, res) + else + super(req, res) + end + end + + def proxy_auth(req, res) + if proc = @config[:ProxyAuthProc] + proc.call(req, res) + end + req.header.delete("proxy-authorization") + end + + def proxy_uri(req, res) + # should return upstream proxy server's URI + return @config[:ProxyURI] + end + + def proxy_service(req, res) + # Proxy Authentication + proxy_auth(req, res) + + begin + public_send("do_#{req.request_method}", req, res) + rescue NoMethodError + raise HTTPStatus::MethodNotAllowed, + "unsupported method `#{req.request_method}'." + rescue => err + logger.debug("#{err.class}: #{err.message}") + raise HTTPStatus::ServiceUnavailable, err.message + end + + # Process contents + if handler = @config[:ProxyContentHandler] + handler.call(req, res) + end + end + + def do_CONNECT(req, res) + # Proxy Authentication + proxy_auth(req, res) + + ua = Thread.current[:WEBrickSocket] # User-Agent + raise HTTPStatus::InternalServerError, + "[BUG] cannot get socket" unless ua + + host, port = req.unparsed_uri.split(":", 2) + # Proxy authentication for upstream proxy server + if proxy = proxy_uri(req, res) + proxy_request_line = "CONNECT #{host}:#{port} HTTP/1.0" + if proxy.userinfo + credentials = "Basic " + [proxy.userinfo].pack("m0") + end + host, port = proxy.host, proxy.port + end + + begin + @logger.debug("CONNECT: upstream proxy is `#{host}:#{port}'.") + os = TCPSocket.new(host, port) # origin server + + if proxy + @logger.debug("CONNECT: sending a Request-Line") + os << proxy_request_line << CRLF + @logger.debug("CONNECT: > #{proxy_request_line}") + if credentials + @logger.debug("CONNECT: sending credentials") + os << "Proxy-Authorization: " << credentials << CRLF + end + os << CRLF + proxy_status_line = os.gets(LF) + @logger.debug("CONNECT: read Status-Line from the upstream server") + @logger.debug("CONNECT: < #{proxy_status_line}") + if %r{^HTTP/\d+\.\d+\s+200\s*} =~ proxy_status_line + while line = os.gets(LF) + break if /\A(#{CRLF}|#{LF})\z/om =~ line + end + else + raise HTTPStatus::BadGateway + end + end + @logger.debug("CONNECT #{host}:#{port}: succeeded") + res.status = HTTPStatus::RC_OK + rescue => ex + @logger.debug("CONNECT #{host}:#{port}: failed `#{ex.message}'") + res.set_error(ex) + raise HTTPStatus::EOFError + ensure + if handler = @config[:ProxyContentHandler] + handler.call(req, res) + end + res.send_response(ua) + access_log(@config, req, res) + + # Should clear request-line not to send the response twice. + # see: HTTPServer#run + req.parse(NullReader) rescue nil + end + + begin + while fds = IO::select([ua, os]) + if fds[0].member?(ua) + buf = ua.readpartial(1024); + @logger.debug("CONNECT: #{buf.bytesize} byte from User-Agent") + os.write(buf) + elsif fds[0].member?(os) + buf = os.readpartial(1024); + @logger.debug("CONNECT: #{buf.bytesize} byte from #{host}:#{port}") + ua.write(buf) + end + end + rescue + os.close + @logger.debug("CONNECT #{host}:#{port}: closed") + end + + raise HTTPStatus::EOFError + end + + def do_GET(req, res) + perform_proxy_request(req, res, Net::HTTP::Get) + end + + def do_HEAD(req, res) + perform_proxy_request(req, res, Net::HTTP::Head) + end + + def do_POST(req, res) + perform_proxy_request(req, res, Net::HTTP::Post, req.body_reader) + end + + def do_OPTIONS(req, res) + res['allow'] = "GET,HEAD,POST,OPTIONS,CONNECT" + end + + private + + # Some header fields should not be transferred. + HopByHop = %w( connection keep-alive proxy-authenticate upgrade + proxy-authorization te trailers transfer-encoding ) + ShouldNotTransfer = %w( set-cookie proxy-connection ) + def split_field(f) f ? f.split(/,\s+/).collect{|i| i.downcase } : [] end + + def choose_header(src, dst) + connections = split_field(src['connection']) + src.each{|key, value| + key = key.downcase + if HopByHop.member?(key) || # RFC2616: 13.5.1 + connections.member?(key) || # RFC2616: 14.10 + ShouldNotTransfer.member?(key) # pragmatics + @logger.debug("choose_header: `#{key}: #{value}'") + next + end + dst[key] = value + } + end + + # Net::HTTP is stupid about the multiple header fields. + # Here is workaround: + def set_cookie(src, dst) + if str = src['set-cookie'] + cookies = [] + str.split(/,\s*/).each{|token| + if /^[^=]+;/o =~ token + cookies[-1] << ", " << token + elsif /=/o =~ token + cookies << token + else + cookies[-1] << ", " << token + end + } + dst.cookies.replace(cookies) + end + end + + def set_via(h) + if @config[:ProxyVia] + if h['via'] + h['via'] << ", " << @via + else + h['via'] = @via + end + end + end + + def setup_proxy_header(req, res) + # Choose header fields to transfer + header = Hash.new + choose_header(req, header) + set_via(header) + return header + end + + def setup_upstream_proxy_authentication(req, res, header) + if upstream = proxy_uri(req, res) + if upstream.userinfo + header['proxy-authorization'] = + "Basic " + [upstream.userinfo].pack("m0") + end + return upstream + end + return FakeProxyURI + end + + def create_net_http(uri, upstream) + Net::HTTP.new(uri.host, uri.port, upstream.host, upstream.port) + end + + def perform_proxy_request(req, res, req_class, body_stream = nil) + uri = req.request_uri + path = uri.path.dup + path << "?" << uri.query if uri.query + header = setup_proxy_header(req, res) + upstream = setup_upstream_proxy_authentication(req, res, header) + + body_tmp = [] + http = create_net_http(uri, upstream) + req_fib = Fiber.new do + http.start do + if @config[:ProxyTimeout] + ################################## these issues are + http.open_timeout = 30 # secs # necessary (maybe because + http.read_timeout = 60 # secs # Ruby's bug, but why?) + ################################## + end + if body_stream && req['transfer-encoding'] =~ /\bchunked\b/i + header['Transfer-Encoding'] = 'chunked' + end + http_req = req_class.new(path, header) + http_req.body_stream = body_stream if body_stream + http.request(http_req) do |response| + # Persistent connection requirements are mysterious for me. + # So I will close the connection in every response. + res['proxy-connection'] = "close" + res['connection'] = "close" + + # stream Net::HTTP::HTTPResponse to WEBrick::HTTPResponse + res.status = response.code.to_i + res.chunked = response.chunked? + choose_header(response, res) + set_cookie(response, res) + set_via(res) + response.read_body do |buf| + body_tmp << buf + Fiber.yield # wait for res.body Proc#call + end + end # http.request + end + end + req_fib.resume # read HTTP response headers and first chunk of the body + res.body = ->(socket) do + while buf = body_tmp.shift + socket.write(buf) + buf.clear + req_fib.resume # continue response.read_body + end + end + end + # :stopdoc: + end +end diff --git a/tool/lib/webrick/httprequest.rb b/tool/lib/webrick/httprequest.rb new file mode 100644 index 0000000000..d34eac7ecf --- /dev/null +++ b/tool/lib/webrick/httprequest.rb @@ -0,0 +1,636 @@ +# frozen_string_literal: false +# +# httprequest.rb -- HTTPRequest Class +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: httprequest.rb,v 1.64 2003/07/13 17:18:22 gotoyuzo Exp $ + +require 'fiber' +require 'uri' +require_relative 'httpversion' +require_relative 'httpstatus' +require_relative 'httputils' +require_relative 'cookie' + +module WEBrick + + ## + # An HTTP request. This is consumed by service and do_* methods in + # WEBrick servlets + + class HTTPRequest + + BODY_CONTAINABLE_METHODS = [ "POST", "PUT" ] # :nodoc: + + # :section: Request line + + ## + # The complete request line such as: + # + # GET / HTTP/1.1 + + attr_reader :request_line + + ## + # The request method, GET, POST, PUT, etc. + + attr_reader :request_method + + ## + # The unparsed URI of the request + + attr_reader :unparsed_uri + + ## + # The HTTP version of the request + + attr_reader :http_version + + # :section: Request-URI + + ## + # The parsed URI of the request + + attr_reader :request_uri + + ## + # The request path + + attr_reader :path + + ## + # The script name (CGI variable) + + attr_accessor :script_name + + ## + # The path info (CGI variable) + + attr_accessor :path_info + + ## + # The query from the URI of the request + + attr_accessor :query_string + + # :section: Header and entity body + + ## + # The raw header of the request + + attr_reader :raw_header + + ## + # The parsed header of the request + + attr_reader :header + + ## + # The parsed request cookies + + attr_reader :cookies + + ## + # The Accept header value + + attr_reader :accept + + ## + # The Accept-Charset header value + + attr_reader :accept_charset + + ## + # The Accept-Encoding header value + + attr_reader :accept_encoding + + ## + # The Accept-Language header value + + attr_reader :accept_language + + # :section: + + ## + # The remote user (CGI variable) + + attr_accessor :user + + ## + # The socket address of the server + + attr_reader :addr + + ## + # The socket address of the client + + attr_reader :peeraddr + + ## + # Hash of request attributes + + attr_reader :attributes + + ## + # Is this a keep-alive connection? + + attr_reader :keep_alive + + ## + # The local time this request was received + + attr_reader :request_time + + ## + # Creates a new HTTP request. WEBrick::Config::HTTP is the default + # configuration. + + def initialize(config) + @config = config + @buffer_size = @config[:InputBufferSize] + @logger = config[:Logger] + + @request_line = @request_method = + @unparsed_uri = @http_version = nil + + @request_uri = @host = @port = @path = nil + @script_name = @path_info = nil + @query_string = nil + @query = nil + @form_data = nil + + @raw_header = Array.new + @header = nil + @cookies = [] + @accept = [] + @accept_charset = [] + @accept_encoding = [] + @accept_language = [] + @body = "" + + @addr = @peeraddr = nil + @attributes = {} + @user = nil + @keep_alive = false + @request_time = nil + + @remaining_size = nil + @socket = nil + + @forwarded_proto = @forwarded_host = @forwarded_port = + @forwarded_server = @forwarded_for = nil + end + + ## + # Parses a request from +socket+. This is called internally by + # WEBrick::HTTPServer. + + def parse(socket=nil) + @socket = socket + begin + @peeraddr = socket.respond_to?(:peeraddr) ? socket.peeraddr : [] + @addr = socket.respond_to?(:addr) ? socket.addr : [] + rescue Errno::ENOTCONN + raise HTTPStatus::EOFError + end + + read_request_line(socket) + if @http_version.major > 0 + read_header(socket) + @header['cookie'].each{|cookie| + @cookies += Cookie::parse(cookie) + } + @accept = HTTPUtils.parse_qvalues(self['accept']) + @accept_charset = HTTPUtils.parse_qvalues(self['accept-charset']) + @accept_encoding = HTTPUtils.parse_qvalues(self['accept-encoding']) + @accept_language = HTTPUtils.parse_qvalues(self['accept-language']) + end + return if @request_method == "CONNECT" + return if @unparsed_uri == "*" + + begin + setup_forwarded_info + @request_uri = parse_uri(@unparsed_uri) + @path = HTTPUtils::unescape(@request_uri.path) + @path = HTTPUtils::normalize_path(@path) + @host = @request_uri.host + @port = @request_uri.port + @query_string = @request_uri.query + @script_name = "" + @path_info = @path.dup + rescue + raise HTTPStatus::BadRequest, "bad URI `#{@unparsed_uri}'." + end + + if /\Aclose\z/io =~ self["connection"] + @keep_alive = false + elsif /\Akeep-alive\z/io =~ self["connection"] + @keep_alive = true + elsif @http_version < "1.1" + @keep_alive = false + else + @keep_alive = true + end + end + + ## + # Generate HTTP/1.1 100 continue response if the client expects it, + # otherwise does nothing. + + def continue # :nodoc: + if self['expect'] == '100-continue' && @config[:HTTPVersion] >= "1.1" + @socket << "HTTP/#{@config[:HTTPVersion]} 100 continue#{CRLF}#{CRLF}" + @header.delete('expect') + end + end + + ## + # Returns the request body. + + def body(&block) # :yields: body_chunk + block ||= Proc.new{|chunk| @body << chunk } + read_body(@socket, block) + @body.empty? ? nil : @body + end + + ## + # Prepares the HTTPRequest object for use as the + # source for IO.copy_stream + + def body_reader + @body_tmp = [] + @body_rd = Fiber.new do + body do |buf| + @body_tmp << buf + Fiber.yield + end + end + @body_rd.resume # grab the first chunk and yield + self + end + + # for IO.copy_stream. + def readpartial(size, buf = ''.b) # :nodoc + res = @body_tmp.shift or raise EOFError, 'end of file reached' + if res.length > size + @body_tmp.unshift(res[size..-1]) + res = res[0..size - 1] + end + buf.replace(res) + res.clear + # get more chunks - check alive? because we can take a partial chunk + @body_rd.resume if @body_rd.alive? + buf + end + + ## + # Request query as a Hash + + def query + unless @query + parse_query() + end + @query + end + + ## + # The content-length header + + def content_length + return Integer(self['content-length']) + end + + ## + # The content-type header + + def content_type + return self['content-type'] + end + + ## + # Retrieves +header_name+ + + def [](header_name) + if @header + value = @header[header_name.downcase] + value.empty? ? nil : value.join(", ") + end + end + + ## + # Iterates over the request headers + + def each + if @header + @header.each{|k, v| + value = @header[k] + yield(k, value.empty? ? nil : value.join(", ")) + } + end + end + + ## + # The host this request is for + + def host + return @forwarded_host || @host + end + + ## + # The port this request is for + + def port + return @forwarded_port || @port + end + + ## + # The server name this request is for + + def server_name + return @forwarded_server || @config[:ServerName] + end + + ## + # The client's IP address + + def remote_ip + return self["client-ip"] || @forwarded_for || @peeraddr[3] + end + + ## + # Is this an SSL request? + + def ssl? + return @request_uri.scheme == "https" + end + + ## + # Should the connection this request was made on be kept alive? + + def keep_alive? + @keep_alive + end + + def to_s # :nodoc: + ret = @request_line.dup + @raw_header.each{|line| ret << line } + ret << CRLF + ret << body if body + ret + end + + ## + # Consumes any remaining body and updates keep-alive status + + def fixup() # :nodoc: + begin + body{|chunk| } # read remaining body + rescue HTTPStatus::Error => ex + @logger.error("HTTPRequest#fixup: #{ex.class} occurred.") + @keep_alive = false + rescue => ex + @logger.error(ex) + @keep_alive = false + end + end + + # This method provides the metavariables defined by the revision 3 + # of "The WWW Common Gateway Interface Version 1.1" + # To browse the current document of CGI Version 1.1, see below: + # http://tools.ietf.org/html/rfc3875 + + def meta_vars + meta = Hash.new + + cl = self["Content-Length"] + ct = self["Content-Type"] + meta["CONTENT_LENGTH"] = cl if cl.to_i > 0 + meta["CONTENT_TYPE"] = ct.dup if ct + meta["GATEWAY_INTERFACE"] = "CGI/1.1" + meta["PATH_INFO"] = @path_info ? @path_info.dup : "" + #meta["PATH_TRANSLATED"] = nil # no plan to be provided + meta["QUERY_STRING"] = @query_string ? @query_string.dup : "" + meta["REMOTE_ADDR"] = @peeraddr[3] + meta["REMOTE_HOST"] = @peeraddr[2] + #meta["REMOTE_IDENT"] = nil # no plan to be provided + meta["REMOTE_USER"] = @user + meta["REQUEST_METHOD"] = @request_method.dup + meta["REQUEST_URI"] = @request_uri.to_s + meta["SCRIPT_NAME"] = @script_name.dup + meta["SERVER_NAME"] = @host + meta["SERVER_PORT"] = @port.to_s + meta["SERVER_PROTOCOL"] = "HTTP/" + @config[:HTTPVersion].to_s + meta["SERVER_SOFTWARE"] = @config[:ServerSoftware].dup + + self.each{|key, val| + next if /^content-type$/i =~ key + next if /^content-length$/i =~ key + name = "HTTP_" + key + name.gsub!(/-/o, "_") + name.upcase! + meta[name] = val + } + + meta + end + + private + + # :stopdoc: + + MAX_URI_LENGTH = 2083 # :nodoc: + + # same as Mongrel, Thin and Puma + MAX_HEADER_LENGTH = (112 * 1024) # :nodoc: + + def read_request_line(socket) + @request_line = read_line(socket, MAX_URI_LENGTH) if socket + raise HTTPStatus::EOFError unless @request_line + + @request_bytes = @request_line.bytesize + if @request_bytes >= MAX_URI_LENGTH and @request_line[-1, 1] != LF + raise HTTPStatus::RequestURITooLarge + end + + @request_time = Time.now + if /^(\S+)\s+(\S++)(?:\s+HTTP\/(\d+\.\d+))?\r?\n/mo =~ @request_line + @request_method = $1 + @unparsed_uri = $2 + @http_version = HTTPVersion.new($3 ? $3 : "0.9") + else + rl = @request_line.sub(/\x0d?\x0a\z/o, '') + raise HTTPStatus::BadRequest, "bad Request-Line `#{rl}'." + end + end + + def read_header(socket) + if socket + while line = read_line(socket) + break if /\A(#{CRLF}|#{LF})\z/om =~ line + if (@request_bytes += line.bytesize) > MAX_HEADER_LENGTH + raise HTTPStatus::RequestEntityTooLarge, 'headers too large' + end + @raw_header << line + end + end + @header = HTTPUtils::parse_header(@raw_header.join) + end + + def parse_uri(str, scheme="http") + if @config[:Escape8bitURI] + str = HTTPUtils::escape8bit(str) + end + str.sub!(%r{\A/+}o, '/') + uri = URI::parse(str) + return uri if uri.absolute? + if @forwarded_host + host, port = @forwarded_host, @forwarded_port + elsif self["host"] + pattern = /\A(#{URI::REGEXP::PATTERN::HOST})(?::(\d+))?\z/n + host, port = *self['host'].scan(pattern)[0] + elsif @addr.size > 0 + host, port = @addr[2], @addr[1] + else + host, port = @config[:ServerName], @config[:Port] + end + uri.scheme = @forwarded_proto || scheme + uri.host = host + uri.port = port ? port.to_i : nil + return URI::parse(uri.to_s) + end + + def read_body(socket, block) + return unless socket + if tc = self['transfer-encoding'] + case tc + when /\Achunked\z/io then read_chunked(socket, block) + else raise HTTPStatus::NotImplemented, "Transfer-Encoding: #{tc}." + end + elsif self['content-length'] || @remaining_size + @remaining_size ||= self['content-length'].to_i + while @remaining_size > 0 + sz = [@buffer_size, @remaining_size].min + break unless buf = read_data(socket, sz) + @remaining_size -= buf.bytesize + block.call(buf) + end + if @remaining_size > 0 && @socket.eof? + raise HTTPStatus::BadRequest, "invalid body size." + end + elsif BODY_CONTAINABLE_METHODS.member?(@request_method) && !@socket.eof + raise HTTPStatus::LengthRequired + end + return @body + end + + def read_chunk_size(socket) + line = read_line(socket) + if /^([0-9a-fA-F]+)(?:;(\S+))?/ =~ line + chunk_size = $1.hex + chunk_ext = $2 + [ chunk_size, chunk_ext ] + else + raise HTTPStatus::BadRequest, "bad chunk `#{line}'." + end + end + + def read_chunked(socket, block) + chunk_size, = read_chunk_size(socket) + while chunk_size > 0 + begin + sz = [ chunk_size, @buffer_size ].min + data = read_data(socket, sz) # read chunk-data + if data.nil? || data.bytesize != sz + raise HTTPStatus::BadRequest, "bad chunk data size." + end + block.call(data) + end while (chunk_size -= sz) > 0 + + read_line(socket) # skip CRLF + chunk_size, = read_chunk_size(socket) + end + read_header(socket) # trailer + CRLF + @header.delete("transfer-encoding") + @remaining_size = 0 + end + + def _read_data(io, method, *arg) + begin + WEBrick::Utils.timeout(@config[:RequestTimeout]){ + return io.__send__(method, *arg) + } + rescue Errno::ECONNRESET + return nil + rescue Timeout::Error + raise HTTPStatus::RequestTimeout + end + end + + def read_line(io, size=4096) + _read_data(io, :gets, LF, size) + end + + def read_data(io, size) + _read_data(io, :read, size) + end + + def parse_query() + begin + if @request_method == "GET" || @request_method == "HEAD" + @query = HTTPUtils::parse_query(@query_string) + elsif self['content-type'] =~ /^application\/x-www-form-urlencoded/ + @query = HTTPUtils::parse_query(body) + elsif self['content-type'] =~ /^multipart\/form-data; boundary=(.+)/ + boundary = HTTPUtils::dequote($1) + @query = HTTPUtils::parse_form_data(body, boundary) + else + @query = Hash.new + end + rescue => ex + raise HTTPStatus::BadRequest, ex.message + end + end + + PrivateNetworkRegexp = / + ^unknown$| + ^((::ffff:)?127.0.0.1|::1)$| + ^(::ffff:)?(10|172\.(1[6-9]|2[0-9]|3[01])|192\.168)\. + /ixo + + # It's said that all X-Forwarded-* headers will contain more than one + # (comma-separated) value if the original request already contained one of + # these headers. Since we could use these values as Host header, we choose + # the initial(first) value. (apr_table_mergen() adds new value after the + # existing value with ", " prefix) + def setup_forwarded_info + if @forwarded_server = self["x-forwarded-server"] + @forwarded_server = @forwarded_server.split(",", 2).first + end + if @forwarded_proto = self["x-forwarded-proto"] + @forwarded_proto = @forwarded_proto.split(",", 2).first + end + if host_port = self["x-forwarded-host"] + host_port = host_port.split(",", 2).first + if host_port =~ /\A(\[[0-9a-fA-F:]+\])(?::(\d+))?\z/ + @forwarded_host = $1 + tmp = $2 + else + @forwarded_host, tmp = host_port.split(":", 2) + end + @forwarded_port = (tmp || (@forwarded_proto == "https" ? 443 : 80)).to_i + end + if addrs = self["x-forwarded-for"] + addrs = addrs.split(",").collect(&:strip) + addrs.reject!{|ip| PrivateNetworkRegexp =~ ip } + @forwarded_for = addrs.first + end + end + + # :startdoc: + end +end diff --git a/tool/lib/webrick/httpresponse.rb b/tool/lib/webrick/httpresponse.rb new file mode 100644 index 0000000000..ba4494ab74 --- /dev/null +++ b/tool/lib/webrick/httpresponse.rb @@ -0,0 +1,564 @@ +# frozen_string_literal: false +# +# httpresponse.rb -- HTTPResponse Class +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: httpresponse.rb,v 1.45 2003/07/11 11:02:25 gotoyuzo Exp $ + +require 'time' +require 'uri' +require_relative 'httpversion' +require_relative 'htmlutils' +require_relative 'httputils' +require_relative 'httpstatus' + +module WEBrick + ## + # An HTTP response. This is filled in by the service or do_* methods of a + # WEBrick HTTP Servlet. + + class HTTPResponse + class InvalidHeader < StandardError + end + + ## + # HTTP Response version + + attr_reader :http_version + + ## + # Response status code (200) + + attr_reader :status + + ## + # Response header + + attr_reader :header + + ## + # Response cookies + + attr_reader :cookies + + ## + # Response reason phrase ("OK") + + attr_accessor :reason_phrase + + ## + # Body may be: + # * a String; + # * an IO-like object that responds to +#read+ and +#readpartial+; + # * a Proc-like object that responds to +#call+. + # + # In the latter case, either #chunked= should be set to +true+, + # or <code>header['content-length']</code> explicitly provided. + # Example: + # + # server.mount_proc '/' do |req, res| + # res.chunked = true + # # or + # # res.header['content-length'] = 10 + # res.body = proc { |out| out.write(Time.now.to_s) } + # end + + attr_accessor :body + + ## + # Request method for this response + + attr_accessor :request_method + + ## + # Request URI for this response + + attr_accessor :request_uri + + ## + # Request HTTP version for this response + + attr_accessor :request_http_version + + ## + # Filename of the static file in this response. Only used by the + # FileHandler servlet. + + attr_accessor :filename + + ## + # Is this a keep-alive response? + + attr_accessor :keep_alive + + ## + # Configuration for this response + + attr_reader :config + + ## + # Bytes sent in this response + + attr_reader :sent_size + + ## + # Creates a new HTTP response object. WEBrick::Config::HTTP is the + # default configuration. + + def initialize(config) + @config = config + @buffer_size = config[:OutputBufferSize] + @logger = config[:Logger] + @header = Hash.new + @status = HTTPStatus::RC_OK + @reason_phrase = nil + @http_version = HTTPVersion::convert(@config[:HTTPVersion]) + @body = '' + @keep_alive = true + @cookies = [] + @request_method = nil + @request_uri = nil + @request_http_version = @http_version # temporary + @chunked = false + @filename = nil + @sent_size = 0 + @bodytempfile = nil + end + + ## + # The response's HTTP status line + + def status_line + "HTTP/#@http_version #@status #@reason_phrase".rstrip << CRLF + end + + ## + # Sets the response's status to the +status+ code + + def status=(status) + @status = status + @reason_phrase = HTTPStatus::reason_phrase(status) + end + + ## + # Retrieves the response header +field+ + + def [](field) + @header[field.downcase] + end + + ## + # Sets the response header +field+ to +value+ + + def []=(field, value) + @chunked = value.to_s.downcase == 'chunked' if field.downcase == 'transfer-encoding' + @header[field.downcase] = value.to_s + end + + ## + # The content-length header + + def content_length + if len = self['content-length'] + return Integer(len) + end + end + + ## + # Sets the content-length header to +len+ + + def content_length=(len) + self['content-length'] = len.to_s + end + + ## + # The content-type header + + def content_type + self['content-type'] + end + + ## + # Sets the content-type header to +type+ + + def content_type=(type) + self['content-type'] = type + end + + ## + # Iterates over each header in the response + + def each + @header.each{|field, value| yield(field, value) } + end + + ## + # Will this response body be returned using chunked transfer-encoding? + + def chunked? + @chunked + end + + ## + # Enables chunked transfer encoding. + + def chunked=(val) + @chunked = val ? true : false + end + + ## + # Will this response's connection be kept alive? + + def keep_alive? + @keep_alive + end + + ## + # Sends the response on +socket+ + + def send_response(socket) # :nodoc: + begin + setup_header() + send_header(socket) + send_body(socket) + rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ENOTCONN => ex + @logger.debug(ex) + @keep_alive = false + rescue Exception => ex + @logger.error(ex) + @keep_alive = false + end + end + + ## + # Sets up the headers for sending + + def setup_header() # :nodoc: + @reason_phrase ||= HTTPStatus::reason_phrase(@status) + @header['server'] ||= @config[:ServerSoftware] + @header['date'] ||= Time.now.httpdate + + # HTTP/0.9 features + if @request_http_version < "1.0" + @http_version = HTTPVersion.new("0.9") + @keep_alive = false + end + + # HTTP/1.0 features + if @request_http_version < "1.1" + if chunked? + @chunked = false + ver = @request_http_version.to_s + msg = "chunked is set for an HTTP/#{ver} request. (ignored)" + @logger.warn(msg) + end + end + + # Determine the message length (RFC2616 -- 4.4 Message Length) + if @status == 304 || @status == 204 || HTTPStatus::info?(@status) + @header.delete('content-length') + @body = "" + elsif chunked? + @header["transfer-encoding"] = "chunked" + @header.delete('content-length') + elsif %r{^multipart/byteranges} =~ @header['content-type'] + @header.delete('content-length') + elsif @header['content-length'].nil? + if @body.respond_to? :readpartial + elsif @body.respond_to? :call + make_body_tempfile + else + @header['content-length'] = (@body ? @body.bytesize : 0).to_s + end + end + + # Keep-Alive connection. + if @header['connection'] == "close" + @keep_alive = false + elsif keep_alive? + if chunked? || @header['content-length'] || @status == 304 || @status == 204 || HTTPStatus.info?(@status) + @header['connection'] = "Keep-Alive" + else + msg = "Could not determine content-length of response body. Set content-length of the response or set Response#chunked = true" + @logger.warn(msg) + @header['connection'] = "close" + @keep_alive = false + end + else + @header['connection'] = "close" + end + + # Location is a single absoluteURI. + if location = @header['location'] + if @request_uri + @header['location'] = @request_uri.merge(location).to_s + end + end + end + + def make_body_tempfile # :nodoc: + return if @bodytempfile + bodytempfile = Tempfile.create("webrick") + if @body.nil? + # nothing + elsif @body.respond_to? :readpartial + IO.copy_stream(@body, bodytempfile) + @body.close + elsif @body.respond_to? :call + @body.call(bodytempfile) + else + bodytempfile.write @body + end + bodytempfile.rewind + @body = @bodytempfile = bodytempfile + @header['content-length'] = bodytempfile.stat.size.to_s + end + + def remove_body_tempfile # :nodoc: + if @bodytempfile + @bodytempfile.close + File.unlink @bodytempfile.path + @bodytempfile = nil + end + end + + + ## + # Sends the headers on +socket+ + + def send_header(socket) # :nodoc: + if @http_version.major > 0 + data = status_line() + @header.each{|key, value| + tmp = key.gsub(/\bwww|^te$|\b\w/){ $&.upcase } + data << "#{tmp}: #{check_header(value)}" << CRLF + } + @cookies.each{|cookie| + data << "Set-Cookie: " << check_header(cookie.to_s) << CRLF + } + data << CRLF + socket.write(data) + end + rescue InvalidHeader => e + @header.clear + @cookies.clear + set_error e + retry + end + + ## + # Sends the body on +socket+ + + def send_body(socket) # :nodoc: + if @body.respond_to? :readpartial then + send_body_io(socket) + elsif @body.respond_to?(:call) then + send_body_proc(socket) + else + send_body_string(socket) + end + end + + ## + # Redirects to +url+ with a WEBrick::HTTPStatus::Redirect +status+. + # + # Example: + # + # res.set_redirect WEBrick::HTTPStatus::TemporaryRedirect + + def set_redirect(status, url) + url = URI(url).to_s + @body = "<HTML><A HREF=\"#{url}\">#{url}</A>.</HTML>\n" + @header['location'] = url + raise status + end + + ## + # Creates an error page for exception +ex+ with an optional +backtrace+ + + def set_error(ex, backtrace=false) + case ex + when HTTPStatus::Status + @keep_alive = false if HTTPStatus::error?(ex.code) + self.status = ex.code + else + @keep_alive = false + self.status = HTTPStatus::RC_INTERNAL_SERVER_ERROR + end + @header['content-type'] = "text/html; charset=ISO-8859-1" + + if respond_to?(:create_error_page) + create_error_page() + return + end + + if @request_uri + host, port = @request_uri.host, @request_uri.port + else + host, port = @config[:ServerName], @config[:Port] + end + + error_body(backtrace, ex, host, port) + end + + private + + def check_header(header_value) + header_value = header_value.to_s + if /[\r\n]/ =~ header_value + raise InvalidHeader + else + header_value + end + end + + # :stopdoc: + + def error_body(backtrace, ex, host, port) + @body = '' + @body << <<-_end_of_html_ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0//EN"> +<HTML> + <HEAD><TITLE>#{HTMLUtils::escape(@reason_phrase)}</TITLE></HEAD> + <BODY> + <H1>#{HTMLUtils::escape(@reason_phrase)}</H1> + #{HTMLUtils::escape(ex.message)} + <HR> + _end_of_html_ + + if backtrace && $DEBUG + @body << "backtrace of `#{HTMLUtils::escape(ex.class.to_s)}' " + @body << "#{HTMLUtils::escape(ex.message)}" + @body << "<PRE>" + ex.backtrace.each{|line| @body << "\t#{line}\n"} + @body << "</PRE><HR>" + end + + @body << <<-_end_of_html_ + <ADDRESS> + #{HTMLUtils::escape(@config[:ServerSoftware])} at + #{host}:#{port} + </ADDRESS> + </BODY> +</HTML> + _end_of_html_ + end + + def send_body_io(socket) + begin + if @request_method == "HEAD" + # do nothing + elsif chunked? + buf = '' + begin + @body.readpartial(@buffer_size, buf) + size = buf.bytesize + data = "#{size.to_s(16)}#{CRLF}#{buf}#{CRLF}" + socket.write(data) + data.clear + @sent_size += size + rescue EOFError + break + end while true + buf.clear + socket.write("0#{CRLF}#{CRLF}") + else + if %r{\Abytes (\d+)-(\d+)/\d+\z} =~ @header['content-range'] + offset = $1.to_i + size = $2.to_i - offset + 1 + else + offset = nil + size = @header['content-length'] + size = size.to_i if size + end + begin + @sent_size = IO.copy_stream(@body, socket, size, offset) + rescue NotImplementedError + @body.seek(offset, IO::SEEK_SET) + @sent_size = IO.copy_stream(@body, socket, size) + end + end + ensure + @body.close + end + remove_body_tempfile + end + + def send_body_string(socket) + if @request_method == "HEAD" + # do nothing + elsif chunked? + body ? @body.bytesize : 0 + while buf = @body[@sent_size, @buffer_size] + break if buf.empty? + size = buf.bytesize + data = "#{size.to_s(16)}#{CRLF}#{buf}#{CRLF}" + buf.clear + socket.write(data) + @sent_size += size + end + socket.write("0#{CRLF}#{CRLF}") + else + if @body && @body.bytesize > 0 + socket.write(@body) + @sent_size = @body.bytesize + end + end + end + + def send_body_proc(socket) + if @request_method == "HEAD" + # do nothing + elsif chunked? + @body.call(ChunkedWrapper.new(socket, self)) + socket.write("0#{CRLF}#{CRLF}") + else + size = @header['content-length'].to_i + if @bodytempfile + @bodytempfile.rewind + IO.copy_stream(@bodytempfile, socket) + else + @body.call(socket) + end + @sent_size = size + end + end + + class ChunkedWrapper + def initialize(socket, resp) + @socket = socket + @resp = resp + end + + def write(buf) + return 0 if buf.empty? + socket = @socket + @resp.instance_eval { + size = buf.bytesize + data = "#{size.to_s(16)}#{CRLF}#{buf}#{CRLF}" + socket.write(data) + data.clear + @sent_size += size + size + } + end + + def <<(*buf) + write(buf) + self + end + end + + # preserved for compatibility with some 3rd-party handlers + def _write_data(socket, data) + socket << data + end + + # :startdoc: + end + +end diff --git a/tool/lib/webrick/https.rb b/tool/lib/webrick/https.rb new file mode 100644 index 0000000000..b0a49bc40b --- /dev/null +++ b/tool/lib/webrick/https.rb @@ -0,0 +1,152 @@ +# frozen_string_literal: false +# +# https.rb -- SSL/TLS enhancement for HTTPServer +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2001 GOTOU Yuuzou +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: https.rb,v 1.15 2003/07/22 19:20:42 gotoyuzo Exp $ + +require_relative 'ssl' +require_relative 'httpserver' + +module WEBrick + module Config + HTTP.update(SSL) + end + + ## + #-- + # Adds SSL functionality to WEBrick::HTTPRequest + + class HTTPRequest + + ## + # HTTP request SSL cipher + + attr_reader :cipher + + ## + # HTTP request server certificate + + attr_reader :server_cert + + ## + # HTTP request client certificate + + attr_reader :client_cert + + # :stopdoc: + + alias orig_parse parse + + def parse(socket=nil) + if socket.respond_to?(:cert) + @server_cert = socket.cert || @config[:SSLCertificate] + @client_cert = socket.peer_cert + @client_cert_chain = socket.peer_cert_chain + @cipher = socket.cipher + end + orig_parse(socket) + end + + alias orig_parse_uri parse_uri + + def parse_uri(str, scheme="https") + if server_cert + return orig_parse_uri(str, scheme) + end + return orig_parse_uri(str) + end + private :parse_uri + + alias orig_meta_vars meta_vars + + def meta_vars + meta = orig_meta_vars + if server_cert + meta["HTTPS"] = "on" + meta["SSL_SERVER_CERT"] = @server_cert.to_pem + meta["SSL_CLIENT_CERT"] = @client_cert ? @client_cert.to_pem : "" + if @client_cert_chain + @client_cert_chain.each_with_index{|cert, i| + meta["SSL_CLIENT_CERT_CHAIN_#{i}"] = cert.to_pem + } + end + meta["SSL_CIPHER"] = @cipher[0] + meta["SSL_PROTOCOL"] = @cipher[1] + meta["SSL_CIPHER_USEKEYSIZE"] = @cipher[2].to_s + meta["SSL_CIPHER_ALGKEYSIZE"] = @cipher[3].to_s + end + meta + end + + # :startdoc: + end + + ## + #-- + # Fake WEBrick::HTTPRequest for lookup_server + + class SNIRequest + + ## + # The SNI hostname + + attr_reader :host + + ## + # The socket address of the server + + attr_reader :addr + + ## + # The port this request is for + + attr_reader :port + + ## + # Creates a new SNIRequest. + + def initialize(sslsocket, hostname) + @host = hostname + @addr = sslsocket.addr + @port = @addr[1] + end + end + + + ## + #-- + # Adds SSL functionality to WEBrick::HTTPServer + + class HTTPServer < ::WEBrick::GenericServer + ## + # ServerNameIndication callback + + def ssl_servername_callback(sslsocket, hostname = nil) + req = SNIRequest.new(sslsocket, hostname) + server = lookup_server(req) + server ? server.ssl_context : nil + end + + # :stopdoc: + + ## + # Check whether +server+ is also SSL server. + # Also +server+'s SSL context will be created. + + alias orig_virtual_host virtual_host + + def virtual_host(server) + if @config[:SSLEnable] && !server.ssl_context + raise ArgumentError, "virtual host must set SSLEnable to true" + end + orig_virtual_host(server) + end + + # :startdoc: + end +end diff --git a/tool/lib/webrick/httpserver.rb b/tool/lib/webrick/httpserver.rb new file mode 100644 index 0000000000..e85d059319 --- /dev/null +++ b/tool/lib/webrick/httpserver.rb @@ -0,0 +1,294 @@ +# frozen_string_literal: false +# +# httpserver.rb -- HTTPServer Class +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: httpserver.rb,v 1.63 2002/10/01 17:16:32 gotoyuzo Exp $ + +require 'io/wait' +require_relative 'server' +require_relative 'httputils' +require_relative 'httpstatus' +require_relative 'httprequest' +require_relative 'httpresponse' +require_relative 'httpservlet' +require_relative 'accesslog' + +module WEBrick + class HTTPServerError < ServerError; end + + ## + # An HTTP Server + + class HTTPServer < ::WEBrick::GenericServer + ## + # Creates a new HTTP server according to +config+ + # + # An HTTP server uses the following attributes: + # + # :AccessLog:: An array of access logs. See WEBrick::AccessLog + # :BindAddress:: Local address for the server to bind to + # :DocumentRoot:: Root path to serve files from + # :DocumentRootOptions:: Options for the default HTTPServlet::FileHandler + # :HTTPVersion:: The HTTP version of this server + # :Port:: Port to listen on + # :RequestCallback:: Called with a request and response before each + # request is serviced. + # :RequestTimeout:: Maximum time to wait between requests + # :ServerAlias:: Array of alternate names for this server for virtual + # hosting + # :ServerName:: Name for this server for virtual hosting + + def initialize(config={}, default=Config::HTTP) + super(config, default) + @http_version = HTTPVersion::convert(@config[:HTTPVersion]) + + @mount_tab = MountTable.new + if @config[:DocumentRoot] + mount("/", HTTPServlet::FileHandler, @config[:DocumentRoot], + @config[:DocumentRootOptions]) + end + + unless @config[:AccessLog] + @config[:AccessLog] = [ + [ $stderr, AccessLog::COMMON_LOG_FORMAT ], + [ $stderr, AccessLog::REFERER_LOG_FORMAT ] + ] + end + + @virtual_hosts = Array.new + end + + ## + # Processes requests on +sock+ + + def run(sock) + while true + req = create_request(@config) + res = create_response(@config) + server = self + begin + timeout = @config[:RequestTimeout] + while timeout > 0 + break if sock.to_io.wait_readable(0.5) + break if @status != :Running + timeout -= 0.5 + end + raise HTTPStatus::EOFError if timeout <= 0 || @status != :Running + raise HTTPStatus::EOFError if sock.eof? + req.parse(sock) + res.request_method = req.request_method + res.request_uri = req.request_uri + res.request_http_version = req.http_version + res.keep_alive = req.keep_alive? + server = lookup_server(req) || self + if callback = server[:RequestCallback] + callback.call(req, res) + elsif callback = server[:RequestHandler] + msg = ":RequestHandler is deprecated, please use :RequestCallback" + @logger.warn(msg) + callback.call(req, res) + end + server.service(req, res) + rescue HTTPStatus::EOFError, HTTPStatus::RequestTimeout => ex + res.set_error(ex) + rescue HTTPStatus::Error => ex + @logger.error(ex.message) + res.set_error(ex) + rescue HTTPStatus::Status => ex + res.status = ex.code + rescue StandardError => ex + @logger.error(ex) + res.set_error(ex, true) + ensure + if req.request_line + if req.keep_alive? && res.keep_alive? + req.fixup() + end + res.send_response(sock) + server.access_log(@config, req, res) + end + end + break if @http_version < "1.1" + break unless req.keep_alive? + break unless res.keep_alive? + end + end + + ## + # Services +req+ and fills in +res+ + + def service(req, res) + if req.unparsed_uri == "*" + if req.request_method == "OPTIONS" + do_OPTIONS(req, res) + raise HTTPStatus::OK + end + raise HTTPStatus::NotFound, "`#{req.unparsed_uri}' not found." + end + + servlet, options, script_name, path_info = search_servlet(req.path) + raise HTTPStatus::NotFound, "`#{req.path}' not found." unless servlet + req.script_name = script_name + req.path_info = path_info + si = servlet.get_instance(self, *options) + @logger.debug(format("%s is invoked.", si.class.name)) + si.service(req, res) + end + + ## + # The default OPTIONS request handler says GET, HEAD, POST and OPTIONS + # requests are allowed. + + def do_OPTIONS(req, res) + res["allow"] = "GET,HEAD,POST,OPTIONS" + end + + ## + # Mounts +servlet+ on +dir+ passing +options+ to the servlet at creation + # time + + def mount(dir, servlet, *options) + @logger.debug(sprintf("%s is mounted on %s.", servlet.inspect, dir)) + @mount_tab[dir] = [ servlet, options ] + end + + ## + # Mounts +proc+ or +block+ on +dir+ and calls it with a + # WEBrick::HTTPRequest and WEBrick::HTTPResponse + + def mount_proc(dir, proc=nil, &block) + proc ||= block + raise HTTPServerError, "must pass a proc or block" unless proc + mount(dir, HTTPServlet::ProcHandler.new(proc)) + end + + ## + # Unmounts +dir+ + + def unmount(dir) + @logger.debug(sprintf("unmount %s.", dir)) + @mount_tab.delete(dir) + end + alias umount unmount + + ## + # Finds a servlet for +path+ + + def search_servlet(path) + script_name, path_info = @mount_tab.scan(path) + servlet, options = @mount_tab[script_name] + if servlet + [ servlet, options, script_name, path_info ] + end + end + + ## + # Adds +server+ as a virtual host. + + def virtual_host(server) + @virtual_hosts << server + @virtual_hosts = @virtual_hosts.sort_by{|s| + num = 0 + num -= 4 if s[:BindAddress] + num -= 2 if s[:Port] + num -= 1 if s[:ServerName] + num + } + end + + ## + # Finds the appropriate virtual host to handle +req+ + + def lookup_server(req) + @virtual_hosts.find{|s| + (s[:BindAddress].nil? || req.addr[3] == s[:BindAddress]) && + (s[:Port].nil? || req.port == s[:Port]) && + ((s[:ServerName].nil? || req.host == s[:ServerName]) || + (!s[:ServerAlias].nil? && s[:ServerAlias].find{|h| h === req.host})) + } + end + + ## + # Logs +req+ and +res+ in the access logs. +config+ is used for the + # server name. + + def access_log(config, req, res) + param = AccessLog::setup_params(config, req, res) + @config[:AccessLog].each{|logger, fmt| + logger << AccessLog::format(fmt+"\n", param) + } + end + + ## + # Creates the HTTPRequest used when handling the HTTP + # request. Can be overridden by subclasses. + def create_request(with_webrick_config) + HTTPRequest.new(with_webrick_config) + end + + ## + # Creates the HTTPResponse used when handling the HTTP + # request. Can be overridden by subclasses. + def create_response(with_webrick_config) + HTTPResponse.new(with_webrick_config) + end + + ## + # Mount table for the path a servlet is mounted on in the directory space + # of the server. Users of WEBrick can only access this indirectly via + # WEBrick::HTTPServer#mount, WEBrick::HTTPServer#unmount and + # WEBrick::HTTPServer#search_servlet + + class MountTable # :nodoc: + def initialize + @tab = Hash.new + compile + end + + def [](dir) + dir = normalize(dir) + @tab[dir] + end + + def []=(dir, val) + dir = normalize(dir) + @tab[dir] = val + compile + val + end + + def delete(dir) + dir = normalize(dir) + res = @tab.delete(dir) + compile + res + end + + def scan(path) + @scanner =~ path + [ $&, $' ] + end + + private + + def compile + k = @tab.keys + k.sort! + k.reverse! + k.collect!{|path| Regexp.escape(path) } + @scanner = Regexp.new("\\A(" + k.join("|") +")(?=/|\\z)") + end + + def normalize(dir) + ret = dir ? dir.dup : "" + ret.sub!(%r|/+\z|, "") + ret + end + end + end +end diff --git a/tool/lib/webrick/httpservlet.rb b/tool/lib/webrick/httpservlet.rb new file mode 100644 index 0000000000..da49a1405b --- /dev/null +++ b/tool/lib/webrick/httpservlet.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: false +# +# httpservlet.rb -- HTTPServlet Utility File +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: httpservlet.rb,v 1.21 2003/02/23 12:24:46 gotoyuzo Exp $ + +require_relative 'httpservlet/abstract' +require_relative 'httpservlet/filehandler' +require_relative 'httpservlet/cgihandler' +require_relative 'httpservlet/erbhandler' +require_relative 'httpservlet/prochandler' + +module WEBrick + module HTTPServlet + FileHandler.add_handler("cgi", CGIHandler) + FileHandler.add_handler("rhtml", ERBHandler) + end +end diff --git a/tool/lib/webrick/httpservlet/abstract.rb b/tool/lib/webrick/httpservlet/abstract.rb new file mode 100644 index 0000000000..bccb091861 --- /dev/null +++ b/tool/lib/webrick/httpservlet/abstract.rb @@ -0,0 +1,152 @@ +# frozen_string_literal: false +# +# httpservlet.rb -- HTTPServlet Module +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2000 TAKAHASHI Masayoshi, GOTOU Yuuzou +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: abstract.rb,v 1.24 2003/07/11 11:16:46 gotoyuzo Exp $ + +require_relative '../htmlutils' +require_relative '../httputils' +require_relative '../httpstatus' + +module WEBrick + module HTTPServlet + class HTTPServletError < StandardError; end + + ## + # AbstractServlet allows HTTP server modules to be reused across multiple + # servers and allows encapsulation of functionality. + # + # By default a servlet will respond to GET, HEAD (through an alias to GET) + # and OPTIONS requests. + # + # By default a new servlet is initialized for every request. A servlet + # instance can be reused by overriding ::get_instance in the + # AbstractServlet subclass. + # + # == A Simple Servlet + # + # class Simple < WEBrick::HTTPServlet::AbstractServlet + # def do_GET request, response + # status, content_type, body = do_stuff_with request + # + # response.status = status + # response['Content-Type'] = content_type + # response.body = body + # end + # + # def do_stuff_with request + # return 200, 'text/plain', 'you got a page' + # end + # end + # + # This servlet can be mounted on a server at a given path: + # + # server.mount '/simple', Simple + # + # == Servlet Configuration + # + # Servlets can be configured via initialize. The first argument is the + # HTTP server the servlet is being initialized for. + # + # class Configurable < Simple + # def initialize server, color, size + # super server + # @color = color + # @size = size + # end + # + # def do_stuff_with request + # content = "<p " \ + # %q{style="color: #{@color}; font-size: #{@size}"} \ + # ">Hello, World!" + # + # return 200, "text/html", content + # end + # end + # + # This servlet must be provided two arguments at mount time: + # + # server.mount '/configurable', Configurable, 'red', '2em' + + class AbstractServlet + + ## + # Factory for servlet instances that will handle a request from +server+ + # using +options+ from the mount point. By default a new servlet + # instance is created for every call. + + def self.get_instance(server, *options) + self.new(server, *options) + end + + ## + # Initializes a new servlet for +server+ using +options+ which are + # stored as-is in +@options+. +@logger+ is also provided. + + def initialize(server, *options) + @server = @config = server + @logger = @server[:Logger] + @options = options + end + + ## + # Dispatches to a +do_+ method based on +req+ if such a method is + # available. (+do_GET+ for a GET request). Raises a MethodNotAllowed + # exception if the method is not implemented. + + def service(req, res) + method_name = "do_" + req.request_method.gsub(/-/, "_") + if respond_to?(method_name) + __send__(method_name, req, res) + else + raise HTTPStatus::MethodNotAllowed, + "unsupported method `#{req.request_method}'." + end + end + + ## + # Raises a NotFound exception + + def do_GET(req, res) + raise HTTPStatus::NotFound, "not found." + end + + ## + # Dispatches to do_GET + + def do_HEAD(req, res) + do_GET(req, res) + end + + ## + # Returns the allowed HTTP request methods + + def do_OPTIONS(req, res) + m = self.methods.grep(/\Ado_([A-Z]+)\z/) {$1} + m.sort! + res["allow"] = m.join(",") + end + + private + + ## + # Redirects to a path ending in / + + def redirect_to_directory_uri(req, res) + if req.path[-1] != ?/ + location = WEBrick::HTTPUtils.escape_path(req.path + "/") + if req.query_string && req.query_string.bytesize > 0 + location << "?" << req.query_string + end + res.set_redirect(HTTPStatus::MovedPermanently, location) + end + end + end + + end +end diff --git a/tool/lib/webrick/httpservlet/cgi_runner.rb b/tool/lib/webrick/httpservlet/cgi_runner.rb new file mode 100644 index 0000000000..0398c16749 --- /dev/null +++ b/tool/lib/webrick/httpservlet/cgi_runner.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: false +# +# cgi_runner.rb -- CGI launcher. +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2000 TAKAHASHI Masayoshi, GOTOU YUUZOU +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: cgi_runner.rb,v 1.9 2002/09/25 11:33:15 gotoyuzo Exp $ + +def sysread(io, size) + buf = "" + while size > 0 + tmp = io.sysread(size) + buf << tmp + size -= tmp.bytesize + end + return buf +end + +STDIN.binmode + +len = sysread(STDIN, 8).to_i +out = sysread(STDIN, len) +STDOUT.reopen(File.open(out, "w")) + +len = sysread(STDIN, 8).to_i +err = sysread(STDIN, len) +STDERR.reopen(File.open(err, "w")) + +len = sysread(STDIN, 8).to_i +dump = sysread(STDIN, len) +hash = Marshal.restore(dump) +ENV.keys.each{|name| ENV.delete(name) } +hash.each{|k, v| ENV[k] = v if v } + +dir = File::dirname(ENV["SCRIPT_FILENAME"]) +Dir::chdir dir + +if ARGV[0] + argv = ARGV.dup + argv << ENV["SCRIPT_FILENAME"] + exec(*argv) + # NOTREACHED +end +exec ENV["SCRIPT_FILENAME"] diff --git a/tool/lib/webrick/httpservlet/cgihandler.rb b/tool/lib/webrick/httpservlet/cgihandler.rb new file mode 100644 index 0000000000..4457770b7a --- /dev/null +++ b/tool/lib/webrick/httpservlet/cgihandler.rb @@ -0,0 +1,126 @@ +# frozen_string_literal: false +# +# cgihandler.rb -- CGIHandler Class +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: cgihandler.rb,v 1.27 2003/03/21 19:56:01 gotoyuzo Exp $ + +require 'rbconfig' +require 'tempfile' +require_relative '../config' +require_relative 'abstract' + +module WEBrick + module HTTPServlet + + ## + # Servlet for handling CGI scripts + # + # Example: + # + # server.mount('/cgi/my_script', WEBrick::HTTPServlet::CGIHandler, + # '/path/to/my_script') + + class CGIHandler < AbstractServlet + Ruby = RbConfig.ruby # :nodoc: + CGIRunner = "\"#{Ruby}\" \"#{WEBrick::Config::LIBDIR}/httpservlet/cgi_runner.rb\"" # :nodoc: + CGIRunnerArray = [Ruby, "#{WEBrick::Config::LIBDIR}/httpservlet/cgi_runner.rb".freeze].freeze # :nodoc: + + ## + # Creates a new CGI script servlet for the script at +name+ + + def initialize(server, name) + super(server, name) + @script_filename = name + @tempdir = server[:TempDir] + interpreter = server[:CGIInterpreter] + if interpreter.is_a?(Array) + @cgicmd = CGIRunnerArray + interpreter + else + @cgicmd = "#{CGIRunner} #{interpreter}" + end + end + + # :stopdoc: + + def do_GET(req, res) + cgi_in = IO::popen(@cgicmd, "wb") + cgi_out = Tempfile.new("webrick.cgiout.", @tempdir, mode: IO::BINARY) + cgi_out.set_encoding("ASCII-8BIT") + cgi_err = Tempfile.new("webrick.cgierr.", @tempdir, mode: IO::BINARY) + cgi_err.set_encoding("ASCII-8BIT") + begin + cgi_in.sync = true + meta = req.meta_vars + meta["SCRIPT_FILENAME"] = @script_filename + meta["PATH"] = @config[:CGIPathEnv] + meta.delete("HTTP_PROXY") + if /mswin|bccwin|mingw/ =~ RUBY_PLATFORM + meta["SystemRoot"] = ENV["SystemRoot"] + end + dump = Marshal.dump(meta) + + cgi_in.write("%8d" % cgi_out.path.bytesize) + cgi_in.write(cgi_out.path) + cgi_in.write("%8d" % cgi_err.path.bytesize) + cgi_in.write(cgi_err.path) + cgi_in.write("%8d" % dump.bytesize) + cgi_in.write(dump) + + req.body { |chunk| cgi_in.write(chunk) } + ensure + cgi_in.close + status = $?.exitstatus + sleep 0.1 if /mswin|bccwin|mingw/ =~ RUBY_PLATFORM + data = cgi_out.read + cgi_out.close(true) + if errmsg = cgi_err.read + if errmsg.bytesize > 0 + @logger.error("CGIHandler: #{@script_filename}:\n" + errmsg) + end + end + cgi_err.close(true) + end + + if status != 0 + @logger.error("CGIHandler: #{@script_filename} exit with #{status}") + end + + data = "" unless data + raw_header, body = data.split(/^[\xd\xa]+/, 2) + raise HTTPStatus::InternalServerError, + "Premature end of script headers: #{@script_filename}" if body.nil? + + begin + header = HTTPUtils::parse_header(raw_header) + if /^(\d+)/ =~ header['status'][0] + res.status = $1.to_i + header.delete('status') + end + if header.has_key?('location') + # RFC 3875 6.2.3, 6.2.4 + res.status = 302 unless (300...400) === res.status + end + if header.has_key?('set-cookie') + header['set-cookie'].each{|k| + res.cookies << Cookie.parse_set_cookie(k) + } + header.delete('set-cookie') + end + header.each{|key, val| res[key] = val.join(", ") } + rescue => ex + raise HTTPStatus::InternalServerError, ex.message + end + res.body = body + end + alias do_POST do_GET + + # :startdoc: + end + + end +end diff --git a/tool/lib/webrick/httpservlet/erbhandler.rb b/tool/lib/webrick/httpservlet/erbhandler.rb new file mode 100644 index 0000000000..cd09e5f216 --- /dev/null +++ b/tool/lib/webrick/httpservlet/erbhandler.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: false +# +# erbhandler.rb -- ERBHandler Class +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: erbhandler.rb,v 1.25 2003/02/24 19:25:31 gotoyuzo Exp $ + +require_relative 'abstract' + +require 'erb' + +module WEBrick + module HTTPServlet + + ## + # ERBHandler evaluates an ERB file and returns the result. This handler + # is automatically used if there are .rhtml files in a directory served by + # the FileHandler. + # + # ERBHandler supports GET and POST methods. + # + # The ERB file is evaluated with the local variables +servlet_request+ and + # +servlet_response+ which are a WEBrick::HTTPRequest and + # WEBrick::HTTPResponse respectively. + # + # Example .rhtml file: + # + # Request to <%= servlet_request.request_uri %> + # + # Query params <%= servlet_request.query.inspect %> + + class ERBHandler < AbstractServlet + + ## + # Creates a new ERBHandler on +server+ that will evaluate and serve the + # ERB file +name+ + + def initialize(server, name) + super(server, name) + @script_filename = name + end + + ## + # Handles GET requests + + def do_GET(req, res) + unless defined?(ERB) + @logger.warn "#{self.class}: ERB not defined." + raise HTTPStatus::Forbidden, "ERBHandler cannot work." + end + begin + data = File.open(@script_filename, &:read) + res.body = evaluate(ERB.new(data), req, res) + res['content-type'] ||= + HTTPUtils::mime_type(@script_filename, @config[:MimeTypes]) + rescue StandardError + raise + rescue Exception => ex + @logger.error(ex) + raise HTTPStatus::InternalServerError, ex.message + end + end + + ## + # Handles POST requests + + alias do_POST do_GET + + private + + ## + # Evaluates +erb+ providing +servlet_request+ and +servlet_response+ as + # local variables. + + def evaluate(erb, servlet_request, servlet_response) + Module.new.module_eval{ + servlet_request.meta_vars + servlet_request.query + erb.result(binding) + } + end + end + end +end diff --git a/tool/lib/webrick/httpservlet/filehandler.rb b/tool/lib/webrick/httpservlet/filehandler.rb new file mode 100644 index 0000000000..010df0e918 --- /dev/null +++ b/tool/lib/webrick/httpservlet/filehandler.rb @@ -0,0 +1,552 @@ +# frozen_string_literal: false +# +# filehandler.rb -- FileHandler Module +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou +# Copyright (c) 2003 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: filehandler.rb,v 1.44 2003/06/07 01:34:51 gotoyuzo Exp $ + +require 'time' + +require_relative '../htmlutils' +require_relative '../httputils' +require_relative '../httpstatus' + +module WEBrick + module HTTPServlet + + ## + # Servlet for serving a single file. You probably want to use the + # FileHandler servlet instead as it handles directories and fancy indexes. + # + # Example: + # + # server.mount('/my_page.txt', WEBrick::HTTPServlet::DefaultFileHandler, + # '/path/to/my_page.txt') + # + # This servlet handles If-Modified-Since and Range requests. + + class DefaultFileHandler < AbstractServlet + + ## + # Creates a DefaultFileHandler instance for the file at +local_path+. + + def initialize(server, local_path) + super(server, local_path) + @local_path = local_path + end + + # :stopdoc: + + def do_GET(req, res) + st = File::stat(@local_path) + mtime = st.mtime + res['etag'] = sprintf("%x-%x-%x", st.ino, st.size, st.mtime.to_i) + + if not_modified?(req, res, mtime, res['etag']) + res.body = '' + raise HTTPStatus::NotModified + elsif req['range'] + make_partial_content(req, res, @local_path, st.size) + raise HTTPStatus::PartialContent + else + mtype = HTTPUtils::mime_type(@local_path, @config[:MimeTypes]) + res['content-type'] = mtype + res['content-length'] = st.size.to_s + res['last-modified'] = mtime.httpdate + res.body = File.open(@local_path, "rb") + end + end + + def not_modified?(req, res, mtime, etag) + if ir = req['if-range'] + begin + if Time.httpdate(ir) >= mtime + return true + end + rescue + if HTTPUtils::split_header_value(ir).member?(res['etag']) + return true + end + end + end + + if (ims = req['if-modified-since']) && Time.parse(ims) >= mtime + return true + end + + if (inm = req['if-none-match']) && + HTTPUtils::split_header_value(inm).member?(res['etag']) + return true + end + + return false + end + + # returns a lambda for webrick/httpresponse.rb send_body_proc + def multipart_body(body, parts, boundary, mtype, filesize) + lambda do |socket| + begin + begin + first = parts.shift + last = parts.shift + socket.write( + "--#{boundary}#{CRLF}" \ + "Content-Type: #{mtype}#{CRLF}" \ + "Content-Range: bytes #{first}-#{last}/#{filesize}#{CRLF}" \ + "#{CRLF}" + ) + + begin + IO.copy_stream(body, socket, last - first + 1, first) + rescue NotImplementedError + body.seek(first, IO::SEEK_SET) + IO.copy_stream(body, socket, last - first + 1) + end + socket.write(CRLF) + end while parts[0] + socket.write("--#{boundary}--#{CRLF}") + ensure + body.close + end + end + end + + def make_partial_content(req, res, filename, filesize) + mtype = HTTPUtils::mime_type(filename, @config[:MimeTypes]) + unless ranges = HTTPUtils::parse_range_header(req['range']) + raise HTTPStatus::BadRequest, + "Unrecognized range-spec: \"#{req['range']}\"" + end + File.open(filename, "rb"){|io| + if ranges.size > 1 + time = Time.now + boundary = "#{time.sec}_#{time.usec}_#{Process::pid}" + parts = [] + ranges.each {|range| + prange = prepare_range(range, filesize) + next if prange[0] < 0 + parts.concat(prange) + } + raise HTTPStatus::RequestRangeNotSatisfiable if parts.empty? + res["content-type"] = "multipart/byteranges; boundary=#{boundary}" + if req.http_version < '1.1' + res['connection'] = 'close' + else + res.chunked = true + end + res.body = multipart_body(io.dup, parts, boundary, mtype, filesize) + elsif range = ranges[0] + first, last = prepare_range(range, filesize) + raise HTTPStatus::RequestRangeNotSatisfiable if first < 0 + res['content-type'] = mtype + res['content-range'] = "bytes #{first}-#{last}/#{filesize}" + res['content-length'] = (last - first + 1).to_s + res.body = io.dup + else + raise HTTPStatus::BadRequest + end + } + end + + def prepare_range(range, filesize) + first = range.first < 0 ? filesize + range.first : range.first + return -1, -1 if first < 0 || first >= filesize + last = range.last < 0 ? filesize + range.last : range.last + last = filesize - 1 if last >= filesize + return first, last + end + + # :startdoc: + end + + ## + # Serves a directory including fancy indexing and a variety of other + # options. + # + # Example: + # + # server.mount('/assets', WEBrick::HTTPServlet::FileHandler, + # '/path/to/assets') + + class FileHandler < AbstractServlet + HandlerTable = Hash.new # :nodoc: + + ## + # Allow custom handling of requests for files with +suffix+ by class + # +handler+ + + def self.add_handler(suffix, handler) + HandlerTable[suffix] = handler + end + + ## + # Remove custom handling of requests for files with +suffix+ + + def self.remove_handler(suffix) + HandlerTable.delete(suffix) + end + + ## + # Creates a FileHandler servlet on +server+ that serves files starting + # at directory +root+ + # + # +options+ may be a Hash containing keys from + # WEBrick::Config::FileHandler or +true+ or +false+. + # + # If +options+ is true or false then +:FancyIndexing+ is enabled or + # disabled respectively. + + def initialize(server, root, options={}, default=Config::FileHandler) + @config = server.config + @logger = @config[:Logger] + @root = File.expand_path(root) + if options == true || options == false + options = { :FancyIndexing => options } + end + @options = default.dup.update(options) + end + + # :stopdoc: + + def set_filesystem_encoding(str) + enc = Encoding.find('filesystem') + if enc == Encoding::US_ASCII + str.b + else + str.dup.force_encoding(enc) + end + end + + def service(req, res) + # if this class is mounted on "/" and /~username is requested. + # we're going to override path information before invoking service. + if defined?(Etc) && @options[:UserDir] && req.script_name.empty? + if %r|^(/~([^/]+))| =~ req.path_info + script_name, user = $1, $2 + path_info = $' + begin + passwd = Etc::getpwnam(user) + @root = File::join(passwd.dir, @options[:UserDir]) + req.script_name = script_name + req.path_info = path_info + rescue + @logger.debug "#{self.class}#do_GET: getpwnam(#{user}) failed" + end + end + end + prevent_directory_traversal(req, res) + super(req, res) + end + + def do_GET(req, res) + unless exec_handler(req, res) + set_dir_list(req, res) + end + end + + def do_POST(req, res) + unless exec_handler(req, res) + raise HTTPStatus::NotFound, "`#{req.path}' not found." + end + end + + def do_OPTIONS(req, res) + unless exec_handler(req, res) + super(req, res) + end + end + + # ToDo + # RFC2518: HTTP Extensions for Distributed Authoring -- WEBDAV + # + # PROPFIND PROPPATCH MKCOL DELETE PUT COPY MOVE + # LOCK UNLOCK + + # RFC3253: Versioning Extensions to WebDAV + # (Web Distributed Authoring and Versioning) + # + # VERSION-CONTROL REPORT CHECKOUT CHECK_IN UNCHECKOUT + # MKWORKSPACE UPDATE LABEL MERGE ACTIVITY + + private + + def trailing_pathsep?(path) + # check for trailing path separator: + # File.dirname("/aaaa/bbbb/") #=> "/aaaa") + # File.dirname("/aaaa/bbbb/x") #=> "/aaaa/bbbb") + # File.dirname("/aaaa/bbbb") #=> "/aaaa") + # File.dirname("/aaaa/bbbbx") #=> "/aaaa") + return File.dirname(path) != File.dirname(path+"x") + end + + def prevent_directory_traversal(req, res) + # Preventing directory traversal on Windows platforms; + # Backslashes (0x5c) in path_info are not interpreted as special + # character in URI notation. So the value of path_info should be + # normalize before accessing to the filesystem. + + # dirty hack for filesystem encoding; in nature, File.expand_path + # should not be used for path normalization. [Bug #3345] + path = req.path_info.dup.force_encoding(Encoding.find("filesystem")) + if trailing_pathsep?(req.path_info) + # File.expand_path removes the trailing path separator. + # Adding a character is a workaround to save it. + # File.expand_path("/aaa/") #=> "/aaa" + # File.expand_path("/aaa/" + "x") #=> "/aaa/x" + expanded = File.expand_path(path + "x") + expanded.chop! # remove trailing "x" + else + expanded = File.expand_path(path) + end + expanded.force_encoding(req.path_info.encoding) + req.path_info = expanded + end + + def exec_handler(req, res) + raise HTTPStatus::NotFound, "`#{req.path}' not found." unless @root + if set_filename(req, res) + handler = get_handler(req, res) + call_callback(:HandlerCallback, req, res) + h = handler.get_instance(@config, res.filename) + h.service(req, res) + return true + end + call_callback(:HandlerCallback, req, res) + return false + end + + def get_handler(req, res) + suffix1 = (/\.(\w+)\z/ =~ res.filename) && $1.downcase + if /\.(\w+)\.([\w\-]+)\z/ =~ res.filename + if @options[:AcceptableLanguages].include?($2.downcase) + suffix2 = $1.downcase + end + end + handler_table = @options[:HandlerTable] + return handler_table[suffix1] || handler_table[suffix2] || + HandlerTable[suffix1] || HandlerTable[suffix2] || + DefaultFileHandler + end + + def set_filename(req, res) + res.filename = @root + path_info = req.path_info.scan(%r|/[^/]*|) + + path_info.unshift("") # dummy for checking @root dir + while base = path_info.first + base = set_filesystem_encoding(base) + break if base == "/" + break unless File.directory?(File.expand_path(res.filename + base)) + shift_path_info(req, res, path_info) + call_callback(:DirectoryCallback, req, res) + end + + if base = path_info.first + base = set_filesystem_encoding(base) + if base == "/" + if file = search_index_file(req, res) + shift_path_info(req, res, path_info, file) + call_callback(:FileCallback, req, res) + return true + end + shift_path_info(req, res, path_info) + elsif file = search_file(req, res, base) + shift_path_info(req, res, path_info, file) + call_callback(:FileCallback, req, res) + return true + else + raise HTTPStatus::NotFound, "`#{req.path}' not found." + end + end + + return false + end + + def check_filename(req, res, name) + if nondisclosure_name?(name) || windows_ambiguous_name?(name) + @logger.warn("the request refers nondisclosure name `#{name}'.") + raise HTTPStatus::NotFound, "`#{req.path}' not found." + end + end + + def shift_path_info(req, res, path_info, base=nil) + tmp = path_info.shift + base = base || set_filesystem_encoding(tmp) + req.path_info = path_info.join + req.script_name << base + res.filename = File.expand_path(res.filename + base) + check_filename(req, res, File.basename(res.filename)) + end + + def search_index_file(req, res) + @config[:DirectoryIndex].each{|index| + if file = search_file(req, res, "/"+index) + return file + end + } + return nil + end + + def search_file(req, res, basename) + langs = @options[:AcceptableLanguages] + path = res.filename + basename + if File.file?(path) + return basename + elsif langs.size > 0 + req.accept_language.each{|lang| + path_with_lang = path + ".#{lang}" + if langs.member?(lang) && File.file?(path_with_lang) + return basename + ".#{lang}" + end + } + (langs - req.accept_language).each{|lang| + path_with_lang = path + ".#{lang}" + if File.file?(path_with_lang) + return basename + ".#{lang}" + end + } + end + return nil + end + + def call_callback(callback_name, req, res) + if cb = @options[callback_name] + cb.call(req, res) + end + end + + def windows_ambiguous_name?(name) + return true if /[. ]+\z/ =~ name + return true if /::\$DATA\z/ =~ name + return false + end + + def nondisclosure_name?(name) + @options[:NondisclosureName].each{|pattern| + if File.fnmatch(pattern, name, File::FNM_CASEFOLD) + return true + end + } + return false + end + + def set_dir_list(req, res) + redirect_to_directory_uri(req, res) + unless @options[:FancyIndexing] + raise HTTPStatus::Forbidden, "no access permission to `#{req.path}'" + end + local_path = res.filename + list = Dir::entries(local_path).collect{|name| + next if name == "." || name == ".." + next if nondisclosure_name?(name) + next if windows_ambiguous_name?(name) + st = (File::stat(File.join(local_path, name)) rescue nil) + if st.nil? + [ name, nil, -1 ] + elsif st.directory? + [ name + "/", st.mtime, -1 ] + else + [ name, st.mtime, st.size ] + end + } + list.compact! + + query = req.query + + d0 = nil + idx = nil + %w[N M S].each_with_index do |q, i| + if d = query.delete(q) + idx ||= i + d0 ||= d + end + end + d0 ||= "A" + idx ||= 0 + d1 = (d0 == "A") ? "D" : "A" + + if d0 == "A" + list.sort!{|a,b| a[idx] <=> b[idx] } + else + list.sort!{|a,b| b[idx] <=> a[idx] } + end + + namewidth = query["NameWidth"] + if namewidth == "*" + namewidth = nil + elsif !namewidth or (namewidth = namewidth.to_i) < 2 + namewidth = 25 + end + query = query.inject('') {|s, (k, v)| s << '&' << HTMLUtils::escape("#{k}=#{v}")} + + type = "text/html" + case enc = Encoding.find('filesystem') + when Encoding::US_ASCII, Encoding::ASCII_8BIT + else + type << "; charset=\"#{enc.name}\"" + end + res['content-type'] = type + + title = "Index of #{HTMLUtils::escape(req.path)}" + res.body = <<-_end_of_html_ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN"> +<HTML> + <HEAD> + <TITLE>#{title}</TITLE> + <style type="text/css"> + <!-- + .name, .mtime { text-align: left; } + .size { text-align: right; } + td { text-overflow: ellipsis; white-space: nowrap; overflow: hidden; } + table { border-collapse: collapse; } + tr th { border-bottom: 2px groove; } + //--> + </style> + </HEAD> + <BODY> + <H1>#{title}</H1> + _end_of_html_ + + res.body << "<TABLE width=\"100%\"><THEAD><TR>\n" + res.body << "<TH class=\"name\"><A HREF=\"?N=#{d1}#{query}\">Name</A></TH>" + res.body << "<TH class=\"mtime\"><A HREF=\"?M=#{d1}#{query}\">Last modified</A></TH>" + res.body << "<TH class=\"size\"><A HREF=\"?S=#{d1}#{query}\">Size</A></TH>\n" + res.body << "</TR></THEAD>\n" + res.body << "<TBODY>\n" + + query.sub!(/\A&/, '?') + list.unshift [ "..", File::mtime(local_path+"/.."), -1 ] + list.each{ |name, time, size| + if name == ".." + dname = "Parent Directory" + elsif namewidth and name.size > namewidth + dname = name[0...(namewidth - 2)] << '..' + else + dname = name + end + s = "<TR><TD class=\"name\"><A HREF=\"#{HTTPUtils::escape(name)}#{query if name.end_with?('/')}\">#{HTMLUtils::escape(dname)}</A></TD>" + s << "<TD class=\"mtime\">" << (time ? time.strftime("%Y/%m/%d %H:%M") : "") << "</TD>" + s << "<TD class=\"size\">" << (size >= 0 ? size.to_s : "-") << "</TD></TR>\n" + res.body << s + } + res.body << "</TBODY></TABLE>" + res.body << "<HR>" + + res.body << <<-_end_of_html_ + <ADDRESS> + #{HTMLUtils::escape(@config[:ServerSoftware])}<BR> + at #{req.host}:#{req.port} + </ADDRESS> + </BODY> +</HTML> + _end_of_html_ + end + + # :startdoc: + end + end +end diff --git a/tool/lib/webrick/httpservlet/prochandler.rb b/tool/lib/webrick/httpservlet/prochandler.rb new file mode 100644 index 0000000000..599ffc4340 --- /dev/null +++ b/tool/lib/webrick/httpservlet/prochandler.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: false +# +# prochandler.rb -- ProcHandler Class +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: prochandler.rb,v 1.7 2002/09/21 12:23:42 gotoyuzo Exp $ + +require_relative 'abstract' + +module WEBrick + module HTTPServlet + + ## + # Mounts a proc at a path that accepts a request and response. + # + # Instead of mounting this servlet with WEBrick::HTTPServer#mount use + # WEBrick::HTTPServer#mount_proc: + # + # server.mount_proc '/' do |req, res| + # res.body = 'it worked!' + # res.status = 200 + # end + + class ProcHandler < AbstractServlet + # :stopdoc: + def get_instance(server, *options) + self + end + + def initialize(proc) + @proc = proc + end + + def do_GET(request, response) + @proc.call(request, response) + end + + alias do_POST do_GET + # :startdoc: + end + + end +end diff --git a/tool/lib/webrick/httpstatus.rb b/tool/lib/webrick/httpstatus.rb new file mode 100644 index 0000000000..c811f21964 --- /dev/null +++ b/tool/lib/webrick/httpstatus.rb @@ -0,0 +1,194 @@ +# frozen_string_literal: false +#-- +# httpstatus.rb -- HTTPStatus Class +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: httpstatus.rb,v 1.11 2003/03/24 20:18:55 gotoyuzo Exp $ + +require_relative 'accesslog' + +module WEBrick + + ## + # This module is used to manager HTTP status codes. + # + # See http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html for more + # information. + module HTTPStatus + + ## + # Root of the HTTP status class hierarchy + class Status < StandardError + class << self + attr_reader :code, :reason_phrase # :nodoc: + end + + # Returns the HTTP status code + def code() self::class::code end + + # Returns the HTTP status description + def reason_phrase() self::class::reason_phrase end + + alias to_i code # :nodoc: + end + + # Root of the HTTP info statuses + class Info < Status; end + # Root of the HTTP success statuses + class Success < Status; end + # Root of the HTTP redirect statuses + class Redirect < Status; end + # Root of the HTTP error statuses + class Error < Status; end + # Root of the HTTP client error statuses + class ClientError < Error; end + # Root of the HTTP server error statuses + class ServerError < Error; end + + class EOFError < StandardError; end + + # HTTP status codes and descriptions + StatusMessage = { # :nodoc: + 100 => 'Continue', + 101 => 'Switching Protocols', + 200 => 'OK', + 201 => 'Created', + 202 => 'Accepted', + 203 => 'Non-Authoritative Information', + 204 => 'No Content', + 205 => 'Reset Content', + 206 => 'Partial Content', + 207 => 'Multi-Status', + 300 => 'Multiple Choices', + 301 => 'Moved Permanently', + 302 => 'Found', + 303 => 'See Other', + 304 => 'Not Modified', + 305 => 'Use Proxy', + 307 => 'Temporary Redirect', + 400 => 'Bad Request', + 401 => 'Unauthorized', + 402 => 'Payment Required', + 403 => 'Forbidden', + 404 => 'Not Found', + 405 => 'Method Not Allowed', + 406 => 'Not Acceptable', + 407 => 'Proxy Authentication Required', + 408 => 'Request Timeout', + 409 => 'Conflict', + 410 => 'Gone', + 411 => 'Length Required', + 412 => 'Precondition Failed', + 413 => 'Request Entity Too Large', + 414 => 'Request-URI Too Large', + 415 => 'Unsupported Media Type', + 416 => 'Request Range Not Satisfiable', + 417 => 'Expectation Failed', + 422 => 'Unprocessable Entity', + 423 => 'Locked', + 424 => 'Failed Dependency', + 426 => 'Upgrade Required', + 428 => 'Precondition Required', + 429 => 'Too Many Requests', + 431 => 'Request Header Fields Too Large', + 451 => 'Unavailable For Legal Reasons', + 500 => 'Internal Server Error', + 501 => 'Not Implemented', + 502 => 'Bad Gateway', + 503 => 'Service Unavailable', + 504 => 'Gateway Timeout', + 505 => 'HTTP Version Not Supported', + 507 => 'Insufficient Storage', + 511 => 'Network Authentication Required', + } + + # Maps a status code to the corresponding Status class + CodeToError = {} # :nodoc: + + # Creates a status or error class for each status code and + # populates the CodeToError map. + StatusMessage.each{|code, message| + message.freeze + var_name = message.gsub(/[ \-]/,'_').upcase + err_name = message.gsub(/[ \-]/,'') + + case code + when 100...200; parent = Info + when 200...300; parent = Success + when 300...400; parent = Redirect + when 400...500; parent = ClientError + when 500...600; parent = ServerError + end + + const_set("RC_#{var_name}", code) + err_class = Class.new(parent) + err_class.instance_variable_set(:@code, code) + err_class.instance_variable_set(:@reason_phrase, message) + const_set(err_name, err_class) + CodeToError[code] = err_class + } + + ## + # Returns the description corresponding to the HTTP status +code+ + # + # WEBrick::HTTPStatus.reason_phrase 404 + # => "Not Found" + def reason_phrase(code) + StatusMessage[code.to_i] + end + + ## + # Is +code+ an informational status? + def info?(code) + code.to_i >= 100 and code.to_i < 200 + end + + ## + # Is +code+ a successful status? + def success?(code) + code.to_i >= 200 and code.to_i < 300 + end + + ## + # Is +code+ a redirection status? + def redirect?(code) + code.to_i >= 300 and code.to_i < 400 + end + + ## + # Is +code+ an error status? + def error?(code) + code.to_i >= 400 and code.to_i < 600 + end + + ## + # Is +code+ a client error status? + def client_error?(code) + code.to_i >= 400 and code.to_i < 500 + end + + ## + # Is +code+ a server error status? + def server_error?(code) + code.to_i >= 500 and code.to_i < 600 + end + + ## + # Returns the status class corresponding to +code+ + # + # WEBrick::HTTPStatus[302] + # => WEBrick::HTTPStatus::NotFound + # + def self.[](code) + CodeToError[code] + end + + module_function :reason_phrase + module_function :info?, :success?, :redirect?, :error? + module_function :client_error?, :server_error? + end +end diff --git a/tool/lib/webrick/httputils.rb b/tool/lib/webrick/httputils.rb new file mode 100644 index 0000000000..f1b9ddf9f0 --- /dev/null +++ b/tool/lib/webrick/httputils.rb @@ -0,0 +1,512 @@ +# frozen_string_literal: false +# +# httputils.rb -- HTTPUtils Module +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: httputils.rb,v 1.34 2003/06/05 21:34:08 gotoyuzo Exp $ + +require 'socket' +require 'tempfile' + +module WEBrick + CR = "\x0d" # :nodoc: + LF = "\x0a" # :nodoc: + CRLF = "\x0d\x0a" # :nodoc: + + ## + # HTTPUtils provides utility methods for working with the HTTP protocol. + # + # This module is generally used internally by WEBrick + + module HTTPUtils + + ## + # Normalizes a request path. Raises an exception if the path cannot be + # normalized. + + def normalize_path(path) + raise "abnormal path `#{path}'" if path[0] != ?/ + ret = path.dup + + ret.gsub!(%r{/+}o, '/') # // => / + while ret.sub!(%r'/\.(?:/|\Z)', '/'); end # /. => / + while ret.sub!(%r'/(?!\.\./)[^/]+/\.\.(?:/|\Z)', '/'); end # /foo/.. => /foo + + raise "abnormal path `#{path}'" if %r{/\.\.(/|\Z)} =~ ret + ret + end + module_function :normalize_path + + ## + # Default mime types + + DefaultMimeTypes = { + "ai" => "application/postscript", + "asc" => "text/plain", + "avi" => "video/x-msvideo", + "bin" => "application/octet-stream", + "bmp" => "image/bmp", + "class" => "application/octet-stream", + "cer" => "application/pkix-cert", + "crl" => "application/pkix-crl", + "crt" => "application/x-x509-ca-cert", + #"crl" => "application/x-pkcs7-crl", + "css" => "text/css", + "dms" => "application/octet-stream", + "doc" => "application/msword", + "dvi" => "application/x-dvi", + "eps" => "application/postscript", + "etx" => "text/x-setext", + "exe" => "application/octet-stream", + "gif" => "image/gif", + "htm" => "text/html", + "html" => "text/html", + "jpe" => "image/jpeg", + "jpeg" => "image/jpeg", + "jpg" => "image/jpeg", + "js" => "application/javascript", + "json" => "application/json", + "lha" => "application/octet-stream", + "lzh" => "application/octet-stream", + "mjs" => "application/javascript", + "mov" => "video/quicktime", + "mpe" => "video/mpeg", + "mpeg" => "video/mpeg", + "mpg" => "video/mpeg", + "pbm" => "image/x-portable-bitmap", + "pdf" => "application/pdf", + "pgm" => "image/x-portable-graymap", + "png" => "image/png", + "pnm" => "image/x-portable-anymap", + "ppm" => "image/x-portable-pixmap", + "ppt" => "application/vnd.ms-powerpoint", + "ps" => "application/postscript", + "qt" => "video/quicktime", + "ras" => "image/x-cmu-raster", + "rb" => "text/plain", + "rd" => "text/plain", + "rtf" => "application/rtf", + "sgm" => "text/sgml", + "sgml" => "text/sgml", + "svg" => "image/svg+xml", + "tif" => "image/tiff", + "tiff" => "image/tiff", + "txt" => "text/plain", + "wasm" => "application/wasm", + "xbm" => "image/x-xbitmap", + "xhtml" => "text/html", + "xls" => "application/vnd.ms-excel", + "xml" => "text/xml", + "xpm" => "image/x-xpixmap", + "xwd" => "image/x-xwindowdump", + "zip" => "application/zip", + } + + ## + # Loads Apache-compatible mime.types in +file+. + + def load_mime_types(file) + # note: +file+ may be a "| command" for now; some people may + # rely on this, but currently we do not use this method by default. + open(file){ |io| + hash = Hash.new + io.each{ |line| + next if /^#/ =~ line + line.chomp! + mimetype, ext0 = line.split(/\s+/, 2) + next unless ext0 + next if ext0.empty? + ext0.split(/\s+/).each{ |ext| hash[ext] = mimetype } + } + hash + } + end + module_function :load_mime_types + + ## + # Returns the mime type of +filename+ from the list in +mime_tab+. If no + # mime type was found application/octet-stream is returned. + + def mime_type(filename, mime_tab) + suffix1 = (/\.(\w+)$/ =~ filename && $1.downcase) + suffix2 = (/\.(\w+)\.[\w\-]+$/ =~ filename && $1.downcase) + mime_tab[suffix1] || mime_tab[suffix2] || "application/octet-stream" + end + module_function :mime_type + + ## + # Parses an HTTP header +raw+ into a hash of header fields with an Array + # of values. + + def parse_header(raw) + header = Hash.new([].freeze) + field = nil + raw.each_line{|line| + case line + when /^([A-Za-z0-9!\#$%&'*+\-.^_`|~]+):\s*(.*?)\s*\z/om + field, value = $1, $2 + field.downcase! + header[field] = [] unless header.has_key?(field) + header[field] << value + when /^\s+(.*?)\s*\z/om + value = $1 + unless field + raise HTTPStatus::BadRequest, "bad header '#{line}'." + end + header[field][-1] << " " << value + else + raise HTTPStatus::BadRequest, "bad header '#{line}'." + end + } + header.each{|key, values| + values.each(&:strip!) + } + header + end + module_function :parse_header + + ## + # Splits a header value +str+ according to HTTP specification. + + def split_header_value(str) + str.scan(%r'\G((?:"(?:\\.|[^"])+?"|[^",]+)+) + (?:,\s*|\Z)'xn).flatten + end + module_function :split_header_value + + ## + # Parses a Range header value +ranges_specifier+ + + def parse_range_header(ranges_specifier) + if /^bytes=(.*)/ =~ ranges_specifier + byte_range_set = split_header_value($1) + byte_range_set.collect{|range_spec| + case range_spec + when /^(\d+)-(\d+)/ then $1.to_i .. $2.to_i + when /^(\d+)-/ then $1.to_i .. -1 + when /^-(\d+)/ then -($1.to_i) .. -1 + else return nil + end + } + end + end + module_function :parse_range_header + + ## + # Parses q values in +value+ as used in Accept headers. + + def parse_qvalues(value) + tmp = [] + if value + parts = value.split(/,\s*/) + parts.each {|part| + if m = %r{^([^\s,]+?)(?:;\s*q=(\d+(?:\.\d+)?))?$}.match(part) + val = m[1] + q = (m[2] or 1).to_f + tmp.push([val, q]) + end + } + tmp = tmp.sort_by{|val, q| -q} + tmp.collect!{|val, q| val} + end + return tmp + end + module_function :parse_qvalues + + ## + # Removes quotes and escapes from +str+ + + def dequote(str) + ret = (/\A"(.*)"\Z/ =~ str) ? $1 : str.dup + ret.gsub!(/\\(.)/, "\\1") + ret + end + module_function :dequote + + ## + # Quotes and escapes quotes in +str+ + + def quote(str) + '"' << str.gsub(/[\\\"]/o, "\\\1") << '"' + end + module_function :quote + + ## + # Stores multipart form data. FormData objects are created when + # WEBrick::HTTPUtils.parse_form_data is called. + + class FormData < String + EmptyRawHeader = [].freeze # :nodoc: + EmptyHeader = {}.freeze # :nodoc: + + ## + # The name of the form data part + + attr_accessor :name + + ## + # The filename of the form data part + + attr_accessor :filename + + attr_accessor :next_data # :nodoc: + protected :next_data + + ## + # Creates a new FormData object. + # + # +args+ is an Array of form data entries. One FormData will be created + # for each entry. + # + # This is called by WEBrick::HTTPUtils.parse_form_data for you + + def initialize(*args) + @name = @filename = @next_data = nil + if args.empty? + @raw_header = [] + @header = nil + super("") + else + @raw_header = EmptyRawHeader + @header = EmptyHeader + super(args.shift) + unless args.empty? + @next_data = self.class.new(*args) + end + end + end + + ## + # Retrieves the header at the first entry in +key+ + + def [](*key) + begin + @header[key[0].downcase].join(", ") + rescue StandardError, NameError + super + end + end + + ## + # Adds +str+ to this FormData which may be the body, a header or a + # header entry. + # + # This is called by WEBrick::HTTPUtils.parse_form_data for you + + def <<(str) + if @header + super + elsif str == CRLF + @header = HTTPUtils::parse_header(@raw_header.join) + if cd = self['content-disposition'] + if /\s+name="(.*?)"/ =~ cd then @name = $1 end + if /\s+filename="(.*?)"/ =~ cd then @filename = $1 end + end + else + @raw_header << str + end + self + end + + ## + # Adds +data+ at the end of the chain of entries + # + # This is called by WEBrick::HTTPUtils.parse_form_data for you. + + def append_data(data) + tmp = self + while tmp + unless tmp.next_data + tmp.next_data = data + break + end + tmp = tmp.next_data + end + self + end + + ## + # Yields each entry in this FormData + + def each_data + tmp = self + while tmp + next_data = tmp.next_data + yield(tmp) + tmp = next_data + end + end + + ## + # Returns all the FormData as an Array + + def list + ret = [] + each_data{|data| + ret << data.to_s + } + ret + end + + ## + # A FormData will behave like an Array + + alias :to_ary :list + + ## + # This FormData's body + + def to_s + String.new(self) + end + end + + ## + # Parses the query component of a URI in +str+ + + def parse_query(str) + query = Hash.new + if str + str.split(/[&;]/).each{|x| + next if x.empty? + key, val = x.split(/=/,2) + key = unescape_form(key) + val = unescape_form(val.to_s) + val = FormData.new(val) + val.name = key + if query.has_key?(key) + query[key].append_data(val) + next + end + query[key] = val + } + end + query + end + module_function :parse_query + + ## + # Parses form data in +io+ with the given +boundary+ + + def parse_form_data(io, boundary) + boundary_regexp = /\A--#{Regexp.quote(boundary)}(--)?#{CRLF}\z/ + form_data = Hash.new + return form_data unless io + data = nil + io.each_line{|line| + if boundary_regexp =~ line + if data + data.chop! + key = data.name + if form_data.has_key?(key) + form_data[key].append_data(data) + else + form_data[key] = data + end + end + data = FormData.new + next + else + if data + data << line + end + end + } + return form_data + end + module_function :parse_form_data + + ##### + + reserved = ';/?:@&=+$,' + num = '0123456789' + lowalpha = 'abcdefghijklmnopqrstuvwxyz' + upalpha = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' + mark = '-_.!~*\'()' + unreserved = num + lowalpha + upalpha + mark + control = (0x0..0x1f).collect{|c| c.chr }.join + "\x7f" + space = " " + delims = '<>#%"' + unwise = '{}|\\^[]`' + nonascii = (0x80..0xff).collect{|c| c.chr }.join + + module_function + + # :stopdoc: + + def _make_regex(str) /([#{Regexp.escape(str)}])/n end + def _make_regex!(str) /([^#{Regexp.escape(str)}])/n end + def _escape(str, regex) + str = str.b + str.gsub!(regex) {"%%%02X" % $1.ord} + # %-escaped string should contain US-ASCII only + str.force_encoding(Encoding::US_ASCII) + end + def _unescape(str, regex) + str = str.b + str.gsub!(regex) {$1.hex.chr} + # encoding of %-unescaped string is unknown + str + end + + UNESCAPED = _make_regex(control+space+delims+unwise+nonascii) + UNESCAPED_FORM = _make_regex(reserved+control+delims+unwise+nonascii) + NONASCII = _make_regex(nonascii) + ESCAPED = /%([0-9a-fA-F]{2})/ + UNESCAPED_PCHAR = _make_regex!(unreserved+":@&=+$,") + + # :startdoc: + + ## + # Escapes HTTP reserved and unwise characters in +str+ + + def escape(str) + _escape(str, UNESCAPED) + end + + ## + # Unescapes HTTP reserved and unwise characters in +str+ + + def unescape(str) + _unescape(str, ESCAPED) + end + + ## + # Escapes form reserved characters in +str+ + + def escape_form(str) + ret = _escape(str, UNESCAPED_FORM) + ret.gsub!(/ /, "+") + ret + end + + ## + # Unescapes form reserved characters in +str+ + + def unescape_form(str) + _unescape(str.gsub(/\+/, " "), ESCAPED) + end + + ## + # Escapes path +str+ + + def escape_path(str) + result = "" + str.scan(%r{/([^/]*)}).each{|i| + result << "/" << _escape(i[0], UNESCAPED_PCHAR) + } + return result + end + + ## + # Escapes 8 bit characters in +str+ + + def escape8bit(str) + _escape(str, NONASCII) + end + end +end diff --git a/tool/lib/webrick/httpversion.rb b/tool/lib/webrick/httpversion.rb new file mode 100644 index 0000000000..8a251944a2 --- /dev/null +++ b/tool/lib/webrick/httpversion.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: false +#-- +# HTTPVersion.rb -- presentation of HTTP version +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: httpversion.rb,v 1.5 2002/09/21 12:23:37 gotoyuzo Exp $ + +module WEBrick + + ## + # Represents an HTTP protocol version + + class HTTPVersion + include Comparable + + ## + # The major protocol version number + + attr_accessor :major + + ## + # The minor protocol version number + + attr_accessor :minor + + ## + # Converts +version+ into an HTTPVersion + + def self.convert(version) + version.is_a?(self) ? version : new(version) + end + + ## + # Creates a new HTTPVersion from +version+. + + def initialize(version) + case version + when HTTPVersion + @major, @minor = version.major, version.minor + when String + if /^(\d+)\.(\d+)$/ =~ version + @major, @minor = $1.to_i, $2.to_i + end + end + if @major.nil? || @minor.nil? + raise ArgumentError, + format("cannot convert %s into %s", version.class, self.class) + end + end + + ## + # Compares this version with +other+ according to the HTTP specification + # rules. + + def <=>(other) + unless other.is_a?(self.class) + other = self.class.new(other) + end + if (ret = @major <=> other.major) == 0 + return @minor <=> other.minor + end + return ret + end + + ## + # The HTTP version as show in the HTTP request and response. For example, + # "1.1" + + def to_s + format("%d.%d", @major, @minor) + end + end +end diff --git a/tool/lib/webrick/log.rb b/tool/lib/webrick/log.rb new file mode 100644 index 0000000000..2c1fdfe602 --- /dev/null +++ b/tool/lib/webrick/log.rb @@ -0,0 +1,156 @@ +# frozen_string_literal: false +#-- +# log.rb -- Log Class +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: log.rb,v 1.26 2002/10/06 17:06:10 gotoyuzo Exp $ + +module WEBrick + + ## + # A generic logging class + + class BasicLog + + # Fatal log level which indicates a server crash + + FATAL = 1 + + # Error log level which indicates a recoverable error + + ERROR = 2 + + # Warning log level which indicates a possible problem + + WARN = 3 + + # Information log level which indicates possibly useful information + + INFO = 4 + + # Debugging error level for messages used in server development or + # debugging + + DEBUG = 5 + + # log-level, messages above this level will be logged + attr_accessor :level + + ## + # Initializes a new logger for +log_file+ that outputs messages at +level+ + # or higher. +log_file+ can be a filename, an IO-like object that + # responds to #<< or nil which outputs to $stderr. + # + # If no level is given INFO is chosen by default + + def initialize(log_file=nil, level=nil) + @level = level || INFO + case log_file + when String + @log = File.open(log_file, "a+") + @log.sync = true + @opened = true + when NilClass + @log = $stderr + else + @log = log_file # requires "<<". (see BasicLog#log) + end + end + + ## + # Closes the logger (also closes the log device associated to the logger) + def close + @log.close if @opened + @log = nil + end + + ## + # Logs +data+ at +level+ if the given level is above the current log + # level. + + def log(level, data) + if @log && level <= @level + data += "\n" if /\n\Z/ !~ data + @log << data + end + end + + ## + # Synonym for log(INFO, obj.to_s) + def <<(obj) + log(INFO, obj.to_s) + end + + # Shortcut for logging a FATAL message + def fatal(msg) log(FATAL, "FATAL " << format(msg)); end + # Shortcut for logging an ERROR message + def error(msg) log(ERROR, "ERROR " << format(msg)); end + # Shortcut for logging a WARN message + def warn(msg) log(WARN, "WARN " << format(msg)); end + # Shortcut for logging an INFO message + def info(msg) log(INFO, "INFO " << format(msg)); end + # Shortcut for logging a DEBUG message + def debug(msg) log(DEBUG, "DEBUG " << format(msg)); end + + # Will the logger output FATAL messages? + def fatal?; @level >= FATAL; end + # Will the logger output ERROR messages? + def error?; @level >= ERROR; end + # Will the logger output WARN messages? + def warn?; @level >= WARN; end + # Will the logger output INFO messages? + def info?; @level >= INFO; end + # Will the logger output DEBUG messages? + def debug?; @level >= DEBUG; end + + private + + ## + # Formats +arg+ for the logger + # + # * If +arg+ is an Exception, it will format the error message and + # the back trace. + # * If +arg+ responds to #to_str, it will return it. + # * Otherwise it will return +arg+.inspect. + def format(arg) + if arg.is_a?(Exception) + "#{arg.class}: #{AccessLog.escape(arg.message)}\n\t" << + arg.backtrace.join("\n\t") << "\n" + elsif arg.respond_to?(:to_str) + AccessLog.escape(arg.to_str) + else + arg.inspect + end + end + end + + ## + # A logging class that prepends a timestamp to each message. + + class Log < BasicLog + # Format of the timestamp which is applied to each logged line. The + # default is <tt>"[%Y-%m-%d %H:%M:%S]"</tt> + attr_accessor :time_format + + ## + # Same as BasicLog#initialize + # + # You can set the timestamp format through #time_format + def initialize(log_file=nil, level=nil) + super(log_file, level) + @time_format = "[%Y-%m-%d %H:%M:%S]" + end + + ## + # Same as BasicLog#log + def log(level, data) + tmp = Time.now.strftime(@time_format) + tmp << " " << data + super(level, tmp) + end + end +end diff --git a/tool/lib/webrick/server.rb b/tool/lib/webrick/server.rb new file mode 100644 index 0000000000..fd6b7a61b5 --- /dev/null +++ b/tool/lib/webrick/server.rb @@ -0,0 +1,381 @@ +# frozen_string_literal: false +# +# server.rb -- GenericServer Class +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: server.rb,v 1.62 2003/07/22 19:20:43 gotoyuzo Exp $ + +require 'socket' +require_relative 'config' +require_relative 'log' + +module WEBrick + + ## + # Server error exception + + class ServerError < StandardError; end + + ## + # Base server class + + class SimpleServer + + ## + # A SimpleServer only yields when you start it + + def SimpleServer.start + yield + end + end + + ## + # A generic module for daemonizing a process + + class Daemon + + ## + # Performs the standard operations for daemonizing a process. Runs a + # block, if given. + + def Daemon.start + Process.daemon + File.umask(0) + yield if block_given? + end + end + + ## + # Base TCP server class. You must subclass GenericServer and provide a #run + # method. + + class GenericServer + + ## + # The server status. One of :Stop, :Running or :Shutdown + + attr_reader :status + + ## + # The server configuration + + attr_reader :config + + ## + # The server logger. This is independent from the HTTP access log. + + attr_reader :logger + + ## + # Tokens control the number of outstanding clients. The + # <code>:MaxClients</code> configuration sets this. + + attr_reader :tokens + + ## + # Sockets listening for connections. + + attr_reader :listeners + + ## + # Creates a new generic server from +config+. The default configuration + # comes from +default+. + + def initialize(config={}, default=Config::General) + @config = default.dup.update(config) + @status = :Stop + @config[:Logger] ||= Log::new + @logger = @config[:Logger] + + @tokens = Thread::SizedQueue.new(@config[:MaxClients]) + @config[:MaxClients].times{ @tokens.push(nil) } + + webrickv = WEBrick::VERSION + rubyv = "#{RUBY_VERSION} (#{RUBY_RELEASE_DATE}) [#{RUBY_PLATFORM}]" + @logger.info("WEBrick #{webrickv}") + @logger.info("ruby #{rubyv}") + + @listeners = [] + @shutdown_pipe = nil + unless @config[:DoNotListen] + raise ArgumentError, "Port must an integer" unless @config[:Port].to_s == @config[:Port].to_i.to_s + + @config[:Port] = @config[:Port].to_i + if @config[:Listen] + warn(":Listen option is deprecated; use GenericServer#listen", uplevel: 1) + end + listen(@config[:BindAddress], @config[:Port]) + if @config[:Port] == 0 + @config[:Port] = @listeners[0].addr[1] + end + end + end + + ## + # Retrieves +key+ from the configuration + + def [](key) + @config[key] + end + + ## + # Adds listeners from +address+ and +port+ to the server. See + # WEBrick::Utils::create_listeners for details. + + def listen(address, port) + @listeners += Utils::create_listeners(address, port) + end + + ## + # Starts the server and runs the +block+ for each connection. This method + # does not return until the server is stopped from a signal handler or + # another thread using #stop or #shutdown. + # + # If the block raises a subclass of StandardError the exception is logged + # and ignored. If an IOError or Errno::EBADF exception is raised the + # exception is ignored. If an Exception subclass is raised the exception + # is logged and re-raised which stops the server. + # + # To completely shut down a server call #shutdown from ensure: + # + # server = WEBrick::GenericServer.new + # # or WEBrick::HTTPServer.new + # + # begin + # server.start + # ensure + # server.shutdown + # end + + def start(&block) + raise ServerError, "already started." if @status != :Stop + server_type = @config[:ServerType] || SimpleServer + + setup_shutdown_pipe + + server_type.start{ + @logger.info \ + "#{self.class}#start: pid=#{$$} port=#{@config[:Port]}" + @status = :Running + call_callback(:StartCallback) + + shutdown_pipe = @shutdown_pipe + + thgroup = ThreadGroup.new + begin + while @status == :Running + begin + sp = shutdown_pipe[0] + if svrs = IO.select([sp, *@listeners]) + if svrs[0].include? sp + # swallow shutdown pipe + buf = String.new + nil while String === + sp.read_nonblock([sp.nread, 8].max, buf, exception: false) + break + end + svrs[0].each{|svr| + @tokens.pop # blocks while no token is there. + if sock = accept_client(svr) + unless config[:DoNotReverseLookup].nil? + sock.do_not_reverse_lookup = !!config[:DoNotReverseLookup] + end + th = start_thread(sock, &block) + th[:WEBrickThread] = true + thgroup.add(th) + else + @tokens.push(nil) + end + } + end + rescue Errno::EBADF, Errno::ENOTSOCK, IOError => ex + # if the listening socket was closed in GenericServer#shutdown, + # IO::select raise it. + rescue StandardError => ex + msg = "#{ex.class}: #{ex.message}\n\t#{ex.backtrace[0]}" + @logger.error msg + rescue Exception => ex + @logger.fatal ex + raise + end + end + ensure + cleanup_shutdown_pipe(shutdown_pipe) + cleanup_listener + @status = :Shutdown + @logger.info "going to shutdown ..." + thgroup.list.each{|th| th.join if th[:WEBrickThread] } + call_callback(:StopCallback) + @logger.info "#{self.class}#start done." + @status = :Stop + end + } + end + + ## + # Stops the server from accepting new connections. + + def stop + if @status == :Running + @status = :Shutdown + end + + alarm_shutdown_pipe {|f| f.write_nonblock("\0")} + end + + ## + # Shuts down the server and all listening sockets. New listeners must be + # provided to restart the server. + + def shutdown + stop + + alarm_shutdown_pipe(&:close) + end + + ## + # You must subclass GenericServer and implement \#run which accepts a TCP + # client socket + + def run(sock) + @logger.fatal "run() must be provided by user." + end + + private + + # :stopdoc: + + ## + # Accepts a TCP client socket from the TCP server socket +svr+ and returns + # the client socket. + + def accept_client(svr) + case sock = svr.to_io.accept_nonblock(exception: false) + when :wait_readable + nil + else + if svr.respond_to?(:start_immediately) + sock = OpenSSL::SSL::SSLSocket.new(sock, ssl_context) + sock.sync_close = true + # we cannot do OpenSSL::SSL::SSLSocket#accept here because + # a slow client can prevent us from accepting connections + # from other clients + end + sock + end + rescue Errno::ECONNRESET, Errno::ECONNABORTED, + Errno::EPROTO, Errno::EINVAL + nil + rescue StandardError => ex + msg = "#{ex.class}: #{ex.message}\n\t#{ex.backtrace[0]}" + @logger.error msg + nil + end + + ## + # Starts a server thread for the client socket +sock+ that runs the given + # +block+. + # + # Sets the socket to the <code>:WEBrickSocket</code> thread local variable + # in the thread. + # + # If any errors occur in the block they are logged and handled. + + def start_thread(sock, &block) + Thread.start{ + begin + Thread.current[:WEBrickSocket] = sock + begin + addr = sock.peeraddr + @logger.debug "accept: #{addr[3]}:#{addr[1]}" + rescue SocketError + @logger.debug "accept: <address unknown>" + raise + end + if sock.respond_to?(:sync_close=) && @config[:SSLStartImmediately] + WEBrick::Utils.timeout(@config[:RequestTimeout]) do + begin + sock.accept # OpenSSL::SSL::SSLSocket#accept + rescue Errno::ECONNRESET, Errno::ECONNABORTED, + Errno::EPROTO, Errno::EINVAL + Thread.exit + end + end + end + call_callback(:AcceptCallback, sock) + block ? block.call(sock) : run(sock) + rescue Errno::ENOTCONN + @logger.debug "Errno::ENOTCONN raised" + rescue ServerError => ex + msg = "#{ex.class}: #{ex.message}\n\t#{ex.backtrace[0]}" + @logger.error msg + rescue Exception => ex + @logger.error ex + ensure + @tokens.push(nil) + Thread.current[:WEBrickSocket] = nil + if addr + @logger.debug "close: #{addr[3]}:#{addr[1]}" + else + @logger.debug "close: <address unknown>" + end + sock.close + end + } + end + + ## + # Calls the callback +callback_name+ from the configuration with +args+ + + def call_callback(callback_name, *args) + @config[callback_name]&.call(*args) + end + + def setup_shutdown_pipe + return @shutdown_pipe ||= IO.pipe + end + + def cleanup_shutdown_pipe(shutdown_pipe) + @shutdown_pipe = nil + shutdown_pipe&.each(&:close) + end + + def alarm_shutdown_pipe + _, pipe = @shutdown_pipe # another thread may modify @shutdown_pipe. + if pipe + if !pipe.closed? + begin + yield pipe + rescue IOError # closed by another thread. + end + end + end + end + + def cleanup_listener + @listeners.each{|s| + if @logger.debug? + addr = s.addr + @logger.debug("close TCPSocket(#{addr[2]}, #{addr[1]})") + end + begin + s.shutdown + rescue Errno::ENOTCONN + # when `Errno::ENOTCONN: Socket is not connected' on some platforms, + # call #close instead of #shutdown. + # (ignore @config[:ShutdownSocketWithoutClose]) + s.close + else + unless @config[:ShutdownSocketWithoutClose] + s.close + end + end + } + @listeners.clear + end + end # end of GenericServer +end diff --git a/tool/lib/webrick/ssl.rb b/tool/lib/webrick/ssl.rb new file mode 100644 index 0000000000..e448095a12 --- /dev/null +++ b/tool/lib/webrick/ssl.rb @@ -0,0 +1,215 @@ +# frozen_string_literal: false +# +# ssl.rb -- SSL/TLS enhancement for GenericServer +# +# Copyright (c) 2003 GOTOU Yuuzou All rights reserved. +# +# $Id$ + +require 'webrick' +require 'openssl' + +module WEBrick + module Config + svrsoft = General[:ServerSoftware] + osslv = ::OpenSSL::OPENSSL_VERSION.split[1] + + ## + # Default SSL server configuration. + # + # WEBrick can automatically create a self-signed certificate if + # <code>:SSLCertName</code> is set. For more information on the various + # SSL options see OpenSSL::SSL::SSLContext. + # + # :ServerSoftware :: + # The server software name used in the Server: header. + # :SSLEnable :: false, + # Enable SSL for this server. Defaults to false. + # :SSLCertificate :: + # The SSL certificate for the server. + # :SSLPrivateKey :: + # The SSL private key for the server certificate. + # :SSLClientCA :: nil, + # Array of certificates that will be sent to the client. + # :SSLExtraChainCert :: nil, + # Array of certificates that will be added to the certificate chain + # :SSLCACertificateFile :: nil, + # Path to a CA certificate file + # :SSLCACertificatePath :: nil, + # Path to a directory containing CA certificates + # :SSLCertificateStore :: nil, + # OpenSSL::X509::Store used for certificate validation of the client + # :SSLTmpDhCallback :: nil, + # Callback invoked when DH parameters are required. + # :SSLVerifyClient :: + # Sets whether the client is verified. This defaults to VERIFY_NONE + # which is typical for an HTTPS server. + # :SSLVerifyDepth :: + # Number of CA certificates to walk when verifying a certificate chain + # :SSLVerifyCallback :: + # Custom certificate verification callback + # :SSLServerNameCallback:: + # Custom servername indication callback + # :SSLTimeout :: + # Maximum session lifetime + # :SSLOptions :: + # Various SSL options + # :SSLCiphers :: + # Ciphers to be used + # :SSLStartImmediately :: + # Immediately start SSL upon connection? Defaults to true + # :SSLCertName :: + # SSL certificate name. Must be set to enable automatic certificate + # creation. + # :SSLCertComment :: + # Comment used during automatic certificate creation. + + SSL = { + :ServerSoftware => "#{svrsoft} OpenSSL/#{osslv}", + :SSLEnable => false, + :SSLCertificate => nil, + :SSLPrivateKey => nil, + :SSLClientCA => nil, + :SSLExtraChainCert => nil, + :SSLCACertificateFile => nil, + :SSLCACertificatePath => nil, + :SSLCertificateStore => nil, + :SSLTmpDhCallback => nil, + :SSLVerifyClient => ::OpenSSL::SSL::VERIFY_NONE, + :SSLVerifyDepth => nil, + :SSLVerifyCallback => nil, # custom verification + :SSLTimeout => nil, + :SSLOptions => nil, + :SSLCiphers => nil, + :SSLStartImmediately => true, + # Must specify if you use auto generated certificate. + :SSLCertName => nil, + :SSLCertComment => "Generated by Ruby/OpenSSL" + } + General.update(SSL) + end + + module Utils + ## + # Creates a self-signed certificate with the given number of +bits+, + # the issuer +cn+ and a +comment+ to be stored in the certificate. + + def create_self_signed_cert(bits, cn, comment) + rsa = OpenSSL::PKey::RSA.new(bits){|p, n| + case p + when 0; $stderr.putc "." # BN_generate_prime + when 1; $stderr.putc "+" # BN_generate_prime + when 2; $stderr.putc "*" # searching good prime, + # n = #of try, + # but also data from BN_generate_prime + when 3; $stderr.putc "\n" # found good prime, n==0 - p, n==1 - q, + # but also data from BN_generate_prime + else; $stderr.putc "*" # BN_generate_prime + end + } + cert = OpenSSL::X509::Certificate.new + cert.version = 2 + cert.serial = 1 + name = (cn.kind_of? String) ? OpenSSL::X509::Name.parse(cn) + : OpenSSL::X509::Name.new(cn) + cert.subject = name + cert.issuer = name + cert.not_before = Time.now + cert.not_after = Time.now + (365*24*60*60) + cert.public_key = rsa.public_key + + ef = OpenSSL::X509::ExtensionFactory.new(nil,cert) + ef.issuer_certificate = cert + cert.extensions = [ + ef.create_extension("basicConstraints","CA:FALSE"), + ef.create_extension("keyUsage", "keyEncipherment, digitalSignature, keyAgreement, dataEncipherment"), + ef.create_extension("subjectKeyIdentifier", "hash"), + ef.create_extension("extendedKeyUsage", "serverAuth"), + ef.create_extension("nsComment", comment), + ] + aki = ef.create_extension("authorityKeyIdentifier", + "keyid:always,issuer:always") + cert.add_extension(aki) + cert.sign(rsa, "SHA256") + + return [ cert, rsa ] + end + module_function :create_self_signed_cert + end + + ## + #-- + # Updates WEBrick::GenericServer with SSL functionality + + class GenericServer + + ## + # SSL context for the server when run in SSL mode + + def ssl_context # :nodoc: + @ssl_context ||= begin + if @config[:SSLEnable] + ssl_context = setup_ssl_context(@config) + @logger.info("\n" + @config[:SSLCertificate].to_text) + ssl_context + end + end + end + + undef listen + + ## + # Updates +listen+ to enable SSL when the SSL configuration is active. + + def listen(address, port) # :nodoc: + listeners = Utils::create_listeners(address, port) + if @config[:SSLEnable] + listeners.collect!{|svr| + ssvr = ::OpenSSL::SSL::SSLServer.new(svr, ssl_context) + ssvr.start_immediately = @config[:SSLStartImmediately] + ssvr + } + end + @listeners += listeners + setup_shutdown_pipe + end + + ## + # Sets up an SSL context for +config+ + + def setup_ssl_context(config) # :nodoc: + unless config[:SSLCertificate] + cn = config[:SSLCertName] + comment = config[:SSLCertComment] + cert, key = Utils::create_self_signed_cert(2048, cn, comment) + config[:SSLCertificate] = cert + config[:SSLPrivateKey] = key + end + ctx = OpenSSL::SSL::SSLContext.new + ctx.key = config[:SSLPrivateKey] + ctx.cert = config[:SSLCertificate] + ctx.client_ca = config[:SSLClientCA] + ctx.extra_chain_cert = config[:SSLExtraChainCert] + ctx.ca_file = config[:SSLCACertificateFile] + ctx.ca_path = config[:SSLCACertificatePath] + ctx.cert_store = config[:SSLCertificateStore] + ctx.tmp_dh_callback = config[:SSLTmpDhCallback] + ctx.verify_mode = config[:SSLVerifyClient] + ctx.verify_depth = config[:SSLVerifyDepth] + ctx.verify_callback = config[:SSLVerifyCallback] + ctx.servername_cb = config[:SSLServerNameCallback] || proc { |args| ssl_servername_callback(*args) } + ctx.timeout = config[:SSLTimeout] + ctx.options = config[:SSLOptions] + ctx.ciphers = config[:SSLCiphers] + ctx + end + + ## + # ServerNameIndication callback + + def ssl_servername_callback(sslsocket, hostname = nil) + # default + end + + end +end diff --git a/tool/lib/webrick/utils.rb b/tool/lib/webrick/utils.rb new file mode 100644 index 0000000000..a96d6f03fd --- /dev/null +++ b/tool/lib/webrick/utils.rb @@ -0,0 +1,265 @@ +# frozen_string_literal: false +# +# utils.rb -- Miscellaneous utilities +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: utils.rb,v 1.10 2003/02/16 22:22:54 gotoyuzo Exp $ + +require 'socket' +require 'io/nonblock' +require 'etc' + +module WEBrick + module Utils + ## + # Sets IO operations on +io+ to be non-blocking + def set_non_blocking(io) + io.nonblock = true if io.respond_to?(:nonblock=) + end + module_function :set_non_blocking + + ## + # Sets the close on exec flag for +io+ + def set_close_on_exec(io) + io.close_on_exec = true if io.respond_to?(:close_on_exec=) + end + module_function :set_close_on_exec + + ## + # Changes the process's uid and gid to the ones of +user+ + def su(user) + if pw = Etc.getpwnam(user) + Process::initgroups(user, pw.gid) + Process::Sys::setgid(pw.gid) + Process::Sys::setuid(pw.uid) + else + warn("WEBrick::Utils::su doesn't work on this platform", uplevel: 1) + end + end + module_function :su + + ## + # The server hostname + def getservername + Socket::gethostname + end + module_function :getservername + + ## + # Creates TCP server sockets bound to +address+:+port+ and returns them. + # + # It will create IPV4 and IPV6 sockets on all interfaces. + def create_listeners(address, port) + unless port + raise ArgumentError, "must specify port" + end + sockets = Socket.tcp_server_sockets(address, port) + sockets = sockets.map {|s| + s.autoclose = false + ts = TCPServer.for_fd(s.fileno) + s.close + ts + } + return sockets + end + module_function :create_listeners + + ## + # Characters used to generate random strings + RAND_CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + + "0123456789" + + "abcdefghijklmnopqrstuvwxyz" + + ## + # Generates a random string of length +len+ + def random_string(len) + rand_max = RAND_CHARS.bytesize + ret = "" + len.times{ ret << RAND_CHARS[rand(rand_max)] } + ret + end + module_function :random_string + + ########### + + require "timeout" + require "singleton" + + ## + # Class used to manage timeout handlers across multiple threads. + # + # Timeout handlers should be managed by using the class methods which are + # synchronized. + # + # id = TimeoutHandler.register(10, Timeout::Error) + # begin + # sleep 20 + # puts 'foo' + # ensure + # TimeoutHandler.cancel(id) + # end + # + # will raise Timeout::Error + # + # id = TimeoutHandler.register(10, Timeout::Error) + # begin + # sleep 5 + # puts 'foo' + # ensure + # TimeoutHandler.cancel(id) + # end + # + # will print 'foo' + # + class TimeoutHandler + include Singleton + + ## + # Mutex used to synchronize access across threads + TimeoutMutex = Thread::Mutex.new # :nodoc: + + ## + # Registers a new timeout handler + # + # +time+:: Timeout in seconds + # +exception+:: Exception to raise when timeout elapsed + def TimeoutHandler.register(seconds, exception) + at = Process.clock_gettime(Process::CLOCK_MONOTONIC) + seconds + instance.register(Thread.current, at, exception) + end + + ## + # Cancels the timeout handler +id+ + def TimeoutHandler.cancel(id) + instance.cancel(Thread.current, id) + end + + def self.terminate + instance.terminate + end + + ## + # Creates a new TimeoutHandler. You should use ::register and ::cancel + # instead of creating the timeout handler directly. + def initialize + TimeoutMutex.synchronize{ + @timeout_info = Hash.new + } + @queue = Thread::Queue.new + @watcher = nil + end + + # :nodoc: + private \ + def watch + to_interrupt = [] + while true + now = Process.clock_gettime(Process::CLOCK_MONOTONIC) + wakeup = nil + to_interrupt.clear + TimeoutMutex.synchronize{ + @timeout_info.each {|thread, ary| + next unless ary + ary.each{|info| + time, exception = *info + if time < now + to_interrupt.push [thread, info.object_id, exception] + elsif !wakeup || time < wakeup + wakeup = time + end + } + } + } + to_interrupt.each {|arg| interrupt(*arg)} + if !wakeup + @queue.pop + elsif (wakeup -= now) > 0 + begin + (th = Thread.start {@queue.pop}).join(wakeup) + ensure + th&.kill&.join + end + end + @queue.clear + end + end + + # :nodoc: + private \ + def watcher + (w = @watcher)&.alive? and return w # usual case + TimeoutMutex.synchronize{ + (w = @watcher)&.alive? and next w # pathological check + @watcher = Thread.start(&method(:watch)) + } + end + + ## + # Interrupts the timeout handler +id+ and raises +exception+ + def interrupt(thread, id, exception) + if cancel(thread, id) && thread.alive? + thread.raise(exception, "execution timeout") + end + end + + ## + # Registers a new timeout handler + # + # +time+:: Timeout in seconds + # +exception+:: Exception to raise when timeout elapsed + def register(thread, time, exception) + info = nil + TimeoutMutex.synchronize{ + (@timeout_info[thread] ||= []) << (info = [time, exception]) + } + @queue.push nil + watcher + return info.object_id + end + + ## + # Cancels the timeout handler +id+ + def cancel(thread, id) + TimeoutMutex.synchronize{ + if ary = @timeout_info[thread] + ary.delete_if{|info| info.object_id == id } + if ary.empty? + @timeout_info.delete(thread) + end + return true + end + return false + } + end + + ## + def terminate + TimeoutMutex.synchronize{ + @timeout_info.clear + @watcher&.kill&.join + } + end + end + + ## + # Executes the passed block and raises +exception+ if execution takes more + # than +seconds+. + # + # If +seconds+ is zero or nil, simply executes the block + def timeout(seconds, exception=Timeout::Error) + return yield if seconds.nil? or seconds.zero? + # raise ThreadError, "timeout within critical session" if Thread.critical + id = TimeoutHandler.register(seconds, exception) + begin + yield(seconds) + ensure + TimeoutHandler.cancel(id) + end + end + module_function :timeout + end +end diff --git a/tool/lib/webrick/version.rb b/tool/lib/webrick/version.rb new file mode 100644 index 0000000000..b62988bdbb --- /dev/null +++ b/tool/lib/webrick/version.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: false +#-- +# version.rb -- version and release date +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2000 TAKAHASHI Masayoshi, GOTOU YUUZOU +# Copyright (c) 2003 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: version.rb,v 1.74 2003/07/22 19:20:43 gotoyuzo Exp $ + +module WEBrick + + ## + # The WEBrick version + + VERSION = "1.7.0" +end diff --git a/tool/lib/zombie_hunter.rb b/tool/lib/zombie_hunter.rb new file mode 100644 index 0000000000..33bc467941 --- /dev/null +++ b/tool/lib/zombie_hunter.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module ZombieHunter + def after_teardown + super + assert_empty(Process.waitall) + end +end + +Test::Unit::TestCase.include ZombieHunter diff --git a/tool/ln_sr.rb b/tool/ln_sr.rb new file mode 100644 index 0000000000..2aa8391e17 --- /dev/null +++ b/tool/ln_sr.rb @@ -0,0 +1,131 @@ +#!/usr/bin/ruby + +target_directory = true +noop = false +force = false +quiet = false + +until ARGV.empty? + case ARGV[0] + when '-n' + noop = true + when '-f' + force = true + when '-T' + target_directory = false + when '-q' + quiet = true + else + break + end + ARGV.shift +end + +unless ARGV.size == 2 + abort "usage: #{$0} src destdir" +end +src, dest = ARGV + +require 'fileutils' + +include FileUtils +unless respond_to?(:ln_sr) + def ln_sr(src, dest, target_directory: true, force: nil, noop: nil, verbose: nil) + options = "#{force ? 'f' : ''}#{target_directory ? '' : 'T'}" + dest = File.path(dest) + srcs = Array(src) + link = proc do |s, target_dir_p = true| + s = File.path(s) + if target_dir_p + d = File.join(destdirs = dest, File.basename(s)) + else + destdirs = File.dirname(d = dest) + end + destdirs = fu_split_path(File.realpath(destdirs)) + if fu_starting_path?(s) + srcdirs = fu_split_path((File.realdirpath(s) rescue File.expand_path(s))) + base = fu_relative_components_from(srcdirs, destdirs) + s = File.join(*base) + else + srcdirs = fu_clean_components(*fu_split_path(s)) + base = fu_relative_components_from(fu_split_path(Dir.pwd), destdirs) + while srcdirs.first&. == ".." and base.last&.!=("..") and !fu_starting_path?(base.last) + srcdirs.shift + base.pop + end + s = File.join(*base, *srcdirs) + end + fu_output_message "ln -s#{options} #{s} #{d}" if verbose + next if noop + remove_file d, true if force + File.symlink s, d + end + case srcs.size + when 0 + when 1 + link[srcs[0], target_directory && File.directory?(dest)] + else + srcs.each(&link) + end + end + + def fu_split_path(path) + path = File.path(path) + list = [] + until (parent, base = File.split(path); parent == path or parent == ".") + list << base + path = parent + end + list << path + list.reverse! + end + + def fu_relative_components_from(target, base) #:nodoc: + i = 0 + while target[i]&.== base[i] + i += 1 + end + Array.new(base.size-i, '..').concat(target[i..-1]) + end + + def fu_clean_components(*comp) + comp.shift while comp.first == "." + return comp if comp.empty? + clean = [comp.shift] + path = File.join(*clean, "") # ending with File::SEPARATOR + while c = comp.shift + if c == ".." and clean.last != ".." and !(fu_have_symlink? && File.symlink?(path)) + clean.pop + path.chomp!(%r((?<=\A|/)[^/]+/\z), "") + else + clean << c + path << c << "/" + end + end + clean + end + + if fu_windows? + def fu_starting_path?(path) + path&.start_with?(%r(\w:|/)) + end + else + def fu_starting_path?(path) + path&.start_with?("/") + end + end +end + +if File.respond_to?(:symlink) + if quiet and File.identical?(src, dest) + exit + end + begin + ln_sr(src, dest, verbose: true, target_directory: target_directory, force: force, noop: noop) + rescue NotImplementedError, Errno::EPERM, Errno::EACCES + else + exit + end +end + +cp_r(src, dest) diff --git a/tool/m4/_colorize_result_prepare.m4 b/tool/m4/_colorize_result_prepare.m4 new file mode 100644 index 0000000000..8439acf3ed --- /dev/null +++ b/tool/m4/_colorize_result_prepare.m4 @@ -0,0 +1,34 @@ +dnl -*- Autoconf -*- +AC_DEFUN([_COLORIZE_RESULT_PREPARE], [ + msg_checking= msg_result_yes= msg_result_no= msg_result_other= msg_reset= + AS_CASE(["x${CONFIGURE_TTY}"], + [xyes|xalways],[configure_tty=1], + [xno|xnever], [configure_tty=0], + [AS_IF([test -t 1], + [configure_tty=1], + [configure_tty=0])]) + AS_IF([test $configure_tty -eq 1], [ + msg_begin="`tput smso 2>/dev/null`" + AS_IF([test -z "$msg_begin"], [msg_begin="`tput so 2>/dev/null`"]) + AS_CASE(["$msg_begin"], ['@<:@'*m], + [msg_begin="`echo "$msg_begin" | sed ['s/[0-9]*m$//']`" + msg_checking="${msg_begin}33m" + AS_IF([test ${TEST_COLORS:+set}], [ + msg_result_yes=[`expr ":$TEST_COLORS:" : ".*:pass=\([^:]*\):"`] + msg_result_no=[`expr ":$TEST_COLORS:" : ".*:fail=\([^:]*\):"`] + msg_result_other=[`expr ":$TEST_COLORS:" : ".*:skip=\([^:]*\):"`] + ]) + msg_result_yes="${msg_begin}${msg_result_yes:-32;1}m" + msg_result_no="${msg_begin}${msg_result_no:-31;1}m" + msg_result_other="${msg_begin}${msg_result_other:-33;1}m" + msg_reset="${msg_begin}m" + ]) + AS_UNSET(msg_begin) + ]) + AS_REQUIRE_SHELL_FN([colorize_result], + [AS_FUNCTION_DESCRIBE([colorize_result], [MSG], [Colorize result])], + [AS_CASE(["$[]1"], + [yes], [_AS_ECHO([${msg_result_yes}$[]1${msg_reset}])], + [no], [_AS_ECHO([${msg_result_no}$[]1${msg_reset}])], + [_AS_ECHO([${msg_result_other}$[]1${msg_reset}])])]) +])dnl diff --git a/tool/m4/ac_msg_result.m4 b/tool/m4/ac_msg_result.m4 new file mode 100644 index 0000000000..12a3617c0d --- /dev/null +++ b/tool/m4/ac_msg_result.m4 @@ -0,0 +1,5 @@ +dnl -*- Autoconf -*- +AC_DEFUN([AC_MSG_RESULT], [dnl +{ _AS_ECHO_LOG([result: $1]) +COLORIZE_RESULT([$1]); dnl +}])dnl diff --git a/tool/m4/colorize_result.m4 b/tool/m4/colorize_result.m4 new file mode 100644 index 0000000000..83912040e5 --- /dev/null +++ b/tool/m4/colorize_result.m4 @@ -0,0 +1,9 @@ +dnl -*- Autoconf -*- +AC_DEFUN([COLORIZE_RESULT], [AC_REQUIRE([_COLORIZE_RESULT_PREPARE])dnl + AS_LITERAL_IF([$1], + [m4_case([$1], + [yes], [_AS_ECHO([${msg_result_yes}$1${msg_reset}])], + [no], [_AS_ECHO([${msg_result_no}$1${msg_reset}])], + [_AS_ECHO([${msg_result_other}$1${msg_reset}])])], + [colorize_result "$1"]) dnl +])dnl diff --git a/tool/m4/ruby_append_option.m4 b/tool/m4/ruby_append_option.m4 new file mode 100644 index 0000000000..ff828d2162 --- /dev/null +++ b/tool/m4/ruby_append_option.m4 @@ -0,0 +1,5 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_APPEND_OPTION], + [# RUBY_APPEND_OPTION($1) + AS_CASE([" [$]{$1-} "], + [*" $2 "*], [], [' '], [ $1="$2"], [ $1="[$]$1 $2"])])dnl diff --git a/tool/m4/ruby_append_options.m4 b/tool/m4/ruby_append_options.m4 new file mode 100644 index 0000000000..14213111ca --- /dev/null +++ b/tool/m4/ruby_append_options.m4 @@ -0,0 +1,7 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_APPEND_OPTIONS], + [# RUBY_APPEND_OPTIONS($1) + for rb_opt in $2; do + AS_CASE([" [$]{$1-} "], + [*" [$]{rb_opt} "*], [], [' '], [ $1="[$]{rb_opt}"], [ $1="[$]$1 [$]{rb_opt}"]) + done])dnl diff --git a/tool/m4/ruby_check_builtin_func.m4 b/tool/m4/ruby_check_builtin_func.m4 new file mode 100644 index 0000000000..40abc78ef8 --- /dev/null +++ b/tool/m4/ruby_check_builtin_func.m4 @@ -0,0 +1,10 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_CHECK_BUILTIN_FUNC], [dnl +AC_CACHE_CHECK([for $1], AS_TR_SH(rb_cv_builtin_$1), + [AC_LINK_IFELSE( + [AC_LANG_PROGRAM([int foo;], [$2;])], + [AS_TR_SH(rb_cv_builtin_$1)=yes], + [AS_TR_SH(rb_cv_builtin_$1)=no])]) +AS_IF([test "${AS_TR_SH(rb_cv_builtin_$1)}" != no], [ + AC_DEFINE(AS_TR_CPP(HAVE_BUILTIN_$1)) +])])dnl diff --git a/tool/m4/ruby_check_builtin_setjmp.m4 b/tool/m4/ruby_check_builtin_setjmp.m4 new file mode 100644 index 0000000000..05118e2243 --- /dev/null +++ b/tool/m4/ruby_check_builtin_setjmp.m4 @@ -0,0 +1,27 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_CHECK_BUILTIN_SETJMP], [ +AS_IF([test x"${ac_cv_func___builtin_setjmp}" = xyes], [ + unset ac_cv_func___builtin_setjmp +]) +AC_CACHE_CHECK(for __builtin_setjmp, ac_cv_func___builtin_setjmp, + [ + ac_cv_func___builtin_setjmp=no + for cast in "" "(void **)"; do + RUBY_WERROR_FLAG( + [AC_LINK_IFELSE([AC_LANG_PROGRAM([[@%:@include <setjmp.h> + @%:@include <stdio.h> + jmp_buf jb; + @%:@ifdef NORETURN + NORETURN(void t(void)); + @%:@endif + void t(void) {__builtin_longjmp($cast jb, 1);} + int jump(void) {(void)(__builtin_setjmp($cast jb) ? 1 : 0); return 0;}]], + [[ + void (*volatile f)(void) = t; + if (!jump()) printf("%d\n", f != 0); + ]])], + [ac_cv_func___builtin_setjmp="yes with cast ($cast)"]) + ]) + test "$ac_cv_func___builtin_setjmp" = no || break + done]) +])dnl diff --git a/tool/m4/ruby_check_printf_prefix.m4 b/tool/m4/ruby_check_printf_prefix.m4 new file mode 100644 index 0000000000..15bb4aee87 --- /dev/null +++ b/tool/m4/ruby_check_printf_prefix.m4 @@ -0,0 +1,29 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_CHECK_PRINTF_PREFIX], [ +AC_CACHE_CHECK([for printf prefix for $1], [rb_cv_pri_prefix_]AS_TR_SH($1),[ + [rb_cv_pri_prefix_]AS_TR_SH($1)=[NONE] + RUBY_WERROR_FLAG(RUBY_APPEND_OPTIONS(CFLAGS, $rb_cv_wsuppress_flags) + for pri in $2; do + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include <stdio.h> + @%:@include <stddef.h> + @%:@ifdef __GNUC__ + @%:@if defined __MINGW_PRINTF_FORMAT + @%:@define PRINTF_ARGS(decl, string_index, first_to_check) \ + decl __attribute__((format(__MINGW_PRINTF_FORMAT, string_index, first_to_check))) + @%:@else + @%:@define PRINTF_ARGS(decl, string_index, first_to_check) \ + decl __attribute__((format(printf, string_index, first_to_check))) + @%:@endif + @%:@else + @%:@define PRINTF_ARGS(decl, string_index, first_to_check) decl + @%:@endif + PRINTF_ARGS(void test_sprintf(const char*, ...), 1, 2);]], + [[printf("%]${pri}[d", (]$1[)42); + test_sprintf("%]${pri}[d", (]$1[)42);]])], + [rb_cv_pri_prefix_]AS_TR_SH($1)[=[$pri]; break]) + done)]) +AS_IF([test "[$rb_cv_pri_prefix_]AS_TR_SH($1)" != NONE], [ + AC_DEFINE_UNQUOTED([PRI_]m4_ifval($3,$3,AS_TR_CPP(m4_bpatsubst([$1],[_t$])))[_PREFIX], + "[$rb_cv_pri_prefix_]AS_TR_SH($1)") +]) +])dnl diff --git a/tool/m4/ruby_check_setjmp.m4 b/tool/m4/ruby_check_setjmp.m4 new file mode 100644 index 0000000000..6020b766b8 --- /dev/null +++ b/tool/m4/ruby_check_setjmp.m4 @@ -0,0 +1,17 @@ +dnl -*- Autoconf -*- +dnl used for AC_ARG_WITH(setjmp-type) +AC_DEFUN([RUBY_CHECK_SETJMP], [ +AC_CACHE_CHECK([for ]$1[ as a macro or function], ac_cv_func_$1, + [AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ +@%:@include <setjmp.h> +]AC_INCLUDES_DEFAULT([$3])[ +@%:@define JMPARGS_1 env +@%:@define JMPARGS_2 env,1 +@%:@define JMPARGS JMPARGS_]m4_ifval($2,2,1)[ +]], + [m4_ifval($2,$2,jmp_buf)[ env; $1(JMPARGS);]])], + ac_cv_func_$1=yes, + ac_cv_func_$1=no)] +) +AS_IF([test "$ac_cv_func_]$1[" = yes], [AC_DEFINE([HAVE_]AS_TR_CPP($1), 1)]) +])dnl diff --git a/tool/m4/ruby_check_signedness.m4 b/tool/m4/ruby_check_signedness.m4 new file mode 100644 index 0000000000..f9fbb3c088 --- /dev/null +++ b/tool/m4/ruby_check_signedness.m4 @@ -0,0 +1,5 @@ +dnl -*- Autoconf -*- +dnl RUBY_CHECK_SIGNEDNESS [typename] [if-signed] [if-unsigned] [included] +AC_DEFUN([RUBY_CHECK_SIGNEDNESS], [dnl + AC_COMPILE_IFELSE([AC_LANG_BOOL_COMPILE_TRY([AC_INCLUDES_DEFAULT([$4])], [($1)-1 > 0])], + [$3], [$2])])dnl diff --git a/tool/m4/ruby_check_sizeof.m4 b/tool/m4/ruby_check_sizeof.m4 new file mode 100644 index 0000000000..975ac6c9be --- /dev/null +++ b/tool/m4/ruby_check_sizeof.m4 @@ -0,0 +1,108 @@ +dnl -*- Autoconf -*- +dnl RUBY_CHECK_SIZEOF [typename], [maybe same size types], [macros], [include] +AC_DEFUN([RUBY_CHECK_SIZEOF], +[dnl +AS_VAR_PUSHDEF([rbcv_var], [rbcv_sizeof_var])dnl +AS_VAR_PUSHDEF([cond], [rbcv_sizeof_cond])dnl +AS_VAR_PUSHDEF([t], [rbcv_sizeof_type])dnl +AS_VAR_PUSHDEF([s], [rbcv_sizeof_size])dnl +] +[m4_bmatch([$1], [\.], [], [if test "$universal_binary" = yes; then]) +AC_CACHE_CHECK([size of $1], [AS_TR_SH([ac_cv_sizeof_$1])], [ + unset AS_TR_SH(ac_cv_sizeof_$1) + rbcv_var=" +typedef m4_bpatsubst([$1], [\..*]) ac__type_sizeof_; +static ac__type_sizeof_ *rbcv_ptr; +@%:@define AS_TR_CPP(SIZEOF_$1) sizeof((*rbcv_ptr)[]m4_bmatch([$1], [\.], .m4_bpatsubst([$1], [^[^.]*\.]))) +" + m4_ifval([$2], [test -z "${AS_TR_SH(ac_cv_sizeof_$1)+set}" && { + for t in $2; do + AC_COMPILE_IFELSE( + [AC_LANG_BOOL_COMPILE_TRY(AC_INCLUDES_DEFAULT([$4] + [$rbcv_var]), + [AS_TR_CPP(SIZEOF_$1) == sizeof($t)])], [ + AS_TR_SH(ac_cv_sizeof_$1)=AS_TR_CPP([SIZEOF_]$t) + break]) + done + }], [ + AC_COMPUTE_INT([AS_TR_SH(ac_cv_sizeof_$1)], [AS_TR_CPP(SIZEOF_$1)], + [AC_INCLUDES_DEFAULT([$4]) +$rbcv_var], + [AS_TR_SH(ac_cv_sizeof_$1)=]) + ]) + unset cond + m4_ifval([$3], [test -z "${AS_TR_SH(ac_cv_sizeof_$1)+set}" && { + for s in 32 64 128; do + for t in $3; do + cond="${cond} +@%:@${cond+el}if defined(__${t}${s}__) || defined(__${t}${s}) || defined(_${t}${s}) || defined(${t}${s})" + hdr="AC_INCLUDES_DEFAULT([$4 +@%:@if defined(__${t}${s}__) || defined(__${t}${s}) || defined(_${t}${s}) || defined(${t}${s}) +@%:@ define AS_TR_CPP(HAVE_$1) 1 +@%:@else +@%:@ define AS_TR_CPP(HAVE_$1) 0 +@%:@endif])" + AC_COMPILE_IFELSE([AC_LANG_BOOL_COMPILE_TRY([$hdr], [!AS_TR_CPP(HAVE_$1)])], [continue]) + AC_COMPILE_IFELSE([AC_LANG_BOOL_COMPILE_TRY([$hdr] + [$rbcv_var], + [AS_TR_CPP(HAVE_$1) == (AS_TR_CPP(SIZEOF_$1) == ($s / $rb_cv_char_bit))])], + [AS_TR_SH(ac_cv_sizeof_$1)="${AS_TR_SH(ac_cv_sizeof_$1)+${AS_TR_SH(ac_cv_sizeof_$1)-} }${t}${s}"; continue]) + AC_COMPILE_IFELSE([AC_LANG_BOOL_COMPILE_TRY([$hdr] +[ +@%:@if AS_TR_CPP(HAVE_$1) +$rbcv_var +@%:@else +@%:@define AS_TR_CPP(SIZEOF_$1) 0 +@%:@endif +], + [AS_TR_CPP(HAVE_$1) == (AS_TR_CPP(SIZEOF_$1) == (m4_bmatch([$2], [^[0-9][0-9]*$], [$2], [($s / $rb_cv_char_bit)])))])], + [AS_TR_SH(ac_cv_sizeof_$1)="${AS_TR_SH(ac_cv_sizeof_$1)+${AS_TR_SH(ac_cv_sizeof_$1)-} }${t}${s}m4_bmatch([$2], [^[0-9][0-9]*$], [:$2])"]) + done + done + }]) + test "${AS_TR_SH(ac_cv_sizeof_$1)@%:@@<:@1-9@:>@}" = "${AS_TR_SH(ac_cv_sizeof_$1)}" && + m4_ifval([$2][$3], + [test "${AS_TR_SH(ac_cv_sizeof_$1)@%:@SIZEOF_}" = "${AS_TR_SH(ac_cv_sizeof_$1)}" && ]){ + test "$universal_binary" = yes && cross_compiling=yes + AC_COMPUTE_INT([t], AS_TR_CPP(SIZEOF_$1), [AC_INCLUDES_DEFAULT([$4])] +[${cond+$cond +@%:@else} +$rbcv_var +${cond+@%:@endif} +@%:@ifndef AS_TR_CPP(SIZEOF_$1) +@%:@define AS_TR_CPP(SIZEOF_$1) 0 +@%:@endif], [t=0]) + test "$universal_binary" = yes && cross_compiling=$real_cross_compiling + AS_IF([test ${t-0} != 0], [ + AS_TR_SH(ac_cv_sizeof_$1)="${AS_TR_SH(ac_cv_sizeof_$1)+${AS_TR_SH(ac_cv_sizeof_$1)-} }${t}" + ]) + } + : ${AS_TR_SH(ac_cv_sizeof_$1)=0} +]) +{ + unset cond + for t in ${AS_TR_SH(ac_cv_sizeof_$1)-}; do + AS_CASE(["$t"], + [[[0-9]*|SIZEOF_*]], [ + ${cond+echo "@%:@else"} + echo "[@%:@define ]AS_TR_CPP(SIZEOF_$1) $t" + break + ], + [ + s=`expr $t : ['.*[^0-9]\([0-9][0-9]*\)$']` + AS_CASE([$t], [*:*], [t="${t%:*}"], [s=`expr $s / $rb_cv_char_bit`]) + echo "@%:@${cond+el}if defined(__${t}__) || defined(__${t}) || defined(_${t}) || defined($t)" + echo "@%:@define AS_TR_CPP(SIZEOF_$1) $s" + cond=1 + ]) + done + ${cond+echo "@%:@endif"} +} >> confdefs.h +m4_bmatch([$1], [\.], [], [else +AC_CHECK_SIZEOF([$1], 0, [$4]) +fi]) +AS_VAR_POPDEF([rbcv_var])dnl +AS_VAR_POPDEF([cond])dnl +AS_VAR_POPDEF([t])dnl +AS_VAR_POPDEF([s])dnl +])dnl diff --git a/tool/m4/ruby_check_sysconf.m4 b/tool/m4/ruby_check_sysconf.m4 new file mode 100644 index 0000000000..f554786e77 --- /dev/null +++ b/tool/m4/ruby_check_sysconf.m4 @@ -0,0 +1,13 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_CHECK_SYSCONF], [dnl +AC_CACHE_CHECK([whether _SC_$1 is supported], rb_cv_have_sc_[]m4_tolower($1), + [AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <unistd.h> + ]], + [[_SC_$1 >= 0]])], + rb_cv_have_sc_[]m4_tolower($1)=yes, + rb_cv_have_sc_[]m4_tolower($1)=no) + ]) +AS_IF([test "$rb_cv_have_sc_[]m4_tolower($1)" = yes], [ + AC_DEFINE(HAVE__SC_$1) +]) +])dnl diff --git a/tool/m4/ruby_cppoutfile.m4 b/tool/m4/ruby_cppoutfile.m4 new file mode 100644 index 0000000000..976cbb1c43 --- /dev/null +++ b/tool/m4/ruby_cppoutfile.m4 @@ -0,0 +1,18 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_CPPOUTFILE], +[AC_CACHE_CHECK(whether ${CPP} accepts -o, rb_cv_cppoutfile, +[save_CPPFLAGS="$CPPFLAGS" +CPPFLAGS='-o conftest-1.i' +rb_cv_cppoutfile=no +AC_PREPROC_IFELSE([AC_LANG_SOURCE([[test-for-cppout]])], + [grep test-for-cppout conftest-1.i > /dev/null && rb_cv_cppoutfile=yes]) +CPPFLAGS="$save_CPPFLAGS" +rm -f conftest*]) +AS_IF([test "$rb_cv_cppoutfile" = yes], [ + CPPOUTFILE='-o conftest.i' +], [test "$rb_cv_cppoutfile" = no], [ + CPPOUTFILE='> conftest.i' +], [test -n "$rb_cv_cppoutfile"], [ + CPPOUTFILE="$rb_cv_cppoutfile" +]) +AC_SUBST(CPPOUTFILE)])dnl diff --git a/tool/m4/ruby_decl_attribute.m4 b/tool/m4/ruby_decl_attribute.m4 new file mode 100644 index 0000000000..a8a73dc870 --- /dev/null +++ b/tool/m4/ruby_decl_attribute.m4 @@ -0,0 +1,45 @@ +dnl -*- Autoconf -*- +dnl RUBY_DECL_ATTRIBUTE(attrib, macroname, cachevar, condition, type, code) +AC_DEFUN([RUBY_DECL_ATTRIBUTE], [dnl +m4_ifval([$2], dnl + [AS_VAR_PUSHDEF([attrib], m4_bpatsubst([$2], [(.*)], []))], dnl + [AS_VAR_PUSHDEF([attrib], m4_toupper(m4_format(%.4s, [$5]))[_]AS_TR_CPP($1))] dnl +)dnl +m4_ifval([$3], dnl + [AS_VAR_PUSHDEF([rbcv],[$3])], dnl + [AS_VAR_PUSHDEF([rbcv],[rb_cv_]m4_format(%.4s, [$5])[_][$1])]dnl +)dnl +m4_pushdef([attrib_code],[m4_bpatsubst([$1],["],[\\"])])dnl +m4_pushdef([attrib_params],[m4_bpatsubst([$2(x)],[^[^()]*(\([^()]*\)).*],[\1])])dnl +m4_ifval([$4], [rbcv_cond=["$4"]; test "$rbcv_cond" || unset rbcv_cond]) +AC_CACHE_CHECK(for m4_ifval([$2],[m4_bpatsubst([$2], [(.*)], [])],[$1]) [$5] attribute, rbcv, dnl +[rbcv=x +RUBY_WERROR_FLAG([ +for mac in \ + "__attribute__ ((attrib_code)) x" \ + "x __attribute__ ((attrib_code))" \ + "__declspec(attrib_code) x" \ + x; do + m4_ifval([$4],mac="$mac"${rbcv_cond+" /* only if $rbcv_cond */"}) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([ + m4_ifval([$4],${rbcv_cond+[@%:@if ]$rbcv_cond}) +[@%:@define ]attrib[](attrib_params)[ $mac] +m4_ifval([$4],${rbcv_cond+[@%:@else]} +${rbcv_cond+[@%:@define ]attrib[](attrib_params)[ x]} +${rbcv_cond+[@%:@endif]}) +$6 +@%:@define mesg ("") +@%:@define san "address" + attrib[](attrib_params)[;]], [[]])], + [rbcv="$mac"; break]) +done +])]) +AS_IF([test "$rbcv" != x], [ + RUBY_DEFINE_IF(m4_ifval([$4],[${rbcv_cond}]), attrib[](attrib_params)[], $rbcv) +]) +m4_ifval([$4], [unset rbcv_cond]) dnl +m4_popdef([attrib_params])dnl +m4_popdef([attrib_code])dnl +AS_VAR_POPDEF([attrib])dnl +AS_VAR_POPDEF([rbcv])dnl +])dnl diff --git a/tool/m4/ruby_default_arch.m4 b/tool/m4/ruby_default_arch.m4 new file mode 100644 index 0000000000..03e52f7776 --- /dev/null +++ b/tool/m4/ruby_default_arch.m4 @@ -0,0 +1,11 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_DEFAULT_ARCH], [ +AC_MSG_CHECKING([arch option]) +AS_CASE([$1], + [arm64], [], + [*64], [ARCH_FLAG=-m64], + [[i[3-6]86]], [ARCH_FLAG=-m32], + [AC_MSG_ERROR(unknown target architecture: $target_archs)] + ) +AC_MSG_RESULT([$ARCH_FLAG]) +])dnl diff --git a/tool/m4/ruby_define_if.m4 b/tool/m4/ruby_define_if.m4 new file mode 100644 index 0000000000..aba55783a2 --- /dev/null +++ b/tool/m4/ruby_define_if.m4 @@ -0,0 +1,6 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_DEFINE_IF], [dnl + m4_ifval([$1], [AS_LITERAL_IF([$1], [], [test "X$1" = X || ])printf "@%:@if %s\n" "$1" >>confdefs.h]) +AC_DEFINE_UNQUOTED($2, $3)dnl + m4_ifval([$1], [AS_LITERAL_IF([$1], [], [test "X$1" = X || ])printf "@%:@endif /* %s */\n" "$1" >>confdefs.h]) +])dnl diff --git a/tool/m4/ruby_defint.m4 b/tool/m4/ruby_defint.m4 new file mode 100644 index 0000000000..e9ed68e5b8 --- /dev/null +++ b/tool/m4/ruby_defint.m4 @@ -0,0 +1,40 @@ +dnl -*- Autoconf -*- +dnl RUBY_DEFINT TYPENAME, SIZE, [UNSIGNED], [INCLUDES = DEFAULT-INCLUDES] +AC_DEFUN([RUBY_DEFINT], [dnl +AS_VAR_PUSHDEF([cond], [rb_defint_cond])dnl +AS_VAR_PUSHDEF([type], [rb_defint_type])dnl +AC_CACHE_CHECK([for $1], [rb_cv_type_$1], +[AC_COMPILE_IFELSE([AC_LANG_PROGRAM([AC_INCLUDES_DEFAULT([$4]) +typedef $1 t; int s = sizeof(t) == 42;])], + [rb_cv_type_$1=yes], + [AS_CASE([m4_bmatch([$2], [^[1-9][0-9]*$], $2, [$ac_cv_sizeof_]AS_TR_SH($2))], + ["1"], [ rb_cv_type_$1="m4_if([$3], [], [signed ], [$3 ])char"], + ["$ac_cv_sizeof_short"], [ rb_cv_type_$1="m4_if([$3], [], [], [$3 ])short"], + ["$ac_cv_sizeof_int"], [ rb_cv_type_$1="m4_if([$3], [], [], [$3 ])int"], + ["$ac_cv_sizeof_long"], [ rb_cv_type_$1="m4_if([$3], [], [], [$3 ])long"], + ["$ac_cv_sizeof_long_long"], [ rb_cv_type_$1="m4_if([$3], [], [], [$3 ])long long"], + ["${ac_cv_sizeof___int64@%:@*:}"], [ rb_cv_type_$1="m4_if([$3], [], [], [$3 ])__int64"], + ["${ac_cv_sizeof___int128@%:@*:}"], [ rb_cv_type_$1="m4_if([$3], [], [], [$3 ])__int128"], + [ rb_cv_type_$1=no])])]) +AS_IF([test "${rb_cv_type_$1}" != no], [ + type="${rb_cv_type_$1@%:@@%:@unsigned }" + AS_IF([test "$type" != yes && eval 'test -n "${ac_cv_sizeof_'$type'+set}"'], [ + eval cond='"${ac_cv_sizeof_'$type'}"' + AS_CASE([$cond], [*:*], [ + cond=AS_TR_CPP($type) + echo "@%:@if defined SIZEOF_"$cond" && SIZEOF_"$cond" > 0" >> confdefs.h + ], [cond=]) + ], [cond=]) + AC_DEFINE([HAVE_]AS_TR_CPP($1), 1) + AS_IF([test "${rb_cv_type_$1}" = yes], [ + m4_bmatch([$2], [^[1-9][0-9]*$], [AC_CHECK_SIZEOF([$1], 0, [AC_INCLUDES_DEFAULT([$4])])], + [RUBY_CHECK_SIZEOF([$1], [$2], [], [AC_INCLUDES_DEFAULT([$4])])]) + ], [ + AC_DEFINE_UNQUOTED($1, [$rb_cv_type_$1]) + AC_DEFINE_UNQUOTED([SIZEOF_]AS_TR_CPP($1), [SIZEOF_]AS_TR_CPP([$type])) + ]) + test -n "$cond" && echo "@%:@endif /* $cond */" >> confdefs.h +]) +AS_VAR_POPDEF([cond])dnl +AS_VAR_POPDEF([type])dnl +])dnl diff --git a/tool/m4/ruby_dtrace_available.m4 b/tool/m4/ruby_dtrace_available.m4 new file mode 100644 index 0000000000..e03b7762fc --- /dev/null +++ b/tool/m4/ruby_dtrace_available.m4 @@ -0,0 +1,20 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_DTRACE_AVAILABLE], +[AC_CACHE_CHECK(whether dtrace USDT is available, rb_cv_dtrace_available, +[ + echo "provider conftest{ probe fire(); };" > conftest_provider.d + rb_cv_dtrace_available=no + AS_FOR(opt, rb_dtrace_opt, ["-xnolibs" ""], [dnl + AS_IF([$DTRACE opt -h -o conftest_provider.h -s conftest_provider.d >/dev/null 2>/dev/null], + [], [continue]) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include "conftest_provider.h"]], [[CONFTEST_FIRE();]])], + [], [continue]) + # DTrace is available on the system + rb_cv_dtrace_available=yes${rb_dtrace_opt:+"(opt)"} + break + ]) + rm -f conftest.[co] conftest_provider.[dho] +]) +AS_CASE(["$rb_cv_dtrace_available"], ["yes("*")"], + [DTRACE_OPT=`expr "$rb_cv_dtrace_available" : "yes(\(.*\))"`]) +])dnl diff --git a/tool/m4/ruby_dtrace_postprocess.m4 b/tool/m4/ruby_dtrace_postprocess.m4 new file mode 100644 index 0000000000..6fd6de7c9a --- /dev/null +++ b/tool/m4/ruby_dtrace_postprocess.m4 @@ -0,0 +1,30 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_DTRACE_POSTPROCESS], +[AC_CACHE_CHECK(whether $DTRACE needs post processing, rb_cv_prog_dtrace_g, +[ + rb_cv_prog_dtrace_g=no + AS_IF([{ + cat >conftest_provider.d <<_PROBES && + provider conftest { + probe fire(); + }; +_PROBES + $DTRACE ${DTRACE_OPT} -h -o conftest_provider.h -s conftest_provider.d >/dev/null 2>/dev/null && + : + }], [ + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include "conftest_provider.h"]], [[CONFTEST_FIRE();]])],[ + AS_IF([{ + cp -p conftest.${ac_objext} conftest.${ac_objext}.save && + $DTRACE ${DTRACE_OPT} -G -s conftest_provider.d conftest.${ac_objext} 2>/dev/null && + : + }], [ + AS_IF([cmp -s conftest.o conftest.${ac_objext}.save], [ + rb_cv_prog_dtrace_g=yes + ], [ + rb_cv_prog_dtrace_g=rebuild + ]) + ])]) + ]) + rm -f conftest.[co] conftest_provider.[dho] +]) +])dnl diff --git a/tool/m4/ruby_func_attribute.m4 b/tool/m4/ruby_func_attribute.m4 new file mode 100644 index 0000000000..bce26fc16a --- /dev/null +++ b/tool/m4/ruby_func_attribute.m4 @@ -0,0 +1,7 @@ +dnl -*- Autoconf -*- +dnl RUBY_FUNC_ATTRIBUTE(attrib, macroname, cachevar, condition) +AC_DEFUN([RUBY_FUNC_ATTRIBUTE], [dnl + RUBY_DECL_ATTRIBUTE([$1], [$2], [$3], [$4], + [function], [@%:@define x int conftest_attribute_check(void)] + ) +])dnl diff --git a/tool/m4/ruby_mingw32.m4 b/tool/m4/ruby_mingw32.m4 new file mode 100644 index 0000000000..98e922340b --- /dev/null +++ b/tool/m4/ruby_mingw32.m4 @@ -0,0 +1,24 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_MINGW32], +[AS_CASE(["$host_os"], +[cygwin*], [ +AC_CACHE_CHECK(for mingw32 environment, rb_cv_mingw32, +[AC_PREPROC_IFELSE([AC_LANG_SOURCE([[ +#ifndef __MINGW32__ +# error +#endif +]])],[rb_cv_mingw32=yes],[rb_cv_mingw32=no]) +rm -f conftest*]) +AS_IF([test "$rb_cv_mingw32" = yes], [ + target_os="mingw32" + : ${ac_tool_prefix:="`expr "$CC" : ['\(.*-\)g\?cc[^/]*$']`"} + AC_DEFINE(__USE_MINGW_ANSI_STDIO, 1) dnl for gnu_printf +]) +]) +AS_CASE(["$target_os"], [mingw*msvc], [ +target_os="`echo ${target_os} | sed 's/msvc$//'`" +]) +AS_CASE(["$target_cpu-$target_os"], [x86_64-mingw*], [ +target_cpu=x64 +]) +])dnl diff --git a/tool/m4/ruby_prepend_option.m4 b/tool/m4/ruby_prepend_option.m4 new file mode 100644 index 0000000000..3b7030a473 --- /dev/null +++ b/tool/m4/ruby_prepend_option.m4 @@ -0,0 +1,5 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_PREPEND_OPTION], + [# RUBY_PREPEND_OPTION($1) + AS_CASE([" [$]{$1-} "], + [*" $2 "*], [], [' '], [ $1="$2"], [ $1="$2 [$]$1"])])dnl diff --git a/tool/m4/ruby_prog_gnu_ld.m4 b/tool/m4/ruby_prog_gnu_ld.m4 new file mode 100644 index 0000000000..b38fb3d527 --- /dev/null +++ b/tool/m4/ruby_prog_gnu_ld.m4 @@ -0,0 +1,10 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_PROG_GNU_LD], +[AC_CACHE_CHECK(whether the linker is GNU ld, rb_cv_prog_gnu_ld, +[AS_IF([`$CC $CFLAGS $CPPFLAGS $LDFLAGS --print-prog-name=ld 2>&1` -v 2>&1 | grep "GNU ld" > /dev/null], [ + rb_cv_prog_gnu_ld=yes +], [ + rb_cv_prog_gnu_ld=no +])]) +GNU_LD=$rb_cv_prog_gnu_ld +AC_SUBST(GNU_LD)])dnl diff --git a/tool/m4/ruby_replace_funcs.m4 b/tool/m4/ruby_replace_funcs.m4 new file mode 100644 index 0000000000..10e85f1ac9 --- /dev/null +++ b/tool/m4/ruby_replace_funcs.m4 @@ -0,0 +1,13 @@ +dnl -*- Autoconf -*- +dnl RUBY_REPLACE_FUNC [func] [included] +AC_DEFUN([RUBY_REPLACE_FUNC], [dnl + AC_CHECK_DECL([$1],dnl + [AC_DEFINE(AS_TR_CPP(HAVE_[$1]))],dnl + [AC_REPLACE_FUNCS($1)],dnl + [$2])dnl +])dnl +dnl +dnl RUBY_REPLACE_FUNCS [funcs] [included] +AC_DEFUN([RUBY_REPLACE_FUNCS] [dnl + m4_map_args_w([$1], [RUBY_REPLACE_FUNC(], [), [$2]])dnl +])dnl diff --git a/tool/m4/ruby_replace_type.m4 b/tool/m4/ruby_replace_type.m4 new file mode 100644 index 0000000000..70674b6cc7 --- /dev/null +++ b/tool/m4/ruby_replace_type.m4 @@ -0,0 +1,58 @@ +dnl -*- Autoconf -*- +dnl RUBY_REPLACE_TYPE [typename] [default type] [macro type] [included] +AC_DEFUN([RUBY_REPLACE_TYPE], [dnl + AC_CHECK_TYPES([$1], + [n="patsubst([$1],["],[\\"])"], + [n="patsubst([$2],["],[\\"])"], + [$4]) + AC_CACHE_CHECK([for convertible type of [$1]], rb_cv_[$1]_convertible, [ + u= t= + AS_CASE(["$n "], + [*" signed "*], [ ], + [*" unsigned "*], [ + u=U], + [RUBY_CHECK_SIGNEDNESS($n, [], [u=U], [$4])]) + AS_IF([test x"$t" = x], [ + for t in "long long" long int short; do + test -n "$u" && t="unsigned $t" + AC_COMPILE_IFELSE( + [AC_LANG_BOOL_COMPILE_TRY([AC_INCLUDES_DEFAULT([$4])] + [typedef $n rbcv_conftest_target_type; + typedef $t rbcv_conftest_replace_type; + extern rbcv_conftest_target_type rbcv_conftest_var; + extern rbcv_conftest_replace_type rbcv_conftest_var; + extern rbcv_conftest_target_type rbcv_conftest_func(void); + extern rbcv_conftest_replace_type rbcv_conftest_func(void); + ], [sizeof(rbcv_conftest_target_type) == sizeof(rbcv_conftest_replace_type)])], + [n="$t"; break]) + done + ]) + AS_CASE([" $n "], + [*" long long "*], [ + t=LL], + [*" long "*], [ + t=LONG], + [*" short "*], [ + t=SHORT], + [ + t=INT]) + rb_cv_[$1]_convertible=${u}${t}]) + AS_IF([test "${AS_TR_SH(ac_cv_type_[$1])}" = "yes"], [ + n="$1" + ], [ + AS_CASE(["${rb_cv_[$1]_convertible}"], + [*LL], [n="long long"], + [*LONG], [n="long"], + [*SHORT], [n="short"], + [n="int"]) + AS_CASE(["${rb_cv_[$1]_convertible}"], + [U*], [n="unsigned $n"]) + ]) + AS_CASE("${rb_cv_[$1]_convertible}", [U*], [u=+1], [u=-1]) + AC_DEFINE_UNQUOTED(rb_[$1], $n) + AC_DEFINE_UNQUOTED([SIGNEDNESS_OF_]AS_TR_CPP($1), $u) + AC_DEFINE_UNQUOTED([$3]2NUM[(v)], [${rb_cv_[$1]_convertible}2NUM(v)]) + AC_DEFINE_UNQUOTED(NUM2[$3][(v)], [NUM2${rb_cv_[$1]_convertible}(v)]) + AC_DEFINE_UNQUOTED(PRI_[$3]_PREFIX, + [PRI_`echo ${rb_cv_[$1]_convertible} | sed ['s/^U//']`_PREFIX]) +])dnl diff --git a/tool/m4/ruby_rm_recursive.m4 b/tool/m4/ruby_rm_recursive.m4 new file mode 100644 index 0000000000..b97701f88e --- /dev/null +++ b/tool/m4/ruby_rm_recursive.m4 @@ -0,0 +1,18 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_RM_RECURSIVE], [dnl +m4_version_prereq([2.70], [], [dnl +# suppress error messages, rm: cannot remove 'conftest.dSYM', from +# AC_EGREP_CPP with CFLAGS=-g on Darwin. +AS_CASE([$build_os], [darwin*], [ +rm() { + rm_recursive='' + for arg do + AS_CASE("$arg", + [--*], [], + [-*r*], [break], + [conftest.*], [AS_IF([test -d "$arg"], [rm_recursive=-r; break])], + []) + done + command rm $rm_recursive "[$]@" +} +])])])dnl diff --git a/tool/m4/ruby_setjmp_type.m4 b/tool/m4/ruby_setjmp_type.m4 new file mode 100644 index 0000000000..4ae26fe5cd --- /dev/null +++ b/tool/m4/ruby_setjmp_type.m4 @@ -0,0 +1,52 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_SETJMP_TYPE], [ +RUBY_CHECK_BUILTIN_SETJMP +RUBY_CHECK_SETJMP(_setjmpex, [], [@%:@include <setjmpex.h>]) +RUBY_CHECK_SETJMP(_setjmp) +RUBY_CHECK_SETJMP(sigsetjmp, [sigjmp_buf]) +AC_MSG_CHECKING(for setjmp type) +setjmp_suffix= +unset setjmp_sigmask +AC_ARG_WITH(setjmp-type, + AS_HELP_STRING([--with-setjmp-type], [select setjmp type]), + [ + AS_CASE([$withval], + [__builtin_setjmp], [setjmp=__builtin_setjmp], + [_setjmp], [ setjmp_prefix=_], + [sigsetjmp,*], [ setjmp_prefix=sig setjmp_sigmask=`expr "$withval" : 'sigsetjmp\(,.*\)'`], + [sigsetjmp], [ setjmp_prefix=sig], + [setjmp], [ setjmp_prefix=], + [setjmpex], [ setjmp_prefix= setjmp_suffix=ex], + [''], [ unset setjmp_prefix], + [ AC_MSG_ERROR(invalid setjmp type: $withval)])], [unset setjmp_prefix]) +setjmp_cast= +AS_IF([test ${setjmp_prefix+set}], [ + AS_IF([test "${setjmp_prefix}" && eval test '$ac_cv_func_'${setjmp_prefix}setjmp${setjmp_suffix} = no], [ + AC_MSG_ERROR(${setjmp_prefix}setjmp${setjmp_suffix} is not available) + ]) +], [{ AS_CASE("$ac_cv_func___builtin_setjmp", [yes*], [true], [false]) }], [ + setjmp_cast=`expr "$ac_cv_func___builtin_setjmp" : "yes with cast (\(.*\))"` + setjmp_prefix=__builtin_ + setjmp_suffix= +], [test "$ac_cv_header_setjmpex_h:$ac_cv_func__setjmpex" = yes:yes], [ + setjmp_prefix= + setjmp_suffix=ex +], [test "$ac_cv_func__setjmp" = yes], [ + setjmp_prefix=_ + setjmp_suffix= +], [test "$ac_cv_func_sigsetjmp" = yes], [ + AS_CASE([$target_os],[solaris*|cygwin*],[setjmp_prefix=],[setjmp_prefix=sig]) + setjmp_suffix= +], [ + setjmp_prefix= + setjmp_suffix= +]) +AS_IF([test x$setjmp_prefix:$setjmp_sigmask = xsig:], [ + setjmp_sigmask=,0 +]) +AC_MSG_RESULT(${setjmp_prefix}setjmp${setjmp_suffix}${setjmp_cast:+\($setjmp_cast\)}${setjmp_sigmask}) +AC_DEFINE_UNQUOTED([RUBY_SETJMP(env)], [${setjmp_prefix}setjmp${setjmp_suffix}($setjmp_cast(env)${setjmp_sigmask})]) +AC_DEFINE_UNQUOTED([RUBY_LONGJMP(env,val)], [${setjmp_prefix}longjmp($setjmp_cast(env),val)]) +AS_IF([test "(" "$GCC" != yes ")" -o x$setjmp_prefix != x__builtin_], AC_DEFINE_UNQUOTED(RUBY_JMP_BUF, ${setjmp_sigmask+${setjmp_prefix}}jmp_buf)) +AS_IF([test x$setjmp_suffix = xex], [AC_DEFINE_UNQUOTED(RUBY_USE_SETJMPEX, 1)]) +])dnl diff --git a/tool/m4/ruby_stack_grow_direction.m4 b/tool/m4/ruby_stack_grow_direction.m4 new file mode 100644 index 0000000000..a4d205cc3c --- /dev/null +++ b/tool/m4/ruby_stack_grow_direction.m4 @@ -0,0 +1,30 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_STACK_GROW_DIRECTION], [ + AS_VAR_PUSHDEF([stack_grow_dir], [rb_cv_stack_grow_dir_$1]) + AC_CACHE_CHECK(stack growing direction on $1, stack_grow_dir, [ +AS_CASE(["$1"], +[m68*|x86*|x64|i?86|ppc*|sparc*|alpha*], [ $2=-1], +[hppa*], [ $2=+1], +[ + AC_RUN_IFELSE([AC_LANG_SOURCE([[ +/* recurse to get rid of inlining */ +static int +stack_growup_p(addr, n) + volatile int *addr, n; +{ + volatile int end; + if (n > 0) + return *addr = stack_growup_p(addr, n - 1); + else + return (&end > addr); +} +int main() +{ + int x; + return stack_growup_p(&x, 10); +} +]])],[$2=-1],[$2=+1],[$2=0]) + ]) +eval stack_grow_dir=\$$2]) +eval $2=\$stack_grow_dir +AS_VAR_POPDEF([stack_grow_dir])])dnl diff --git a/tool/m4/ruby_thread.m4 b/tool/m4/ruby_thread.m4 new file mode 100644 index 0000000000..3831bc4c06 --- /dev/null +++ b/tool/m4/ruby_thread.m4 @@ -0,0 +1,33 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_THREAD], [ +AC_ARG_WITH(thread, + AS_HELP_STRING([--with-thread=IMPLEMENTATION], [specify the thread implementation to use]), + [THREAD_MODEL=$withval], [ + THREAD_MODEL= + AS_CASE(["$target_os"], + [mingw*], [ + THREAD_MODEL=win32 + ], + [ + AS_IF([test "$rb_with_pthread" = "yes"], [ + THREAD_MODEL=pthread + ]) + ] + ) +]) + +AS_CASE(["$THREAD_MODEL"], +[pthread], [AC_CHECK_HEADERS(pthread.h)], +[win32], [], +[""], [AC_MSG_ERROR(thread model is missing)], + [AC_MSG_ERROR(unknown thread model $THREAD_MODEL)]) + +THREAD_IMPL_H=thread_$THREAD_MODEL.h +AS_IF([test ! -f "$srcdir/$THREAD_IMPL_H"], + [AC_MSG_ERROR('$srcdir/$THREAD_IMPL_H' must exist)]) +THREAD_IMPL_SRC=thread_$THREAD_MODEL.c +AS_IF([test ! -f "$srcdir/$THREAD_IMPL_SRC"], + [AC_MSG_ERROR('$srcdir/$THREAD_IMPL_SRC' must exist)]) +AC_DEFINE_UNQUOTED(THREAD_IMPL_H, ["$THREAD_IMPL_H"]) +AC_DEFINE_UNQUOTED(THREAD_IMPL_SRC, ["$THREAD_IMPL_SRC"]) +])dnl diff --git a/tool/m4/ruby_try_cflags.m4 b/tool/m4/ruby_try_cflags.m4 new file mode 100644 index 0000000000..f2c6a3094e --- /dev/null +++ b/tool/m4/ruby_try_cflags.m4 @@ -0,0 +1,12 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_TRY_CFLAGS], [ + AC_MSG_CHECKING([whether ]$1[ is accepted as CFLAGS]) + RUBY_WERROR_FLAG([ + CFLAGS="[$]CFLAGS $1" + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[$4]], [[$5]])], + [$2 + AC_MSG_RESULT(yes)], + [$3 + AC_MSG_RESULT(no)]) + ]) +])dnl diff --git a/tool/m4/ruby_try_cxxflags.m4 b/tool/m4/ruby_try_cxxflags.m4 new file mode 100644 index 0000000000..06f645f546 --- /dev/null +++ b/tool/m4/ruby_try_cxxflags.m4 @@ -0,0 +1,17 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_TRY_CXXFLAGS], [ + save_CXXFLAGS="$CXXFLAGS" + CXXFLAGS="[$]CXXFLAGS $1" + AC_MSG_CHECKING([whether ]$1[ is accepted as CXXFLAGS]) + RUBY_WERROR_FLAG([ + AC_LANG_PUSH([C++]) + AC_LINK_IFELSE([AC_LANG_PROGRAM([[$4]], [[$5]])], + [$2 + AC_MSG_RESULT(yes)], + [$3 + AC_MSG_RESULT(no)]) + ]) + AC_LANG_POP([C++]) + CXXFLAGS="$save_CXXFLAGS" + save_CXXFLAGS= +])dnl diff --git a/tool/m4/ruby_try_ldflags.m4 b/tool/m4/ruby_try_ldflags.m4 new file mode 100644 index 0000000000..c3a6be0fb3 --- /dev/null +++ b/tool/m4/ruby_try_ldflags.m4 @@ -0,0 +1,15 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_TRY_LDFLAGS], [ + save_LDFLAGS="$LDFLAGS" + LDFLAGS="[$]LDFLAGS $1" + AC_MSG_CHECKING([whether $1 is accepted as LDFLAGS]) + RUBY_WERROR_FLAG([ + AC_LINK_IFELSE([AC_LANG_PROGRAM([[$4]], [[$5]])], + [$2 + AC_MSG_RESULT(yes)], + [$3 + AC_MSG_RESULT(no)]) + ]) + LDFLAGS="$save_LDFLAGS" + save_LDFLAGS= +])dnl diff --git a/tool/m4/ruby_type_attribute.m4 b/tool/m4/ruby_type_attribute.m4 new file mode 100644 index 0000000000..5ea1219c6e --- /dev/null +++ b/tool/m4/ruby_type_attribute.m4 @@ -0,0 +1,8 @@ +dnl -*- Autoconf -*- +dnl RUBY_TYPE_ATTRIBUTE(attrib, macroname, cachevar, condition) +AC_DEFUN([RUBY_TYPE_ATTRIBUTE], [dnl + RUBY_DECL_ATTRIBUTE([$1], [$2], [$3], [$4], + [type], [ +@%:@define x struct conftest_attribute_check {int i;} +]) +])dnl diff --git a/tool/m4/ruby_universal_arch.m4 b/tool/m4/ruby_universal_arch.m4 new file mode 100644 index 0000000000..375cdd98d2 --- /dev/null +++ b/tool/m4/ruby_universal_arch.m4 @@ -0,0 +1,122 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_UNIVERSAL_ARCH], [ +# RUBY_UNIVERSAL_ARCH begin +ARCH_FLAG=`expr " $CXXFLAGS " : ['.* \(-m[0-9][0-9]*\) ']` +test ${CXXFLAGS+set} && CXXFLAGS=`echo "$CXXFLAGS" | sed [-e 's/ *-arch *[^ ]*//g' -e 's/ *-m32//g' -e 's/ *-m64//g']` +ARCH_FLAG=`expr " $CFLAGS " : ['.* \(-m[0-9][0-9]*\) ']` +test ${CFLAGS+set} && CFLAGS=`echo "$CFLAGS" | sed [-e 's/ *-arch *[^ ]*//g' -e 's/ *-m32//g' -e 's/ *-m64//g']` +test ${LDFLAGS+set} && LDFLAGS=`echo "$LDFLAGS" | sed [-e 's/ *-arch *[^ ]*//g' -e 's/ *-m32//g' -e 's/ *-m64//g']` +unset universal_binary universal_archnames +AS_IF([test ${target_archs+set}], [ + AC_MSG_CHECKING([target architectures]) + target_archs=`echo $target_archs | tr , ' '` + # /usr/lib/arch_tool -archify_list $TARGET_ARCHS + for archs in $target_archs + do + AS_CASE([",$universal_binary,"],[*",$archs,"*], [],[ + cpu=$archs + cpu=`echo $cpu | sed 's/-.*-.*//'` + universal_binary="${universal_binary+$universal_binary,}$cpu" + universal_archnames="${universal_archnames} ${archs}=${cpu}" + ARCH_FLAG="${ARCH_FLAG+$ARCH_FLAG }-arch $archs" + ]) + done + target_archs="$universal_binary" + unset universal_binary + AS_CASE(["$target_archs"], + [*,*], [universal_binary=yes], + [unset universal_archnames]) + AC_MSG_RESULT([$target_archs]) + + target=`echo $target | sed "s/^$target_cpu-/-/"` + target_alias=`echo $target_alias | sed "s/^$target_cpu-/-/"` + AS_IF([test "${universal_binary-no}" = yes], [ + AC_SUBST(try_header,try_compile) + target_cpu=universal + real_cross_compiling=$cross_compiling + ], [ + AS_IF([test x"$target_cpu" != x"${target_archs}"], [ + echo 'int main(){return 0;}' > conftest.c + AS_IF([$CC $CFLAGS $ARCH_FLAG -o conftest conftest.c > /dev/null 2>&1], [ + rm -fr conftest.* + ], [test -z "$ARCH_FLAG"], [ + RUBY_DEFAULT_ARCH("$target_archs") + ]) + ]) + target_cpu=${target_archs} + ]) + AS_CASE(["$target"], [-*], [ target="$target_cpu${target}"]) + AS_CASE(["$target_alias"], [-*], [ target_alias="$target_cpu${target_alias}"]) +], [ + AS_IF([test x"$target_alias" = x], [ + AS_CASE(["$target_os"], + [darwin*], [ + AC_MSG_CHECKING([for real target cpu]) + target=`echo $target | sed "s/^$target_cpu-/-/"` + target_cpu=`$CC -E - 2>/dev/null <<EOF | +#ifdef __x86_64__ +"processor-name=x86_64" +#endif +#ifdef __i386__ +"processor-name=i386" +#endif +#ifdef __ppc__ +"processor-name=powerpc" +#endif +#ifdef __ppc64__ +"processor-name=powerpc64" +#endif +#ifdef __arm64__ +"processor-name=arm64" +#endif +EOF + sed -n 's/^"processor-name=\(.*\)"/\1/p'` + target="$target_cpu${target}" + AC_MSG_RESULT([$target_cpu]) + ]) + ]) + target_archs="$target_cpu" +]) +AS_IF([test "${target_archs}" != "${rb_cv_target_archs-${target_archs}}"], [ + AC_MSG_ERROR([target arch(s) has changed from ${rb_cv_target_archs-nothing} to ${target_archs}]) +], [ + rb_cv_target_archs=${target_archs} +]) +AS_IF([test "x${ARCH_FLAG}" != x], [ + CFLAGS="${CFLAGS:+$CFLAGS }${ARCH_FLAG}" + LDFLAGS="${LDFLAGS:+$LDFLAGS }${ARCH_FLAG}" +]) +# RUBY_UNIVERSAL_ARCH end +])dnl +dnl +AC_DEFUN([RUBY_UNIVERSAL_CHECK_HEADER_COND], [ dnl + AC_CACHE_CHECK([for $2 when $1], [$3], + AC_COMPILE_IFELSE([AC_LANG_PROGRAM( + [AC_INCLUDES_DEFAULT([$6])[ + @%:@if ]$1[ + @%:@include <]$2[> + @%:@endif]], [[]])], + [AS_VAR_SET($3, yes)], + [AS_VAR_SET($3, no)])) + AS_VAR_IF([$3], [yes], [dnl + printf "@%:@if %s\n" "$1" >>confdefs.h + AC_DEFINE_UNQUOTED(HAVE_[]AS_TR_CPP($2), 1)dnl + printf "@%:@endif\n" >>confdefs.h dnl + $4], [$5]) +])dnl +dnl +# RUBY_UNIVERSAL_CHECK_HEADER(CPU-LIST, HEADER, +# [ACTION-IF-FOUND], [ACTION-IF-NOT-FOUND], +# [INCLUDES = DEFAULT-INCLUDES]) +AC_DEFUN([RUBY_UNIVERSAL_CHECK_HEADER], [ dnl + m4_if([$# dnl + ], [0], [], [ dnl + m4_foreach([rb_Header], [$1], + [AS_CASE([",$target_archs,"], [*,]rb_Header[,*], + [RUBY_UNIVERSAL_CHECK_HEADER_COND]([defined(__[]rb_Header[]__)], + [$2], [rb_cv_header_[]AS_TR_SH($2)_on_[]AS_TR_SH(rb_Header)], + [$3], [$4], [$5]) + ) + ]) + ])dnl +])dnl diff --git a/tool/m4/ruby_werror_flag.m4 b/tool/m4/ruby_werror_flag.m4 new file mode 100644 index 0000000000..616a7f6abf --- /dev/null +++ b/tool/m4/ruby_werror_flag.m4 @@ -0,0 +1,18 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_WERROR_FLAG], [dnl +save_CFLAGS="$CFLAGS" +CFLAGS="$CFLAGS $rb_cv_warnflags" +AS_IF([test "${ac_c_werror_flag+set}"], [ + rb_c_werror_flag="$ac_c_werror_flag" +], [ + unset rb_c_werror_flag +]) +ac_c_werror_flag=yes +$1 +CFLAGS="$save_CFLAGS" +save_CFLAGS= +AS_IF([test "${rb_c_werror_flag+set}"], [ + ac_c_werror_flag="$rb_c_werror_flag" +], [ + unset ac_c_werror_flag +])])dnl diff --git a/tool/make-snapshot b/tool/make-snapshot new file mode 100755 index 0000000000..942e85b933 --- /dev/null +++ b/tool/make-snapshot @@ -0,0 +1,654 @@ +#!/usr/bin/ruby -s +# -*- coding: us-ascii -*- +require 'rubygems' +require 'rubygems/package' +require 'rubygems/package/tar_writer' +require 'uri' +require 'digest/sha1' +require 'digest/sha2' +require 'fileutils' +require 'shellwords' +require 'tmpdir' +require 'pathname' +require 'yaml' +require 'json' +require File.expand_path("../lib/vcs", __FILE__) +require File.expand_path("../lib/colorize", __FILE__) +STDOUT.sync = true + +$srcdir ||= nil +$archname = nil if ($archname ||= nil) == "" +$keep_temp ||= nil +$patch_file ||= nil +$packages ||= nil +$digests ||= nil +$tooldir = File.expand_path("..", __FILE__) +$unicode_version = nil if ($unicode_version ||= nil) == "" +$colorize = Colorize.new + +def usage + <<USAGE +usage: #{File.basename $0} [option...] new-directory-to-save [version ...] +options: + -srcdir=PATH source directory path + -archname=NAME make the basename of snapshots NAME + -keep_temp keep temporary working directory + -patch_file=PATCH apply PATCH file after export + -packages=PKG[,...] make PKG packages (#{PACKAGES.keys.join(", ")}) + -digests=ALG[,...] show ALG digests (#{DIGESTS.join(", ")}) + -unicode_version=VER Unicode version to generate encodings + -svn[=URL] make snapshot from SVN repository + (#{SVNURL}) + -help, --help show this message +version: + master, trunk, stable, branches/*, tags/*, X.Y, X.Y.Z, X.Y.Z-pL +each versions may be followed by optional @revision. +USAGE +end + +DIGESTS = %w[SHA1 SHA256 SHA512] +PACKAGES = { + "tar" => %w".tar", + "bzip" => %w".tar.bz2 bzip2 -c", + "gzip" => %w".tar.gz gzip -c", + "xz" => %w".tar.xz xz -c", + "zip" => %w".zip zip -Xqr", +} +DEFAULT_PACKAGES = PACKAGES.keys - ["tar"] +if !$no7z and system("7z", out: IO::NULL) + PACKAGES["gzip"] = %w".tar.gz 7z a dummy -tgzip -mx -so" + PACKAGES["zip"] = %w".zip 7z a -tzip -mx -mtc=off" << {out: IO::NULL} +elsif gzip = ENV.delete("GZIP") + PACKAGES["gzip"].concat(gzip.shellsplit) +end + +if mflags = ENV["GNUMAKEFLAGS"] and /\A-(\S*)j\d*/ =~ mflags + mflags = mflags.gsub(/(\A|\s)(-\S*)j\d*/, '\1\2') + mflags.strip! + ENV["GNUMAKEFLAGS"] = (mflags unless mflags.empty?) +end +ENV["LC_ALL"] = ENV["LANG"] = "C" +SVNURL = URI.parse("https://svn.ruby-lang.org/repos/ruby/") +# https git clone is disabled at git.ruby-lang.org/ruby.git. +GITURL = URI.parse("https://github.com/ruby/ruby.git") +RUBY_VERSION_PATTERN = /^\#define\s+RUBY_VERSION\s+"([\d.]+)"/ + +ENV["VPATH"] ||= "include/ruby" +YACC = ENV["YACC"] ||= "bison" +ENV["BASERUBY"] ||= "ruby" +ENV["RUBY"] ||= "ruby" +ENV["MV"] ||= "mv" +ENV["RM"] ||= "rm -f" +ENV["MINIRUBY"] ||= "ruby" +ENV["PROGRAM"] ||= "ruby" +ENV["AUTOCONF"] ||= "autoconf" +ENV["BUILTIN_TRANSOBJS"] ||= "newline.o" +ENV["TZ"] = "UTC" + +class String + # for older ruby + alias bytesize size unless method_defined?(:bytesize) +end + +class Dir + def self.mktmpdir(path) + path = File.join(tmpdir, path+"-#{$$}-#{rand(100000)}") + begin + mkdir(path) + rescue Errno::EEXIST + path.succ! + retry + end + path + end unless respond_to?(:mktmpdir) +end + +$packages &&= $packages.split(/[, ]+/).tap {|pkg| + if all = pkg.index("all") + pkg[all, 1] = DEFAULT_PACKAGES - pkg + end + pkg -= PACKAGES.keys + pkg.empty? or abort "#{File.basename $0}: unknown packages - #{pkg.join(", ")}" +} +$packages ||= DEFAULT_PACKAGES + +$digests &&= $digests.split(/[, ]+/).tap {|dig| + dig -= DIGESTS + dig.empty? or abort "#{File.basename $0}: unknown digests - #{dig.join(", ")}" +} +$digests ||= DIGESTS + +$patch_file &&= File.expand_path($patch_file) +path = ENV["PATH"].split(File::PATH_SEPARATOR) +%w[YACC BASERUBY RUBY MV MINIRUBY].each do |var| + cmd, = ENV[var].shellsplit + unless path.any? {|dir| + file = File.expand_path(cmd, dir) + File.file?(file) and File.executable?(file) + } + abort "#{File.basename $0}: #{var} command not found - #{cmd}" + end +end + +%w[BASERUBY RUBY MINIRUBY].each do |var| + %x[#{ENV[var]} --disable-gem -e1 2>&1] + if $?.success? + ENV[var] += ' --disable-gem' + end +end + +if defined?($help) or defined?($_help) + puts usage + exit +end +unless destdir = ARGV.shift + abort usage +end +revisions = ARGV.empty? ? [nil] : ARGV + +if $exported + abort "#{File.basename $0}: -exported option is deprecated; use -srcdir instead" +end + +FileUtils.mkpath(destdir) +destdir = File.expand_path(destdir) +tmp = Dir.mktmpdir("ruby-snapshot") +FileUtils.mkpath(tmp) +at_exit { + Dir.chdir "/" + FileUtils.rm_rf(tmp) +} unless $keep_temp + +def tar_create(tarball, dir) + header = Gem::Package::TarHeader + dir_type = "5" + uname = gname = "ruby" + File.open(tarball, "wb") do |f| + w = Gem::Package::TarWriter.new(f) + list = Dir.glob("#{dir}/**/*", File::FNM_DOTMATCH) + list.reject! {|name| name.end_with?("/.")} + list.sort_by! {|name| name.split("/")} + list.each do |path| + next if File.basename(path) == "." + s = File.stat(path) + mode = 0644 + case + when s.file? + type = nil + size = s.size + mode |= 0111 if s.executable? + when s.directory? + path += "/" + type = dir_type + size = 0 + mode |= 0111 + else + next + end + name, prefix = w.split_name(path) + h = header.new(name: name, prefix: prefix, typeflag: type, + mode: mode, size: size, mtime: s.mtime, + uname: uname, gname: gname) + f.write(h) + if size > 0 + IO.copy_stream(path, f) + f.write("\0" * (-size % 512)) + end + end + end + true +rescue => e + warn e.message + false +end + +def touch_all(time, pattern, opt, &cond) + Dir.glob(pattern, opt) do |n| + stat = File.stat(n) + if stat.file? or stat.directory? + next if cond and !yield(n, stat) + File.utime(time, time, n) + end + end +rescue + false +else + true +end + +class MAKE < Struct.new(:prog, :args) + def initialize(vars) + vars = vars.map {|arg| arg.join("=")} + super(ENV["MAKE"] || ENV["make"] || "make", vars) + end + + def run(target) + err = IO.pipe do |r, w| + begin + pid = Process.spawn(self.prog, *self.args, target, {:err => w, r => :close}) + w.close + r.read + ensure + Process.wait(pid) + end + end + if $?.success? + true + else + STDERR.puts err + $colorize.fail("#{target} failed") + false + end + end +end + +def measure + clock = Process::CLOCK_MONOTONIC + t0 = Process.clock_gettime(clock) + STDOUT.flush + result = yield + printf(" %6.3f", Process.clock_gettime(clock) - t0) + STDOUT.flush + result +end + +def package(vcs, rev, destdir, tmp = nil) + pwd = Dir.pwd + patchlevel = false + prerelease = false + if rev and revision = rev[/@(\h+)\z/, 1] + rev = $` + end + case rev + when nil + url = nil + when /\A(?:master|trunk)\z/ + url = vcs.trunk + when /\Abranches\// + url = vcs.branch($') + when /\Atags\// + url = vcs.tag($') + when /\Astable\z/ + vcs.branch_list("ruby_[0-9]*") {|n| url = n[/\Aruby_\d+_\d+\z/]} + url &&= vcs.branch(url) + when /\A(.*)\.(.*)\.(.*)-(preview|rc)(\d+)/ + prerelease = true + tag = "#{$4}#{$5}" + url = vcs.tag("v#{$1}_#{$2}_#{$3}_#{$4}#{$5}") + when /\A(.*)\.(.*)\.(.*)-p(\d+)/ + patchlevel = true + tag = "p#{$4}" + url = vcs.tag("v#{$1}_#{$2}_#{$3}_#{$4}") + when /\A(\d+)\.(\d+)(?:\.(\d+))?\z/ + if $3 && ($1 > "2" || $1 == "2" && $2 >= "1") + patchlevel = true + tag = "" + url = vcs.tag("v#{$1}_#{$2}_#{$3}") + else + url = vcs.branch("ruby_#{rev.tr('.', '_')}") + end + else + warn "#{$0}: unknown version - #{rev}" + return + end + if info = vcs.get_revisions(url) + modified = info[2] + else + modified = Time.now - 10 + end + if !revision and info + revision = info + url ||= vcs.branch(revision[3]) + revision = revision[1] + end + version = nil + unless revision + url = vcs.trunk + vcs.grep(RUBY_VERSION_PATTERN, url, "version.h") {version = $1} + unless rev == version + warn "#{$0}: #{rev} not found" + return + end + revision = vcs.get_revisions(url)[1] + end + + v = "ruby" + puts "Exporting #{rev}@#{revision}" + exported = tmp ? File.join(tmp, v) : v + unless vcs.export(revision, url, exported, true) {|line| print line} + warn("Export failed") + return + end + if $srcdir + Dir.glob($srcdir + "/{tool/config.{guess,sub},gems/*.gem,.downloaded-cache/*,enc/unicode/data/**/*.txt}") do |file| + puts "copying #{file}" if $VERBOSE + dest = exported + file[$srcdir.size..-1] + FileUtils.mkpath(File.dirname(dest)) + begin + FileUtils.cp_r(file, dest) + FileUtils.chmod_R("a+rwX,go-w", dest) + rescue SystemCallError + end + end + end + + status = IO.read(File.dirname(__FILE__) + "/prereq.status") + Dir.chdir(tmp) if tmp + + if !File.directory?(v) + v = Dir.glob("ruby-*").select(&File.method(:directory?)) + v.size == 1 or abort "#{File.basename $0}: not exported" + v = v[0] + end + + File.open("#{v}/revision.h", "wb") {|f| + short = vcs.short_revision(revision) + f.puts "#define RUBY_REVISION #{short.inspect}" + unless short == revision + f.puts "#define RUBY_FULL_REVISION #{revision.inspect}" + end + } + version ||= (versionhdr = IO.read("#{v}/version.h"))[RUBY_VERSION_PATTERN, 1] + version ||= + begin + include_ruby_versionhdr = IO.read("#{v}/include/ruby/version.h") + api_major_version = include_ruby_versionhdr[/^\#define\s+RUBY_API_VERSION_MAJOR\s+([\d.]+)/, 1] + api_minor_version = include_ruby_versionhdr[/^\#define\s+RUBY_API_VERSION_MINOR\s+([\d.]+)/, 1] + version_teeny = versionhdr[/^\#define\s+RUBY_VERSION_TEENY\s+(\d+)/, 1] + [api_major_version, api_minor_version, version_teeny].join('.') + end + version or return + if patchlevel + unless tag.empty? + versionhdr ||= IO.read("#{v}/version.h") + patchlevel = versionhdr[/^\#define\s+RUBY_PATCHLEVEL\s+(\d+)/, 1] + tag = (patchlevel ? "p#{patchlevel}" : vcs.revision_name(revision)) + end + elsif prerelease + versionhdr ||= IO.read("#{v}/version.h") + versionhdr.sub!(/^\#define\s+RUBY_PATCHLEVEL_STR\s+"\K.+?(?=")/, tag) + IO.write("#{v}/version.h", versionhdr) + else + tag ||= vcs.revision_name(revision) + end + + if $archname + n = $archname + elsif tag.empty? + n = "ruby-#{version}" + else + n = "ruby-#{version}-#{tag}" + end + File.directory?(n) or File.rename v, n + v = n + + if $patch_file && !system(*%W"patch -d #{v} -p0 -i #{$patch_file}") + puts $colorize.fail("patching failed") + return + end + def (clean = []).add(n) push(n); n end + def clean.create(file, content = "") File.binwrite(add(file), content) end + Dir.chdir(v) do + unless File.exist?("ChangeLog") + vcs.export_changelog(url, nil, revision, "ChangeLog") + end + + unless touch_all(modified, "**/*", File::FNM_DOTMATCH) + modified = nil + colors = %w[red yellow green cyan blue magenta] + "take a breath, and go ahead".scan(/./) do |c| + if c == ' ' + print c + else + colors.push(color = colors.shift) + print $colorize.decorate(c, color) + end + sleep(c == "," ? 0.7 : 0.05) + end + puts + end + + File.open(clean.add("cross.rb"), "w") do |f| + f.puts "Object.__send__(:remove_const, :CROSS_COMPILING) if defined?(CROSS_COMPILING)" + f.puts "CROSS_COMPILING=true" + f.puts "Object.__send__(:remove_const, :RUBY_PLATFORM)" + f.puts "RUBY_PLATFORM='none'" + f.puts "Object.__send__(:remove_const, :RUBY_VERSION)" + f.puts "RUBY_VERSION='#{version}'" + end + unless File.exist?("configure") + print "creating configure..." + unless system([ENV["AUTOCONF"]]*2) + puts $colorize.fail(" failed") + return + end + puts $colorize.pass(" done") + end + clean.add("autom4te.cache") + clean.add("enc/unicode/data") + print "creating prerequisites..." + if File.file?("common.mk") && /^prereq/ =~ commonmk = IO.read("common.mk") + puts + extout = clean.add('tmp') + begin + status = IO.read("tool/prereq.status") + rescue Errno::ENOENT + # use fallback file + end + clean.create("config.status", status) + clean.create("noarch-fake.rb", "require_relative 'cross'\n") + FileUtils.mkpath(hdrdir = "#{extout}/include/ruby") + File.binwrite("#{hdrdir}/config.h", "") + FileUtils.mkpath(defaults = "#{extout}/rubygems/defaults") + File.binwrite("#{defaults}/operating_system.rb", "") + File.binwrite("#{defaults}/ruby.rb", "") + miniruby = ENV['MINIRUBY'] + " -I. -I#{extout} -rcross" + baseruby = ENV["BASERUBY"] + mk = (IO.read("template/Makefile.in") rescue IO.read("Makefile.in")). + gsub(/^@.*\n/, '') + vars = { + "EXTOUT"=>extout, + "PATH_SEPARATOR"=>File::PATH_SEPARATOR, + "MINIRUBY"=>miniruby, + "RUBY"=>ENV["RUBY"], + "BASERUBY"=>baseruby, + "PWD"=>Dir.pwd, + "ruby_version"=>version, + "MAJOR"=>api_major_version, + "MINOR"=>api_minor_version, + "TEENY"=>version_teeny, + } + status.scan(/^s([%,])@([A-Za-z_][A-Za-z_0-9]*)@\1(.*?)\1g$/) do + vars[$2] ||= $3 + end + vars.delete("UNICODE_FILES") # for stable branches + vars["UNICODE_VERSION"] = $unicode_version if $unicode_version + args = vars.dup + mk.gsub!(/@([A-Za-z_]\w*)@/) {args.delete($1); vars[$1] || ENV[$1]} + mk << commonmk.gsub(/\{\$([^(){}]*)[^{}]*\}/, "").sub(/^revision\.tmp::$/, '\& Makefile') + mk << <<-'APPEND' + +update-download:: touch-unicode-files +prepare-package: prereq after-update +clean-cache: $(CLEAN_CACHE) +after-update:: extract-gems +extract-gems: update-gems +update-gems: +$(UNICODE_SRC_DATA_DIR)/.unicode-tables.time: +touch-unicode-files: + APPEND + clean.create("Makefile", mk) + clean.create("revision.tmp") + clean.create(".revision.time") + ENV["CACHE_SAVE"] = "no" + make = MAKE.new(args) + return unless make.run("update-download") + clean.push("rbconfig.rb", ".rbconfig.time", "enc.mk", "ext/ripper/y.output", ".revision.time") + Dir.glob("**/*") do |dest| + next unless File.symlink?(dest) + orig = File.expand_path(File.readlink(dest), File.dirname(dest)) + File.unlink(dest) + FileUtils.cp_r(orig, dest) + end + File.utime(modified, modified, *Dir.glob(["tool/config.{guess,sub}", "gems/*.gem", "tool"])) + return unless make.run("prepare-package") + return unless make.run("clean-cache") + if modified + new_time = modified + 2 + touch_all(new_time, "**/*", File::FNM_DOTMATCH) do |name, stat| + stat.mtime > modified unless clean.include?(name) + end + modified = new_time + end + print "prerequisites" + else + system(*%W"#{YACC} -o parse.c parse.y") + end + vcs.after_export(".") if exported + clean.concat(Dir.glob("ext/**/autom4te.cache")) + FileUtils.rm_rf(clean) unless $keep_temp + FileUtils.rm_rf(".downloaded-cache") + if File.exist?("gems/bundled_gems") + gems = Dir.glob("gems/*.gem") + gems -= File.readlines("gems/bundled_gems").map {|line| + next if /^\s*(?:#|$)/ =~ line + name, version, _ = line.split(' ') + "gems/#{name}-#{version}.gem" + } + FileUtils.rm_f(gems) + else + FileUtils.rm_rf("gems") + end + if modified + touch_all(modified, "**/*/", 0) do |name, stat| + stat.mtime > modified + end + File.utime(modified, modified, ".") + end + unless $?.success? + puts $colorize.fail(" failed") + return + end + puts $colorize.pass(" done") + end + + if v == "." + v = File.basename(Dir.pwd) + Dir.chdir ".." + else + Dir.chdir(File.dirname(v)) + v = File.basename(v) + end + + tarball = nil + return $packages.collect do |mesg| + (ext, *cmd) = PACKAGES[mesg] + File.directory?(destdir) or FileUtils.mkpath(destdir) + file = File.join(destdir, "#{$archname||v}#{ext}") + case ext + when /\.tar/ + if tarball + next if tarball.empty? + else + tarball = ext == ".tar" ? file : "#{$archname||v}.tar" + print "creating tarball... #{tarball}" + if measure {tar_create(tarball, v)} + puts $colorize.pass(" done") + File.utime(modified, modified, tarball) if modified + next if tarball == file + else + puts $colorize.fail(" failed") + tarball = "" + next + end + end + print "creating #{mesg} tarball... #{file}" + done = measure {system(*cmd, tarball, out: file)} + else + print "creating #{mesg} archive... #{file}" + if Hash === cmd.last + *cmd, opt = *cmd + cmd << file << v << opt + else + (cmd = cmd.dup) << file << v + end + done = measure {system(*cmd)} + end + if done + puts $colorize.pass(" done") + file + else + puts $colorize.fail(" failed") + nil + end + end.compact +ensure + FileUtils.rm_rf(tmp ? File.join(tmp, v) : v) if v and !$keep_temp + Dir.chdir(pwd) +end + +if [$srcdir, ($svn||=nil), ($git||=nil)].compact.size > 1 + abort "#{File.basename $0}: -srcdir, -svn, and -git are exclusive" +end +if $srcdir + vcs = VCS.detect($srcdir) +elsif $svn + vcs = VCS::SVN.new($svn == true ? SVNURL : URI.parse($svn)) +elsif $git + abort "#{File.basename $0}: use -srcdir with cloned local repository" +else + begin + vcs = VCS.detect(File.expand_path("../..", __FILE__)) + rescue VCS::NotFoundError + vcs = VCS::SVN.new(SVNURL) + end +end + +release_date = Time.now.getutc +info = {} + +success = true +revisions.collect {|rev| package(vcs, rev, destdir, tmp)}.flatten.each do |name| + if !name + success = false + next + end + str = File.binread(name) + pathname = Pathname(name) + basename = pathname.basename.to_s + extname = pathname.extname.sub(/\A\./, '') + version = basename[/\Aruby-(.*)\.(?:tar|zip)/, 1] + key = basename[/\A(.*)\.(?:tar|zip)/, 1] + info[key] ||= Hash.new{|h,k|h[k]={}} + info[key]['version'] = version if version + info[key]['date'] = release_date.strftime('%Y-%m-%d') + if version + info[key]['post'] = "/en/news/#{release_date.strftime('%Y/%m/%d')}/ruby-#{version.tr('.', '-')}-released/" + info[key]['url'][extname] = "https://cache.ruby-lang.org/pub/ruby/#{version[/\A\d+\.\d+/]}/#{basename}" + else + info[key]['filename'][extname] = basename + end + info[key]['size'][extname] = str.bytesize + puts "* #{$colorize.pass(name)}" + puts " SIZE: #{str.bytesize} bytes" + $digests.each do |alg| + digest = Digest(alg).hexdigest(str) + info[key][alg.downcase][extname] = digest + printf " %-8s%s\n", "#{alg}:", digest + end +end + +yaml = info.values.to_yaml +json = info.values.to_json +puts "#{$colorize.pass('YAML:')}" +puts yaml +puts "#{$colorize.pass('JSON:')}" +puts json +infodir = Pathname(destdir) + 'info' +infodir.mkpath +(infodir+'info.yml').write(yaml) +(infodir+'info.json').write(json) + +exit false if !success + +# vim:fileencoding=US-ASCII sw=2 ts=4 noexpandtab ff=unix diff --git a/tool/make_hgraph.rb b/tool/make_hgraph.rb new file mode 100644 index 0000000000..0f388814dd --- /dev/null +++ b/tool/make_hgraph.rb @@ -0,0 +1,95 @@ +# +# Make dot file of internal class/module hierarchy graph. +# + +require 'objspace' + +module ObjectSpace + def self.object_id_of obj + if obj.kind_of?(ObjectSpace::InternalObjectWrapper) + obj.internal_object_id + else + obj.object_id + end + end + + T_ICLASS_NAME = {} + + def self.class_name_of klass + case klass + when Class, Module + # (singleton class).name returns nil + klass.name || klass.inspect + when InternalObjectWrapper # T_ICLASS + if klass.type == :T_ICLASS + "#<I:#{class_name_of(ObjectSpace.internal_class_of(klass))}>" + else + klass.inspect + end + else + klass.inspect + end + end + + def self.module_refenreces klass + h = {} # object_id -> [klass, class_of, super] + stack = [klass] + while klass = stack.pop + obj_id = ObjectSpace.object_id_of(klass) + next if h.has_key?(obj_id) + cls = ObjectSpace.internal_class_of(klass) + sup = ObjectSpace.internal_super_of(klass) + stack << cls if cls + stack << sup if sup + h[obj_id] = [klass, cls, sup].map{|e| ObjectSpace.class_name_of(e)} + end + h.values + end + + def self.module_refenreces_dot klass + result = [] + rank_set = {} + + result << "digraph mod_h {" + # result << " rankdir=LR;" + module_refenreces(klass).each{|(m, k, s)| + # next if /singleton/ =~ m + result << "#{m.dump} -> #{s.dump} [label=\"super\"];" + result << "#{m.dump} -> #{k.dump} [label=\"klass\"];" + + unless rank = rank_set[m] + rank = rank_set[m] = 0 + end + unless rank_set[s] + rank_set[s] = rank + 1 + end + unless rank_set[k] + rank_set[k] = rank + end + } + + rs = [] # [[mods...], ...] + rank_set.each{|m, r| + rs[r] = [] unless rs[r] + rs[r] << m + } + + rs.each{|ms| + result << "{rank = same; #{ms.map{|m| m.dump}.join(", ")}};" + } + result << "}" + result.join("\n") + end + + def self.module_refenreces_image klass, file + dot = module_refenreces_dot(klass) + img = nil + IO.popen("dot -Tpng", 'r+'){|io| + # + io.puts dot + io.close_write + img = io.read + } + open(File.expand_path(file), 'w+'){|f| f.puts img} + end +end diff --git a/tool/mdoc2man.rb b/tool/mdoc2man.rb new file mode 100755 index 0000000000..e005fcf19a --- /dev/null +++ b/tool/mdoc2man.rb @@ -0,0 +1,505 @@ +#!/usr/bin/env ruby +### +### mdoc2man - mdoc to man converter +### +### Quick usage: mdoc2man.rb < mdoc_manpage.8 > man_manpage.8 +### +### Ported from Perl by Akinori MUSHA. +### +### Copyright (c) 2001 University of Illinois Board of Trustees +### Copyright (c) 2001 Mark D. Roth +### Copyright (c) 2002, 2003 Akinori MUSHA +### All rights reserved. +### +### Redistribution and use in source and binary forms, with or without +### modification, are permitted provided that the following conditions +### are met: +### 1. Redistributions of source code must retain the above copyright +### notice, this list of conditions and the following disclaimer. +### 2. Redistributions in binary form must reproduce the above copyright +### notice, this list of conditions and the following disclaimer in the +### documentation and/or other materials provided with the distribution. +### 3. All advertising materials mentioning features or use of this software +### must display the following acknowledgement: +### This product includes software developed by the University of +### Illinois at Urbana, and their contributors. +### 4. The University nor the names of their +### contributors may be used to endorse or promote products derived from +### this software without specific prior written permission. +### +### THIS SOFTWARE IS PROVIDED BY THE TRUSTEES AND CONTRIBUTORS ``AS IS'' AND +### ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +### IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +### ARE DISCLAIMED. IN NO EVENT SHALL THE TRUSTEES OR CONTRIBUTORS BE LIABLE +### FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +### DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +### OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +### HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +### LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +### OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +### SUCH DAMAGE. +### +### $Id$ +### + +class Mdoc2Man + ANGLE = 1 + OPTION = 2 + PAREN = 3 + + RE_PUNCT = /^[!"'),\.\/:;>\?\]`]$/ + + def initialize + @name = @date = @id = nil + @refauthors = @reftitle = @refissue = @refdate = @refopt = nil + + @optlist = 0 ### 1 = bullet, 2 = enum, 3 = tag, 4 = item + @oldoptlist = 0 + @nospace = 0 ### 0, 1, 2 + @enum = 0 + @synopsis = true + @reference = false + @ext = false + @extopt = false + @literal = false + end + + def mdoc2man(i, o) + i.each { |line| + if /^\./ !~ line + o.print line + o.print ".br\n" if @literal + next + end + + line.slice!(0, 1) + + next if /\\"/ =~ line + + line = parse_macro(line) and o.print line + } + + initialize + end + + def shift_arg(words) + case words[0] + when nil, RE_PUNCT + nil + when /\A"(.+)/ + words.shift + word = $1 + loop { + break if word.chomp!('"') + token = words.shift or break + word << ' ' << token + } + word + else + words.shift + end + end + + def parse_macro(line) + words = line.split + retval = '' + + quote = [] + dl = false + + while word = words.shift + case word + when RE_PUNCT + next retval << word if word == ':' + while q = quote.pop + case q + when OPTION + retval << ']' + when PAREN + retval << ')' + when ANGLE + retval << '>' + end + end + retval << word + next + when 'Li', 'Pf' + @nospace = 1 + next + when 'Xo' + @ext = true + retval << ' ' unless retval.empty? || /[\n ]\z/ =~ retval + next + when 'Xc' + @ext = false + retval << "\n" unless @extopt + break + when 'Bd' + @literal = true if words[0] == '-literal' + retval << "\n" + break + when 'Ed' + @literal = false + break + when 'Ns' + @nospace = 1 if @nospace == 0 + retval.chomp!(' ') + next + when 'No' + retval.chomp!(' ') + retval << words.shift + next + when 'Dq' + retval << '``' + begin + retval << words.shift << ' ' + end until words.empty? || RE_PUNCT =~ words[0] + retval.chomp!(' ') + retval << '\'\'' + @nospace = 1 if @nospace == 0 && RE_PUNCT =~ words[0] + next + when 'Sq', 'Ql' + retval << '`' << words.shift << '\'' + @nospace = 1 if @nospace == 0 && RE_PUNCT =~ words[0] + next + # when 'Ic' + # retval << '\\fB' << words.shift << '\\fP' + # next + when 'Oo' + #retval << "[\\c\n" + @extopt = true + @nospace = 1 if @nospace == 0 + retval << '[' + next + when 'Oc' + @extopt = false + retval << ']' + next + when 'Ao' + @nospace = 1 if @nospace == 0 + retval << '<' + next + when 'Ac' + retval << '>' + next + end + + retval << ' ' if @nospace == 0 && !(retval.empty? || /[\n ]\z/ =~ retval) + @nospace = 0 if @nospace == 1 + + case word + when 'Dd' + @date = words.join(' ') + return nil + when 'Dt' + if words.size >= 2 && words[1] == '""' && + /^(.*)\(([0-9])\)$/ =~ words[0] + words[0] = $1 + words[1] = $2 + end + @id = words.join(' ') + return nil + when 'Os' + retval << '.TH ' << @id << ' "' << @date << '" "' << + words.join(' ') << '"' + break + when 'Sh' + retval << '.SH' + @synopsis = (words[0] == 'SYNOPSIS') + next + when 'Xr' + retval << '\\fB' << words.shift << + '\\fP(' << words.shift << ')' << (words.shift||'') + break + when 'Rs' + @refauthors = [] + @reftitle = '' + @refissue = '' + @refdate = '' + @refopt = '' + @reference = true + break + when 'Re' + retval << "\n" + + # authors + while @refauthors.size > 1 + retval << @refauthors.shift << ', ' + end + retval << 'and ' unless retval.empty? + retval << @refauthors.shift + + # title + retval << ', \\fI' << @reftitle << '\\fP' + + # issue + retval << ', ' << @refissue unless @refissue.empty? + + # date + retval << ', ' << @refdate unless @refdate.empty? + + # optional info + retval << ', ' << @refopt unless @refopt.empty? + + retval << ".\n" + + @reference = false + break + when 'An' + next + when 'Dl' + retval << ".nf\n" << '\\& ' + dl = true + next + when 'Ux' + retval << "UNIX" + next + when 'Bro' + retval << '{' + @nospace = 1 if @nospace == 0 + next + when 'Brc' + retval.sub!(/ *\z/, '}') + next + end + + if @reference + case word + when '%A' + @refauthors.unshift(words.join(' ')) + break + when '%T' + @reftitle = words.join(' ') + @reftitle.sub!(/^"/, '') + @reftitle.sub!(/"$/, '') + break + when '%N' + @refissue = words.join(' ') + break + when '%D' + @refdate = words.join(' ') + break + when '%O' + @refopt = words.join(' ') + break + end + end + + case word + when 'Nm' + name = words.empty? ? @name : words.shift + @name ||= name + retval << ".br\n" if @synopsis + retval << "\\fB" << name << "\\fP" + @nospace = 1 if @nospace == 0 && RE_PUNCT =~ words[0] + next + when 'Nd' + retval << '\\-' + next + when 'Fl' + retval << '\\fB\\-' << words.shift << '\\fP' + @nospace = 1 if @nospace == 0 && RE_PUNCT =~ words[0] + next + when 'Ar' + retval << '\\fI' + if words.empty? + retval << 'file ...\\fP' + else + retval << words.shift << '\\fP' + while words[0] == '|' + retval << ' ' << words.shift << ' \\fI' << words.shift << '\\fP' + end + @nospace = 1 if @nospace == 0 && RE_PUNCT =~ words[0] + next + end + when 'Cm' + retval << '\\fB' << words.shift << '\\fP' + while RE_PUNCT =~ words[0] + retval << words.shift + end + next + when 'Op' + quote << OPTION + @nospace = 1 if @nospace == 0 + retval << '[' + # words.push(words.pop + ']') + next + when 'Aq' + quote << ANGLE + @nospace = 1 if @nospace == 0 + retval << '<' + # words.push(words.pop + '>') + next + when 'Pp' + retval << "\n" + next + when 'Ss' + retval << '.SS' + next + end + + case word + when 'Pa' + if !quote.include?(OPTION) + retval << '\\fI' + retval << '\\&' if /^\./ =~ words[0] + retval << words.shift << '\\fP' + while RE_PUNCT =~ words[0] + retval << words.shift + end + # @nospace = 1 if @nospace == 0 && RE_PUNCT =~ words[0] + next + end + when 'Lk' + if !quote.include?(OPTION) + url = words.shift + if name = shift_arg(words) + retval << '\\fI' << name << ':\\fP ' + end + retval << '\\fB' + retval << '\\&' if /\A\./ =~ url + retval << url << '\\fP' + next + end + end + + case word + when 'Dv' + retval << '.BR' + next + when 'Em', 'Ev' + retval << '.IR' + next + when 'Pq' + retval << '(' + @nospace = 1 + quote << PAREN + next + when 'Sx', 'Sy' + retval << '.B ' << words.join(' ') + break + when 'Ic' + retval << '\\fB' + until words.empty? || RE_PUNCT =~ words[0] + case words[0] + when 'Op' + words.shift + retval << '[' + words.push(words.pop + ']') + next + when 'Aq' + words.shift + retval << '<' + words.push(words.pop + '>') + next + when 'Ar' + words.shift + retval << '\\fI' << words.shift << '\\fP' + else + retval << words.shift + end + + retval << ' ' if @nospace == 0 + end + + retval.chomp!(' ') + retval << '\\fP' + retval << words.shift unless words.empty? + break + when 'Bl' + @oldoptlist = @optlist + + case words[0] + when '-bullet' + @optlist = 1 + when '-enum' + @optlist = 2 + @enum = 0 + when '-tag' + @optlist = 3 + when '-item' + @optlist = 4 + end + + break + when 'El' + @optlist = @oldoptlist + next + end + + if @optlist != 0 && word == 'It' + case @optlist + when 1 + # bullets + retval << '.IP \\(bu' + when 2 + # enum + @enum += 1 + retval << '.IP ' << @enum << '.' + when 3 + # tags + retval << ".TP\n" + case words[0] + when 'Pa', 'Ev', 'Lk' + words.shift + retval << '.B' + end + when 4 + # item + retval << ".IP\n" + end + + next + end + + case word + when 'Sm' + case words[0] + when 'off' + @nospace = 2 + when 'on' + # retval << "\n" + @nospace = 0 + end + words.shift + next + end + + retval << word + end + + return nil if retval == '.' + + retval.sub!(/\A\.([^a-zA-Z])/, "\\1") + # retval.chomp!(' ') + + while q = quote.pop + case q + when OPTION + retval << ']' + when PAREN + retval << ')' + when ANGLE + retval << '>' + end + end + + # retval << ' ' unless @nospace == 0 || retval.empty? || /\n\z/ =~ retval + + retval << ' ' unless !@ext || @extopt || / $/ =~ retval + + retval << "\n" unless @ext || @extopt || retval.empty? || /\n\z/ =~ retval + + retval << ".fi\n" if dl + + return retval + end + + def self.mdoc2man(i, o) + new.mdoc2man(i, o) + end +end + +if $0 == __FILE__ + Mdoc2Man.mdoc2man(ARGF, STDOUT) +end diff --git a/tool/merger.rb b/tool/merger.rb new file mode 100755 index 0000000000..a690b47da3 --- /dev/null +++ b/tool/merger.rb @@ -0,0 +1,314 @@ +#!/bin/sh +# -*- ruby -*- +exec "${RUBY-ruby}" "-x" "$0" "$@" && [ ] if false +#!ruby +# This needs ruby 2.0 and Git. +# As a Ruby committer, run this in a git repository to commit a change. + +require 'tempfile' +require 'net/http' +require 'uri' +require 'shellwords' + +ENV['LC_ALL'] = 'C' +ORIGIN = 'git@git.ruby-lang.org:ruby.git' +GITHUB = 'git@github.com:ruby/ruby.git' + +class << Merger = Object.new + def help + puts <<-HELP +\e[1msimple backport\e[0m + ruby #$0 1234abc + +\e[1mrevision increment\e[0m + ruby #$0 revisionup + +\e[1mteeny increment\e[0m + ruby #$0 teenyup + +\e[1mtagging major release\e[0m + ruby #$0 tag 3.2.0 + +\e[1mtagging patch release\e[0m (for 2.1.0 or later, it means X.Y.Z (Z > 0) release) + ruby #$0 tag + +\e[1mtagging preview/RC\e[0m + ruby #$0 tag 3.2.0-preview1 + +\e[1mremove tag\e[0m + ruby #$0 removetag 3.2.9 + +\e[33;1m* all operations shall be applied to the working directory.\e[0m + HELP + end + + def interactive(str, editfile = nil) + loop do + yield if block_given? + STDERR.puts "\e[1;33m#{str} ([y]es|[a]bort|[r]etry#{'|[e]dit' if editfile})\e[0m" + case STDIN.gets + when /\Aa/i then exit 1 + when /\Ar/i then redo + when /\Ay/i then break + when /\Ae/i then system(ENV['EDITOR'], editfile) + else exit 1 + end + end + end + + def version_up(teeny: false) + now = Time.now + now = now.localtime(9*60*60) # server is Japan Standard Time +09:00 + system('git', 'checkout', 'HEAD', 'version.h') + v, pl = version + + if teeny + v[2].succ! + end + if pl != '-1' # trunk does not have patchlevel + pl.succ! + end + + str = open('version.h', 'rb', &:read) + ruby_release_date = str[/RUBY_RELEASE_YEAR_STR"-"RUBY_RELEASE_MONTH_STR"-"RUBY_RELEASE_DAY_STR/] || now.strftime('"%Y-%m-%d"') + [%W[RUBY_VERSION "#{v.join('.')}"], + %W[RUBY_VERSION_CODE #{v.join('')}], + %W[RUBY_VERSION_MAJOR #{v[0]}], + %W[RUBY_VERSION_MINOR #{v[1]}], + %W[RUBY_VERSION_TEENY #{v[2]}], + %W[RUBY_RELEASE_DATE #{ruby_release_date}], + %W[RUBY_RELEASE_CODE #{now.strftime('%Y%m%d')}], + %W[RUBY_PATCHLEVEL #{pl}], + %W[RUBY_RELEASE_YEAR #{now.year}], + %W[RUBY_RELEASE_MONTH #{now.month}], + %W[RUBY_RELEASE_DAY #{now.day}], + ].each do |(k, i)| + str.sub!(/^(#define\s+#{k}\s+).*$/, "\\1#{i}") + end + str.sub!(/\s+\z/m, '') + fn = sprintf('version.h.tmp.%032b', rand(1 << 31)) + File.rename('version.h', fn) + open('version.h', 'wb') do |f| + f.puts(str) + end + File.unlink(fn) + end + + def tag(relname) + # relname: + # * 2.2.0-preview1 + # * 2.2.0-rc1 + # * 2.2.0 + v, pl = version + if relname + abort "patchlevel is not -1 but '#{pl}' for preview or rc" if pl != '-1' && /-(?:preview|rc)/ =~ relname + abort "patchlevel is not 0 but '#{pl}' for the first release" if pl != '0' && relname.end_with?(".0") + pl = relname[/-(.*)\z/, 1] + curver = "#{v.join('.')}#{("-#{pl}" if pl)}" + if relname != curver + abort "given relname '#{relname}' conflicts current version '#{curver}'" + end + else + if pl == '-1' + abort 'no relname is given and not in a release branch even if this is patch release' + end + end + tagname = "v#{v.join('_')}#{("_#{pl}" if v[0] < "2" || (v[0] == "2" && v[1] < "1") || /^(?:preview|rc)/ =~ pl)}" + + unless execute('git', 'tag', tagname) + abort 'specfied tag already exists. check tag name and remove it if you want to force re-tagging' + end + execute('git', 'push', ORIGIN, tagname, interactive: true) + end + + def remove_tag(relname) + # relname: + # * 2.2.0-preview1 + # * 2.2.0-rc1 + # * 2.2.0 + # * v2_2_0_preview1 + # * v2_2_0_rc1 + # * v2_2_0 + unless relname + raise ArgumentError, 'relname is not specified' + end + if /^v/ !~ relname + tagname = "v#{relname.gsub(/[.-]/, '_')}" + else + tagname = relname + end + + execute('git', 'tag', '-d', tagname) + execute('git', 'push', ORIGIN, ":#{tagname}", interactive: true) + execute('git', 'push', GITHUB, ":#{tagname}", interactive: true) + end + + def update_revision_h + execute('ruby tool/file2lastrev.rb --revision.h . > revision.tmp') + execute('tool/ifchange', '--timestamp=.revision.time', 'revision.h', 'revision.tmp') + execute('rm', '-f', 'revision.tmp') + end + + def stat + `git status --short` + end + + def diff(file = nil) + command = %w[git diff --color HEAD] + IO.popen(command + [file].compact, &:read) + end + + def commit(file) + current_branch = IO.popen(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], &:read).strip + execute('git', 'add', '.') && execute('git', 'commit', '-F', file) + end + + def has_conflicts? + changes = IO.popen(%w[git status --porcelain -z]) { |io| io.readlines("\0", chomp: true) } + # Discover unmerged files + # AU: unmerged, added by us + # DU: unmerged, deleted by us + # UU: unmerged, both modified + # AA: unmerged, both added + conflict = changes.grep(/\A(?:.U|AA) /) {$'} + !conflict.empty? + end + + private + + # Prints the version of Ruby found in version.h + def version + v = p = nil + open 'version.h', 'rb' do |f| + f.each_line do |l| + case l + when /^#define RUBY_VERSION "(\d+)\.(\d+)\.(\d+)"$/ + v = $~.captures + when /^#define RUBY_VERSION_TEENY (\d+)$/ + (v ||= [])[2] = $1 + when /^#define RUBY_PATCHLEVEL (-?\d+)$/ + p = $1 + end + end + end + if v and !v[0] + open 'include/ruby/version.h', 'rb' do |f| + f.each_line do |l| + case l + when /^#define RUBY_API_VERSION_MAJOR (\d+)/ + v[0] = $1 + when /^#define RUBY_API_VERSION_MINOR (\d+)/ + v[1] = $1 + end + end + end + end + return v, p + end + + def execute(*cmd, interactive: false) + if interactive + Merger.interactive("OK?: #{cmd.shelljoin}") + end + puts "+ #{cmd.shelljoin}" + system(*cmd) + end +end + +case ARGV[0] +when "teenyup" + Merger.version_up(teeny: true) + puts Merger.diff('version.h') +when "up", /\A(ver|version|rev|revision|lv|level|patch\s*level)\s*up\z/ + Merger.version_up + puts Merger.diff('version.h') +when "tag" + Merger.tag(ARGV[1]) +when /\A(?:remove|rm|del)_?tag\z/ + Merger.remove_tag(ARGV[1]) +when nil, "-h", "--help" + Merger.help + exit +else + Merger.update_revision_h + + case ARGV[0] + when /--ticket=(.*)/ + tickets = $1.split(/,/) + ARGV.shift + else + tickets = [] + detect_ticket = true + end + + revstr = ARGV[0].gsub(%r!https://github\.com/ruby/ruby/commit/|https://bugs\.ruby-lang\.org/projects/ruby-master/repository/git/revisions/!, '') + revstr = revstr.delete('^, :\-0-9a-fA-F') + revs = revstr.split(/[,\s]+/) + commit_message = '' + + revs.each do |rev| + git_rev = nil + case rev + when /\A\h{7,40}\z/ + git_rev = rev + when nil then + puts "#$0 revision" + exit + else + puts "invalid revision part '#{rev}' in '#{ARGV[0]}'" + exit + end + + # Merge revision from Git patch + git_uri = "https://git.ruby-lang.org/ruby.git/patch/?id=#{git_rev}" + resp = Net::HTTP.get_response(URI(git_uri)) + if resp.code != '200' + abort "'#{git_uri}' returned status '#{resp.code}':\n#{resp.body}" + end + patch = resp.body.sub(/^diff --git a\/version\.h b\/version\.h\nindex .*\n--- a\/version\.h\n\+\+\+ b\/version\.h\n@@ .* @@\n(?:[-\+ ].*\n|\n)+/, '') + + if detect_ticket + tickets += patch.scan(/\[(?:Bug|Feature|Misc) #(\d+)\]/i).map(&:first) + end + + message = "#{(patch[/^Subject: (.*)\n---\n /m, 1] || "Message not found for revision: #{git_rev}\n")}" + message.gsub!(/\G(.*)\n( .*)/, "\\1\\2") + message = "\n\n#{message}" + + puts '+ git apply' + IO.popen(['git', 'apply', '--3way'], 'wb') { |f| f.write(patch) } + + commit_message << message.sub(/\A-+\nr.*/, '').sub(/\n-+\n\z/, '').gsub(/^./, "\t\\&") + end + + if Merger.diff.empty? + Merger.interactive('Nothing is modified, right?') + end + + Merger.version_up + f = Tempfile.new 'merger.rb' + f.printf "merge revision(s) %s:%s", revstr, tickets.map{|num| " [Backport ##{num}]"}.join + f.write commit_message + f.flush + f.close + + if Merger.has_conflicts? + Merger.interactive('conflicts resolved?', f.path) do + IO.popen(ENV['PAGER'] || ['less', '-R'], 'w') do |g| + g << Merger.stat + g << "\n\n" + f.open + g << f.read + f.close + g << "\n\n" + g << Merger.diff + end + end + end + + unless Merger.commit(f.path) + puts 'commit failed; try again.' + end + + f.close(true) +end diff --git a/tool/mjit_archflag.sh b/tool/mjit_archflag.sh new file mode 100644 index 0000000000..082fb4bcd0 --- /dev/null +++ b/tool/mjit_archflag.sh @@ -0,0 +1,40 @@ +# -*- sh -*- + +quote() { + printf "#${indent}define $1" + shift + ${1+printf} ${1+' "%s"'$sep} ${1+"$@"} + echo +} + +archs="" +arch_flag="" + +parse_arch_flags() { + for arch in $1; do + archs="${archs:+$archs }${arch%=*}" + done + + while shift && [ "$#" -gt 0 ]; do + case "$1" in + -arch) + shift + archs="${archs:+$archs }$1" + ;; + *) + arch_flag="${arch_flag:+${arch_flag} }$1" + ;; + esac + done +} + +define_arch_flags() { + ${archs:+echo} ${archs:+'#if 0'} + for arch in $archs; do + echo "#elif defined __${arch}__" + quote "MJIT_ARCHFLAG " -arch "${arch}" + done + ${archs:+echo} ${archs:+'#else'} + quote "MJIT_ARCHFLAG /* ${arch_flag:-no flag} */" ${arch_flag} + ${archs:+echo} ${archs:+'#endif'} +} diff --git a/tool/mjit_tabs.rb b/tool/mjit_tabs.rb new file mode 100644 index 0000000000..edcbf6cfcb --- /dev/null +++ b/tool/mjit_tabs.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true +# This is a script to run a command in ARGV, expanding tabs in some files +# included by vm.c to normalize indentation of MJIT header. You can enable +# this feature by passing `--without-mjit-tabs` in configure. +# +# Note that preprocessor of GCC converts a hard tab to one spaces, where +# we expect it to be shown as 8 spaces. To obviate this script, we need +# to convert all tabs to spaces in these files. + +require 'fileutils' + +EXPAND_TARGETS = %w[ + vm*.* + include/ruby/ruby.h +] + +# These files have no hard tab indentations. Skip normalizing these files from the glob result. +SKIPPED_FILES = %w[ + vm_callinfo.h + vm_debug.h + vm_exec.h + vm_opts.h + vm_sync.h + vm_sync.c +] + +srcdir = File.expand_path('..', __dir__) +targets = EXPAND_TARGETS.flat_map { |t| Dir.glob(File.join(srcdir, t)) } - SKIPPED_FILES.map { |f| File.join(srcdir, f) } +sources = {} +mtimes = {} + +mjit_tabs, *command = ARGV + +targets.each do |target| + next if mjit_tabs != 'false' + unless File.writable?(target) + puts "tool/mjit_tabs.rb: Skipping #{target.dump} as it's not writable." + next + end + source = File.read(target) + begin + expanded = source.gsub(/^\t+/) { |tab| ' ' * 8 * tab.length } + rescue ArgumentError # invalid byte sequence in UTF-8 (Travis, RubyCI) + puts "tool/mjit_tabs.rb: Skipping #{target.dump} as the encoding is #{source.encoding}." + next + end + + sources[target] = source + mtimes[target] = File.mtime(target) + + if sources[target] == expanded + puts "#{target.dump} has no hard tab indentation. This should be ignored in tool/mjit_tabs.rb." + end + File.write(target, expanded) + FileUtils.touch(target, mtime: mtimes[target]) +end + +result = system(*command) + +targets.each do |target| + if sources.key?(target) + File.write(target, sources[target]) + FileUtils.touch(target, mtime: mtimes.fetch(target)) + end +end + +exit result diff --git a/tool/mk_builtin_loader.rb b/tool/mk_builtin_loader.rb new file mode 100644 index 0000000000..02941735f7 --- /dev/null +++ b/tool/mk_builtin_loader.rb @@ -0,0 +1,370 @@ +# Parse built-in script and make rbinc file + +require 'ripper' +require 'stringio' +require_relative 'ruby_vm/helpers/c_escape' + +def string_literal(lit, str = []) + while lit + case lit.first + when :string_concat, :string_embexpr, :string_content + _, *lit = lit + lit.each {|s| string_literal(s, str)} + return str + when :string_literal + _, lit = lit + when :@tstring_content + str << lit[1] + return str + else + raise "unexpected #{lit.first}" + end + end +end + +def inline_text argc, arg1 + raise "argc (#{argc}) of inline! should be 1" unless argc == 1 + arg1 = string_literal(arg1) + raise "1st argument should be string literal" unless arg1 + arg1.join("").rstrip +end + +def make_cfunc_name inlines, name, lineno + case name + when /\[\]/ + name = '_GETTER' + when /\[\]=/ + name = '_SETTER' + else + name = name.tr('!?', 'EP') + end + + base = "builtin_inline_#{name}_#{lineno}" + if inlines[base] + 1000.times{|i| + name = "#{base}_#{i}" + return name unless inlines[name] + } + raise "too many functions in same line..." + else + base + end +end + +def collect_locals tree + _type, name, (line, _cols) = tree + if locals = LOCALS_DB[[name, line]] + locals + else + if false # for debugging + pp LOCALS_DB + raise "not found: [#{name}, #{line}]" + end + end +end + +def collect_builtin base, tree, name, bs, inlines, locals = nil + while tree + recv = sep = mid = args = nil + case tree.first + when :def + locals = collect_locals(tree[1]) + tree = tree[3] + next + when :defs + locals = collect_locals(tree[3]) + tree = tree[5] + next + when :class + name = 'class' + tree = tree[3] + next + when :sclass, :module + name = 'class' + tree = tree[2] + next + when :method_add_arg + _, mid, (_, (_, args)) = tree + case mid.first + when :call + _, recv, sep, mid = mid + when :fcall + _, mid = mid + else + mid = nil + end + when :vcall + _, mid = tree + when :command # FCALL + _, mid, (_, args) = tree + when :call, :command_call # CALL + _, recv, sep, mid, (_, args) = tree + end + + if mid + raise "unknown sexp: #{mid.inspect}" unless %i[@ident @const].include?(mid.first) + _, mid, (lineno,) = mid + if recv + func_name = nil + case recv.first + when :var_ref + _, recv = recv + if recv.first == :@const and recv[1] == "Primitive" + func_name = mid.to_s + end + when :vcall + _, recv = recv + if recv.first == :@ident and recv[1] == "__builtin" + func_name = mid.to_s + end + end + collect_builtin(base, recv, name, bs, inlines) unless func_name + else + func_name = mid[/\A__builtin_(.+)/, 1] + end + if func_name + cfunc_name = func_name + args.pop unless (args ||= []).last + argc = args.size + + if /(.+)[\!\?]\z/ =~ func_name + case $1 + when 'attr' + text = inline_text(argc, args.first) + if text != 'inline' + raise "Only 'inline' is allowed to be annotated (but got: '#{text}')" + end + break + when 'cstmt' + text = inline_text argc, args.first + + func_name = "_bi#{inlines.size}" + cfunc_name = make_cfunc_name(inlines, name, lineno) + inlines[cfunc_name] = [lineno, text, locals, func_name] + argc -= 1 + when 'cexpr', 'cconst' + text = inline_text argc, args.first + code = "return #{text};" + + func_name = "_bi#{inlines.size}" + cfunc_name = make_cfunc_name(inlines, name, lineno) + + locals = [] if $1 == 'cconst' + inlines[cfunc_name] = [lineno, code, locals, func_name] + argc -= 1 + when 'cinit' + text = inline_text argc, args.first + func_name = nil # required + inlines[inlines.size] = [lineno, text, nil, nil] + argc -= 1 + when 'mandatory_only' + func_name = nil + when 'arg' + argc == 1 or raise "unexpected argument number #{argc}" + (arg = args.first)[0] == :symbol_literal or raise "symbol literal expected #{args}" + (arg = arg[1])[0] == :symbol or raise "symbol expected #{arg}" + (var = arg[1] and var = var[1]) or raise "argument name expected #{arg}" + func_name = nil + end + end + + if bs[func_name] && + bs[func_name] != [argc, cfunc_name] + raise "same builtin function \"#{func_name}\", but different arity (was #{bs[func_name]} but #{argc})" + end + + bs[func_name] = [argc, cfunc_name] if func_name + end + break unless tree = args + end + + tree.each do |t| + collect_builtin base, t, name, bs, inlines, locals if Array === t + end + break + end +end + +# ruby mk_builtin_loader.rb TARGET_FILE.rb +# #=> generate TARGET_FILE.rbinc +# + +LOCALS_DB = {} # [method_name, first_line] = locals + +def collect_iseq iseq_ary + # iseq_ary.each_with_index{|e, i| p [i, e]} + label = iseq_ary[5] + first_line = iseq_ary[8] + type = iseq_ary[9] + locals = iseq_ary[10] + insns = iseq_ary[13] + + if type == :method + LOCALS_DB[[label, first_line].freeze] = locals + end + + insns.each{|insn| + case insn + when Integer + # ignore + when Array + # p insn.shift # insn name + insn.each{|op| + if Array === op && op[0] == "YARVInstructionSequence/SimpleDataFormat" + collect_iseq op + end + } + end + } +end + +def generate_cexpr(ofile, lineno, line_file, body_lineno, text, locals, func_name) + f = StringIO.new + f.puts '{' + lineno += 1 + locals.reverse_each.with_index{|param, i| + next unless Symbol === param + f.puts "MAYBE_UNUSED(const VALUE) #{param} = rb_vm_lvar(ec, #{-3 - i});" + lineno += 1 + } + f.puts "#line #{body_lineno} \"#{line_file}\"" + lineno += 1 + + f.puts text + lineno += text.count("\n") + 1 + + f.puts "#line #{lineno + 2} \"#{ofile}\"" # TODO: restore line number. + f.puts "}" + f.puts + lineno += 3 + + return lineno, f.string +end + +def mk_builtin_header file + base = File.basename(file, '.rb') + ofile = "#{file}inc" + + # bs = { func_name => argc } + code = File.read(file) + collect_iseq RubyVM::InstructionSequence.compile(code).to_a + collect_builtin(base, Ripper.sexp(code), 'top', bs = {}, inlines = {}) + + begin + f = open(ofile, 'w') + rescue Errno::EACCES + # Fall back to the current directory + f = open(File.basename(ofile), 'w') + end + begin + if File::ALT_SEPARATOR + file = file.tr(File::ALT_SEPARATOR, File::SEPARATOR) + ofile = ofile.tr(File::ALT_SEPARATOR, File::SEPARATOR) + end + lineno = __LINE__ + f.puts "// -*- c -*-" + f.puts "// DO NOT MODIFY THIS FILE DIRECTLY." + f.puts "// auto-generated file" + f.puts "// by #{__FILE__}" + f.puts "// with #{file}" + f.puts '#include "internal/compilers.h" /* for MAYBE_UNUSED */' + f.puts '#include "internal/warnings.h" /* for COMPILER_WARNING_PUSH */' + f.puts '#include "ruby/ruby.h" /* for VALUE */' + f.puts '#include "builtin.h" /* for RB_BUILTIN_FUNCTION */' + f.puts 'struct rb_execution_context_struct; /* in vm_core.h */' + f.puts + lineno = __LINE__ - lineno - 1 + line_file = file + + inlines.each{|cfunc_name, (body_lineno, text, locals, func_name)| + if String === cfunc_name + f.puts "static VALUE #{cfunc_name}(struct rb_execution_context_struct *ec, const VALUE self)" + lineno += 1 + lineno, str = generate_cexpr(ofile, lineno, line_file, body_lineno, text, locals, func_name) + f.write str + else + # cinit! + f.puts "#line #{body_lineno} \"#{line_file}\"" + lineno += 1 + f.puts text + lineno += text.count("\n") + 1 + f.puts "#line #{lineno + 2} \"#{ofile}\"" # TODO: restore line number. + lineno += 1 + end + } + + bs.each_pair{|func, (argc, cfunc_name)| + decl = ', VALUE' * argc + argv = argc \ + . times \ + . map {|i|", argv[#{i}]"} \ + . join('') + f.puts %'static void' + f.puts %'mjit_compile_invokebuiltin_for_#{func}(FILE *f, long index, unsigned stack_size, bool inlinable_p)' + f.puts %'{' + f.puts %' fprintf(f, " VALUE self = GET_SELF();\\n");' + f.puts %' fprintf(f, " typedef VALUE (*func)(rb_execution_context_t *, VALUE#{decl});\\n");' + if inlines.has_key? cfunc_name + body_lineno, text, locals, func_name = inlines[cfunc_name] + lineno, str = generate_cexpr(ofile, lineno, line_file, body_lineno, text, locals, func_name) + f.puts %' if (inlinable_p) {' + str.gsub(/^(?!#)/, ' ').each_line {|i| + j = RubyVM::CEscape.rstring2cstr(i).dup + j.sub!(/^ return\b/ , ' val =') + f.printf(%' fprintf(f, "%%s", %s);\n', j) + } + f.puts(%' return;') + f.puts(%' }') + end + if argc > 0 + f.puts %' if (index == -1) {' + f.puts %' fprintf(f, " const VALUE *argv = &stack[%d];\\n", stack_size - #{argc});' + f.puts %' }' + f.puts %' else {' + f.puts %' fprintf(f, " const unsigned int lnum = GET_ISEQ()->body->local_table_size;\\n");' + f.puts %' fprintf(f, " const VALUE *argv = GET_EP() - lnum - VM_ENV_DATA_SIZE + 1 + %ld;\\n", index);' + f.puts %' }' + end + f.puts %' fprintf(f, " func f = (func)%"PRIuVALUE"; /* == #{cfunc_name} */\\n", (VALUE)#{cfunc_name});' + f.puts %' fprintf(f, " val = f(ec, self#{argv});\\n");' + f.puts %'}' + f.puts + } + + f.puts "void Init_builtin_#{base}(void)" + f.puts "{" + + table = "#{base}_table" + f.puts " // table definition" + f.puts " static const struct rb_builtin_function #{table}[] = {" + bs.each.with_index{|(func, (argc, cfunc_name)), i| + f.puts " RB_BUILTIN_FUNCTION(#{i}, #{func}, #{cfunc_name}, #{argc}, mjit_compile_invokebuiltin_for_#{func})," + } + f.puts " RB_BUILTIN_FUNCTION(-1, NULL, NULL, 0, 0)," + f.puts " };" + + f.puts + f.puts " // arity_check" + f.puts "COMPILER_WARNING_PUSH" + f.puts "#if GCC_VERSION_SINCE(5, 1, 0) || defined __clang__" + f.puts "COMPILER_WARNING_ERROR(-Wincompatible-pointer-types)" + f.puts "#endif" + bs.each{|func, (argc, cfunc_name)| + f.puts " if (0) rb_builtin_function_check_arity#{argc}(#{cfunc_name});" + } + f.puts "COMPILER_WARNING_POP" + + f.puts + f.puts " // load" + f.puts " rb_load_with_builtin_functions(#{base.dump}, #{table});" + + f.puts "}" + ensure + f.close + end +end + +ARGV.each{|file| + # feature.rb => load_feature.inc + mk_builtin_header file +} diff --git a/tool/mkconfig.rb b/tool/mkconfig.rb new file mode 100755 index 0000000000..41bee02247 --- /dev/null +++ b/tool/mkconfig.rb @@ -0,0 +1,392 @@ +#!./miniruby -s + +# This script, which is run when ruby is built, generates rbconfig.rb by +# parsing information from config.status. rbconfig.rb contains build +# information for ruby (compiler flags, paths, etc.) and is used e.g. by +# mkmf to build compatible native extensions. + +# avoid warnings with -d. +$install_name ||= nil +$so_name ||= nil +$unicode_version ||= nil +$unicode_emoji_version ||= nil +arch = $arch or raise "missing -arch" +version = $version or raise "missing -version" + +srcdir = File.expand_path('../..', __FILE__) +$:.unshift(".") + +mkconfig = File.basename($0) + +fast = {'prefix'=>true, 'ruby_install_name'=>true, 'INSTALL'=>true, 'EXEEXT'=>true} + +win32 = /mswin/ =~ arch +universal = /universal.*darwin/ =~ arch +v_fast = [] +v_others = [] +vars = {} +continued_name = nil +continued_line = nil +install_name = nil +so_name = nil +platform = nil +File.foreach "config.status" do |line| + next if /^#/ =~ line + name = nil + case line + when /^s([%,])@(\w+)@\1(?:\|\#_!!_\#\|)?(.*)\1/ + name = $2 + val = $3.gsub(/\\(?=,)/, '') + when /^S\["(\w+)"\]\s*=\s*"(.*)"\s*(\\)?$/ + name = $1 + val = $2 + if $3 + continued_line = [val] + continued_name = name + next + end + when /^"(.*)"\s*(\\)?$/ + next if !continued_line + continued_line << $1 + next if $2 + continued_line.each {|s| s.sub!(/\\n\z/, "\n")} + val = continued_line.join + name = continued_name + continued_line = nil + when /^(?:ac_given_)?INSTALL=(.*)/ + v_fast << " CONFIG[\"INSTALL\"] = " + $1 + "\n" + end + + if name + case name + when /^(?:ac_.*|configure_input|(?:top_)?srcdir|\w+OBJS)$/; next + when /^(?:X|(?:MINI|RUN|(?:HAVE_)?BASE|BOOTSTRAP|BTEST)RUBY(?:_COMMAND)?$)/; next + when /^INSTALLDOC|TARGET$/; next + when /^DTRACE/; next + when /^MJIT_(CC|SUPPORT)$/; # pass + when /^MJIT_/; next + when /^(?:MAJOR|MINOR|TEENY)$/; vars[name] = val; next + when /^LIBRUBY_D?LD/; next + when /^RUBY_INSTALL_NAME$/; next vars[name] = (install_name = val).dup if $install_name + when /^RUBY_SO_NAME$/; next vars[name] = (so_name = val).dup if $so_name + when /^arch$/; if val.empty? then val = arch else arch = val end + when /^sitearch$/; val = '$(arch)' if val.empty? + when /^DESTDIR$/; next + when /RUBYGEMS/; next + end + case val + when /^\$\(ac_\w+\)$/; next + when /^\$\{ac_\w+\}$/; next + when /^\$ac_\w+$/; next + end + if /^program_transform_name$/ =~ name + val.sub!(/\As(\\?\W)(?:\^|\${1,2})\1\1(;|\z)/, '') + if val.empty? + $install_name ||= "ruby" + next + end + unless $install_name + $install_name = "ruby" + val.gsub!(/\$\$/, '$') + val.scan(%r[\G[\s;]*(/(?:\\.|[^/])*+/)?([sy])(\\?\W)((?:(?!\3)(?:\\.|.))*+)\3((?:(?!\3)(?:\\.|.))*+)\3([gi]*)]) do + |addr, cmd, sep, pat, rep, opt| + if addr + Regexp.new(addr[/\A\/(.*)\/\z/, 1]) =~ $install_name or next + end + case cmd + when 's' + pat = Regexp.new(pat, opt.include?('i')) + if opt.include?('g') + $install_name.gsub!(pat, rep) + else + $install_name.sub!(pat, rep) + end + when 'y' + $install_name.tr!(Regexp.quote(pat), rep) + end + end + end + end + eq = win32 && vars[name] ? '<< "\n"' : '=' + vars[name] = val + if name == "configure_args" + val.gsub!(/--with-out-ext/, "--without-ext") + end + val = val.gsub(/\$(?:\$|\{?(\w+)\}?)/) {$1 ? "$(#{$1})" : $&}.dump + case name + when /^prefix$/ + val = "(TOPDIR || DESTDIR + #{val})" + when /^ARCH_FLAG$/ + val = "arch_flag || #{val}" if universal + when /^UNIVERSAL_ARCHNAMES$/ + universal, val = val, 'universal' if universal + when /^arch$/ + if universal + platform = val.sub(/universal/, %q[#{arch && universal[/(?:\A|\s)#{Regexp.quote(arch)}=(\S+)/, 1] || RUBY_PLATFORM[/\A[^-]*/]}]) + end + when /^includedir$/ + val = '"$(SDKROOT)"'+val if /darwin/ =~ arch + end + v = " CONFIG[\"#{name}\"] #{eq} #{val}\n" + if fast[name] + v_fast << v + else + v_others << v + end + #case name + #when "RUBY_PROGRAM_VERSION" + # version = val[/\A"(.*)"\z/, 1] + #end + end +# break if /^CEOF/ +end + +drive = File::PATH_SEPARATOR == ';' + +def vars.expand(val, config = self) + newval = val.gsub(/\$\$|\$\(([^()]+)\)|\$\{([^{}]+)\}/) { + var = $& + if !(v = $1 || $2) + '$' + elsif key = config[v = v[/\A[^:]+(?=(?::(.*?)=(.*))?\z)/]] + pat, sub = $1, $2 + config[v] = false + config[v] = expand(key, config) + key = key.gsub(/#{Regexp.quote(pat)}(?=\s|\z)/n) {sub} if pat + key + else + var + end + } + val.replace(newval) unless newval == val + val +end +prefix = vars.expand(vars["prefix"] ||= "") +rubyarchdir = vars.expand(vars["rubyarchdir"] ||= "") +relative_archdir = rubyarchdir.rindex(prefix, 0) ? rubyarchdir[prefix.size..-1] : rubyarchdir + +puts %[\ +# encoding: ascii-8bit +# frozen-string-literal: false +# +# The module storing Ruby interpreter configurations on building. +# +# This file was created by #{mkconfig} when ruby was built. It contains +# build information for ruby which is used e.g. by mkmf to build +# compatible native extensions. Any changes made to this file will be +# lost the next time ruby is built. + +module RbConfig + RUBY_VERSION.start_with?("#{version[/^[0-9]+\.[0-9]+\./] || version}") or + raise "ruby lib version (#{version}) doesn't match executable version (\#{RUBY_VERSION})" + +] +print " # Ruby installed directory.\n" +print " TOPDIR = File.dirname(__FILE__).chomp!(#{relative_archdir.dump})\n" +print " # DESTDIR on make install.\n" +print " DESTDIR = ", (drive ? "TOPDIR && TOPDIR[/\\A[a-z]:/i] || " : ""), "'' unless defined? DESTDIR\n" +print <<'ARCH' if universal + arch_flag = ENV['ARCHFLAGS'] || ((e = ENV['RC_ARCHS']) && e.split.uniq.map {|a| "-arch #{a}"}.join(' ')) + arch = arch_flag && arch_flag[/\A\s*-arch\s+(\S+)\s*\z/, 1] +ARCH +print " universal = #{universal}\n" if universal +print " # The hash configurations stored.\n" +print " CONFIG = {}\n" +print " CONFIG[\"DESTDIR\"] = DESTDIR\n" + +versions = {} +IO.foreach(File.join(srcdir, "version.h")) do |l| + m = /^\s*#\s*define\s+RUBY_(PATCHLEVEL)\s+(-?\d+)/.match(l) + if m + versions[m[1]] = m[2] + break if versions.size == 4 + next + end + m = /^\s*#\s*define\s+RUBY_VERSION_(\w+)\s+(-?\d+)/.match(l) + if m + versions[m[1]] = m[2] + break if versions.size == 4 + next + end + m = /^\s*#\s*define\s+RUBY_VERSION\s+\W?([.\d]+)/.match(l) + if m + versions['MAJOR'], versions['MINOR'], versions['TEENY'] = m[1].split('.') + break if versions.size == 4 + next + end +end +if versions.size != 4 + IO.foreach(File.join(srcdir, "include/ruby/version.h")) do |l| + m = /^\s*#\s*define\s+RUBY_API_VERSION_(\w+)\s+(-?\d+)/.match(l) + if m + versions[m[1]] ||= m[2] + break if versions.size == 4 + next + end + end +end +%w[MAJOR MINOR TEENY PATCHLEVEL].each do |v| + print " CONFIG[#{v.dump}] = #{(versions[v]||vars[v]).dump}\n" +end + +dest = drive ? %r'= "(?!\$[\(\{])(?i:[a-z]:)' : %r'= "(?!\$[\(\{])' +v_disabled = {} +v_others.collect! do |x| + if /^\s*CONFIG\["((?!abs_|old)[a-z]+(?:_prefix|dir))"\]/ === x + name = $1 + if /= "no"$/ =~ x + v_disabled[name] = true + v_others.delete(name) + next + end + x.sub(dest, '= "$(DESTDIR)') + else + x + end +end +v_others.compact! + +if $install_name + if install_name and vars.expand("$(RUBY_INSTALL_NAME)") == $install_name + $install_name = install_name + end + v_fast << " CONFIG[\"ruby_install_name\"] = \"" + $install_name + "\"\n" + v_fast << " CONFIG[\"RUBY_INSTALL_NAME\"] = \"" + $install_name + "\"\n" +end +if $so_name + if so_name and vars.expand("$(RUBY_SO_NAME)") == $so_name + $so_name = so_name + end + v_fast << " CONFIG[\"RUBY_SO_NAME\"] = \"" + $so_name + "\"\n" +end + +print(*v_fast) +print(*v_others) +print <<EOS if $unicode_version + CONFIG["UNICODE_VERSION"] = #{$unicode_version.dump} +EOS +print <<EOS if $unicode_emoji_version + CONFIG["UNICODE_EMOJI_VERSION"] = #{$unicode_emoji_version.dump} +EOS +print prefix.start_with?("/System/") ? <<EOS : <<EOS if /darwin/ =~ arch + if sdkroot = ENV["SDKROOT"] + sdkroot = sdkroot.dup + elsif File.exist?(File.join(CONFIG["prefix"], "include")) || + !(sdkroot = (IO.popen(%w[/usr/bin/xcrun --sdk macosx --show-sdk-path], in: IO::NULL, err: IO::NULL, &:read) rescue nil)) + sdkroot = +"" + else + sdkroot.chomp! + end + CONFIG["SDKROOT"] = sdkroot +EOS + CONFIG["SDKROOT"] = "" +EOS +print <<EOS + CONFIG["platform"] = #{platform || '"$(arch)"'} + CONFIG["archdir"] = "$(rubyarchdir)" + CONFIG["topdir"] = File.dirname(__FILE__) + # Almost same with CONFIG. MAKEFILE_CONFIG has other variable + # reference like below. + # + # MAKEFILE_CONFIG["bindir"] = "$(exec_prefix)/bin" + # + # The values of this constant is used for creating Makefile. + # + # require 'rbconfig' + # + # print <<-END_OF_MAKEFILE + # prefix = \#{RbConfig::MAKEFILE_CONFIG['prefix']} + # exec_prefix = \#{RbConfig::MAKEFILE_CONFIG['exec_prefix']} + # bindir = \#{RbConfig::MAKEFILE_CONFIG['bindir']} + # END_OF_MAKEFILE + # + # => prefix = /usr/local + # exec_prefix = $(prefix) + # bindir = $(exec_prefix)/bin MAKEFILE_CONFIG = {} + # + # RbConfig.expand is used for resolving references like above in rbconfig. + # + # require 'rbconfig' + # p RbConfig.expand(RbConfig::MAKEFILE_CONFIG["bindir"]) + # # => "/usr/local/bin" + MAKEFILE_CONFIG = {} + CONFIG.each{|k,v| MAKEFILE_CONFIG[k] = v.dup} + + # call-seq: + # + # RbConfig.expand(val) -> string + # RbConfig.expand(val, config) -> string + # + # expands variable with given +val+ value. + # + # RbConfig.expand("$(bindir)") # => /home/foobar/all-ruby/ruby19x/bin + def RbConfig::expand(val, config = CONFIG) + newval = val.gsub(/\\$\\$|\\$\\(([^()]+)\\)|\\$\\{([^{}]+)\\}/) { + var = $& + if !(v = $1 || $2) + '$' + elsif key = config[v = v[/\\A[^:]+(?=(?::(.*?)=(.*))?\\z)/]] + pat, sub = $1, $2 + config[v] = false + config[v] = RbConfig::expand(key, config) + key = key.gsub(/\#{Regexp.quote(pat)}(?=\\s|\\z)/n) {sub} if pat + key + else + var + end + } + val.replace(newval) unless newval == val + val + end + CONFIG.each_value do |val| + RbConfig::expand(val) + end + + # :nodoc: + # call-seq: + # + # RbConfig.fire_update!(key, val) -> array + # RbConfig.fire_update!(key, val, mkconf, conf) -> array + # + # updates +key+ in +mkconf+ with +val+, and all values depending on + # the +key+ in +mkconf+. + # + # RbConfig::MAKEFILE_CONFIG.values_at("CC", "LDSHARED") # => ["gcc", "$(CC) -shared"] + # RbConfig::CONFIG.values_at("CC", "LDSHARED") # => ["gcc", "gcc -shared"] + # RbConfig.fire_update!("CC", "gcc-8") # => ["CC", "LDSHARED"] + # RbConfig::MAKEFILE_CONFIG.values_at("CC", "LDSHARED") # => ["gcc-8", "$(CC) -shared"] + # RbConfig::CONFIG.values_at("CC", "LDSHARED") # => ["gcc-8", "gcc-8 -shared"] + # + # returns updated keys list, or +nil+ if nothing changed. + def RbConfig.fire_update!(key, val, mkconf = MAKEFILE_CONFIG, conf = CONFIG) + return if mkconf[key] == val + mkconf[key] = val + keys = [key] + deps = [] + begin + re = Regexp.new("\\\\$\\\\((?:%1$s)\\\\)|\\\\$\\\\{(?:%1$s)\\\\}" % keys.join('|')) + deps |= keys + keys.clear + mkconf.each {|k,v| keys << k if re =~ v} + end until keys.empty? + deps.each {|k| conf[k] = mkconf[k].dup} + deps.each {|k| expand(conf[k])} + deps + end + + # call-seq: + # + # RbConfig.ruby -> path + # + # returns the absolute pathname of the ruby command. + def RbConfig.ruby + File.join( + RbConfig::CONFIG["bindir"], + RbConfig::CONFIG["ruby_install_name"] + RbConfig::CONFIG["EXEEXT"] + ) + end +end +CROSS_COMPILING = nil unless defined? CROSS_COMPILING +EOS + +# vi:set sw=2: diff --git a/tool/mkrunnable.rb b/tool/mkrunnable.rb new file mode 100755 index 0000000000..3b71b0751b --- /dev/null +++ b/tool/mkrunnable.rb @@ -0,0 +1,149 @@ +#!./miniruby +# -*- coding: us-ascii -*- + +# Used by "make runnable" target, to make symbolic links from a build +# directory. + +require './rbconfig' +require 'fileutils' + +case ARGV[0] +when "-n" + ARGV.shift + include FileUtils::DryRun +when "-v" + ARGV.shift + include FileUtils::Verbose +else + include FileUtils +end + +module Mswin + def ln_safe(src, dest, *opt) + cmd = ["mklink", dest.tr("/", "\\"), src.tr("/", "\\")] + cmd[1, 0] = opt + return if system("cmd", "/c", *cmd) + # TODO: use RUNAS or something + puts cmd.join(" ") + end + + def ln_dir_safe(src, dest) + ln_safe(src, dest, "/d") + end +end + +def clean_link(src, dest) + begin + link = File.readlink(dest) + rescue + else + return if link == src + File.unlink(dest) + end + yield src, dest +end + +def ln_safe(src, dest) + ln_sf(src, dest) +rescue Errno::ENOENT + # Windows disallows to create broken symboic links, probably because + # it is a kind of reparse points. + raise if File.exist?(src) +end + +alias ln_dir_safe ln_safe + +case RUBY_PLATFORM +when /linux|darwin|solaris/ + def ln_exe(src, dest) + ln(src, dest, force: true) + end +else + alias ln_exe ln_safe +end + +if !File.respond_to?(:symlink) && /mingw|mswin/ =~ (CROSS_COMPILING || RUBY_PLATFORM) + extend Mswin +end + +def clean_path(path) + path = "#{path}/".gsub(/(\A|\/)(?:\.\/)+/, '\1').tr_s('/', '/') + nil while path.sub!(/[^\/]+\/\.\.\//, '') + path +end + +def relative_path_from(path, base) + path = clean_path(path) + base = clean_path(base) + path, base = [path, base].map{|s|s.split("/")} + until path.empty? or base.empty? or path[0] != base[0] + path.shift + base.shift + end + path, base = [path, base].map{|s|s.join("/")} + if /(\A|\/)\.\.\// =~ base + File.expand_path(path) + else + base.gsub!(/[^\/]+/, '..') + File.join(base, path) + end +end + +def ln_relative(src, dest, executable = false) + return if File.identical?(src, dest) + parent = File.dirname(dest) + File.directory?(parent) or mkdir_p(parent) + return ln_exe(src, dest) if executable + clean_link(relative_path_from(src, parent), dest) {|s, d| ln_safe(s, d)} +end + +def ln_dir_relative(src, dest) + return if File.identical?(src, dest) + parent = File.dirname(dest) + File.directory?(parent) or mkdir_p(parent) + clean_link(relative_path_from(src, parent), dest) {|s, d| ln_dir_safe(s, d)} +end + +config = RbConfig::MAKEFILE_CONFIG.merge("prefix" => ".", "exec_prefix" => ".") +config.each_value {|s| RbConfig.expand(s, config)} +srcdir = config["srcdir"] ||= File.dirname(__FILE__) +top_srcdir = config["top_srcdir"] ||= File.dirname(srcdir) +extout = ARGV[0] || config["EXTOUT"] +arch = config["arch"] +bindir = config["bindir"] +libdirname = config["libdirname"] +libdir = config[libdirname || "libdir"] +vendordir = config["vendordir"] +rubylibdir = config["rubylibdir"] +rubyarchdir = config["rubyarchdir"] +archdir = "#{extout}/#{arch}" +[bindir, libdir, archdir].uniq.each do |dir| + File.directory?(dir) or mkdir_p(dir) +end + +exeext = config["EXEEXT"] +ruby_install_name = config["ruby_install_name"] +rubyw_install_name = config["rubyw_install_name"] +goruby_install_name = "go" + ruby_install_name +[ruby_install_name, rubyw_install_name, goruby_install_name].map do |ruby| + if ruby and !ruby.empty? + ruby += exeext + ln_relative(ruby, "#{bindir}/#{ruby}", true) + end +end +so = config["LIBRUBY_SO"] +libruby = [config["LIBRUBY_A"]] +if /\.dll\z/i =~ so + ln_relative(so, "#{bindir}/#{so}") +else + libruby << so +end +libruby.concat(config["LIBRUBY_ALIASES"].split) +libruby.each {|lib|ln_relative(lib, "#{libdir}/#{lib}")} +ln_dir_relative("#{extout}/common", rubylibdir) +rubyarchdir.sub!(rubylibdir, "#{extout}/common") +vendordir.sub!(rubylibdir, "#{extout}/common") +ln_dir_relative(archdir, rubyarchdir) +vendordir.sub!(rubyarchdir, archdir) +ln_dir_relative("#{top_srcdir}/lib", vendordir) +ln_relative("rbconfig.rb", "#{archdir}/rbconfig.rb") diff --git a/tool/node_name.rb b/tool/node_name.rb new file mode 100755 index 0000000000..dc0584e821 --- /dev/null +++ b/tool/node_name.rb @@ -0,0 +1,8 @@ +#! ./miniruby -n + +# Used when making Ruby to generate node_name.inc. +# See common.mk for details. + +if (t ||= /^enum node_type \{/ =~ $_) and (t = /^\};/ !~ $_) + /(NODE_.+),/ =~ $_ and puts(" case #{$1}:\n\treturn \"#{$1}\";") +end diff --git a/tool/parse.rb b/tool/parse.rb new file mode 100644 index 0000000000..93ae3e43cb --- /dev/null +++ b/tool/parse.rb @@ -0,0 +1,16 @@ +# Used as part of the "make parse" Makefile target. +# See common.mk for details. + +$file = ARGV[0] +$str = ARGF.read.sub(/^__END__.*\z/m, '') +puts '# ' + '-' * 70 +puts "# target program: " +puts '# ' + '-' * 70 +puts $str +puts '# ' + '-' * 70 + +$parsed = RubyVM::InstructionSequence.compile_file($file) +puts "# disasm result: " +puts '# ' + '-' * 70 +puts $parsed.disasm +puts '# ' + '-' * 70 diff --git a/tool/prereq.status b/tool/prereq.status new file mode 100644 index 0000000000..6de00c8a92 --- /dev/null +++ b/tool/prereq.status @@ -0,0 +1,44 @@ +s,@EXTOUT@,tmp,g +s,@ruby_version@,0.0.0,g +s,@NULLCMD@,:,g +s,@ARCH_FLAG@,,g +s,@ASMEXT@,S,g +s,@BASERUBY@,ruby,g +s,@BOOTSTRAPRUBY@,$(BASERUBY),g +s,@CC@,false,g +s,@CFLAGS@,,g +s,@CHDIR@,cd,g +s,@CONFIGURE@,configure,g +s,@CP@,cp,g +s,@CPPFLAGS@,,g +s,@CXXFLAGS@,,g +s,@DLDFLAGS@,,g +s,@DTRACE_EXT@,dmyh,g +s,@EXEEXT@,,g +s,@HAVE_BASERUBY@,yes,g +s,@IFCHANGE@,tool/ifchange,g +s,@LDFLAGS@,,g +s,@LIBEXT@,a,g +s,@LIBRUBY@,libruby.a,g +s,@LIBRUBY_A@,libruby.a,g +s,@MINIRUBY@,$(BASERUBY),g +s,@MKDIR_P@,mkdir -p,g +s,@OBJEXT@,o,g +s,@PATH_SEPARATOR@,:,g +s,@PWD@,.,g +s,@RM@,rm -f,g +s,@RMALL@,rm -fr,g +s,@RMDIR@,rmdir,g +s,@RMDIRS@,$(RMDIR) -p,g +s,@RUBY@,$(BASERUBY),g +s,@RUNRUBY@,$(MINIRUBY),g +s,@arch@,noarch,g +s,@bindir@,,g +s,@configure_args@,,g +s,@ruby_install_name@,,g +s,@rubyarchdir@,,g +s,@rubylibprefix@,,g +s,@srcdir@,.,g + +s/@[A-Za-z][A-Za-z0-9_]*@//g +s/{\$([A-Za-z]*)}//g diff --git a/tool/probes_to_wiki.rb b/tool/probes_to_wiki.rb new file mode 100644 index 0000000000..ba8204c188 --- /dev/null +++ b/tool/probes_to_wiki.rb @@ -0,0 +1,16 @@ +### +# Converts the probes.d file to redmine wiki format. Usage: +# +# ruby tool/probes_to_wiki.rb probes.d + +File.read(ARGV[0]).scan(/\/\*.*?\*\//m).grep(/ruby/) do |comment| + comment.gsub!(/^(\/\*|[ ]*)|\*\/$/, '').strip! + puts + comment.each_line.with_index do |line, i| + if i == 0 + puts "=== #{line.chomp}" + else + puts line.gsub(/`([^`]*)`/, '(({\1}))') + end + end +end diff --git a/tool/pure_parser.rb b/tool/pure_parser.rb new file mode 100755 index 0000000000..21c87cc5d6 --- /dev/null +++ b/tool/pure_parser.rb @@ -0,0 +1,24 @@ +#!/usr/bin/ruby -pi.bak +BEGIN { + # pathological setting + ENV['LANG'] = ENV['LC_MESSAGES'] = ENV['LC_ALL'] = 'C' + + require_relative 'lib/colorize' + + colorize = Colorize.new + file = ARGV.shift + begin + version = IO.popen(ARGV+%w[--version], "rb", &:read) + rescue Errno::ENOENT + abort "Failed to run `#{colorize.fail ARGV.join(' ')}'; You may have to install it." + end + unless /\Abison .* (\d+)\.\d+/ =~ version + puts colorize.fail("not bison") + exit + end + exit if $1.to_i >= 3 + ARGV.clear + ARGV.push(file) +} +$_.sub!(/^%define\s+api\.pure/, '%pure-parser') +$_.sub!(/^%define\s+.*/, '') diff --git a/tool/rbinstall.rb b/tool/rbinstall.rb new file mode 100755 index 0000000000..ec232fb896 --- /dev/null +++ b/tool/rbinstall.rb @@ -0,0 +1,1142 @@ +#!./miniruby + +# Used by the "make install" target to install Ruby. +# See common.mk for more details. + +ENV["SDKROOT"] ||= "" if /darwin/ =~ RUBY_PLATFORM + +begin + load "./rbconfig.rb" +rescue LoadError + CONFIG = Hash.new {""} +else + include RbConfig + $".unshift File.expand_path("./rbconfig.rb") +end + +srcdir = File.expand_path('../..', __FILE__) +unless defined?(CROSS_COMPILING) and CROSS_COMPILING + $:.replace([srcdir+"/lib", Dir.pwd]) +end +require 'fileutils' +require 'shellwords' +require 'optparse' +require 'optparse/shellwords' +require 'rubygems' +begin + require "zlib" +rescue LoadError + $" << "zlib.rb" +end + +INDENT = " "*36 +STDOUT.sync = true +File.umask(022) + +def parse_args(argv = ARGV) + $mantype = 'doc' + $destdir = nil + $extout = nil + $make = 'make' + $mflags = [] + $install = [] + $installed = {} + $installed_list = nil + $exclude = [] + $dryrun = false + $rdocdir = nil + $htmldir = nil + $data_mode = 0644 + $prog_mode = 0755 + $dir_mode = nil + $script_mode = nil + $strip = false + $debug_symbols = nil + $cmdtype = (if File::ALT_SEPARATOR == '\\' + File.exist?("rubystub.exe") ? 'exe' : 'cmd' + end) + mflags = [] + gnumake = false + opt = OptionParser.new + opt.on('-n', '--dry-run') {$dryrun = true} + opt.on('--dest-dir=DIR') {|dir| $destdir = dir} + opt.on('--extout=DIR') {|dir| $extout = (dir unless dir.empty?)} + opt.on('--ext-build-dir=DIR') {|v| $ext_build_dir = v } + opt.on('--make=COMMAND') {|make| $make = make} + opt.on('--mantype=MAN') {|man| $mantype = man} + opt.on('--make-flags=FLAGS', '--mflags', Shellwords) do |v| + if arg = v.first + arg.insert(0, '-') if /\A[^-][^=]*\Z/ =~ arg + end + $mflags.concat(v) + end + opt.on('-i', '--install=TYPE', $install_procs.keys) do |ins| + $install << ins + end + opt.on('-x', '--exclude=TYPE', $install_procs.keys) do |exc| + $exclude << exc + end + opt.on('--data-mode=OCTAL-MODE', OptionParser::OctalInteger) do |mode| + $data_mode = mode + end + opt.on('--prog-mode=OCTAL-MODE', OptionParser::OctalInteger) do |mode| + $prog_mode = mode + end + opt.on('--dir-mode=OCTAL-MODE', OptionParser::OctalInteger) do |mode| + $dir_mode = mode + end + opt.on('--script-mode=OCTAL-MODE', OptionParser::OctalInteger) do |mode| + $script_mode = mode + end + opt.on('--installed-list [FILENAME]') {|name| $installed_list = name} + opt.on('--rdoc-output [DIR]') {|dir| $rdocdir = dir} + opt.on('--html-output [DIR]') {|dir| $htmldir = dir} + opt.on('--cmd-type=TYPE', %w[cmd plain]) {|cmd| $cmdtype = (cmd unless cmd == 'plain')} + opt.on('--[no-]strip') {|strip| $strip = strip} + opt.on('--gnumake') {gnumake = true} + opt.on('--debug-symbols=SUFFIX', /\w+/) {|name| $debug_symbols = ".#{name}"} + + opt.order!(argv) do |v| + case v + when /\AINSTALL[-_]([-\w]+)=(.*)/ + argv.unshift("--#{$1.tr('_', '-')}=#{$2}") + when /\A\w[-\w]*=/ + mflags << v + when /\A\w[-\w+]*\z/ + $install << v.intern + else + raise OptionParser::InvalidArgument, v + end + end rescue abort "#{$!.message}\n#{opt.help}" + + unless defined?(RbConfig) + puts opt.help + exit + end + + $make, *rest = Shellwords.shellwords($make) + $mflags.unshift(*rest) unless rest.empty? + $mflags.unshift(*mflags) + $mflags.reject! {|v| /\A-[OW]/ =~ v} if gnumake + + def $mflags.set?(flag) + grep(/\A-(?!-).*#{flag.chr}/i) { return true } + false + end + def $mflags.defined?(var) + grep(/\A#{var}=(.*)/) {return block_given? ? yield($1) : $1} + false + end + + if $mflags.set?(?n) + $dryrun = true + else + $mflags << '-n' if $dryrun + end + + $destdir ||= $mflags.defined?("DESTDIR") + if $extout ||= $mflags.defined?("EXTOUT") + RbConfig.expand($extout) + end + + $continue = $mflags.set?(?k) + + if $installed_list ||= $mflags.defined?('INSTALLED_LIST') + RbConfig.expand($installed_list, RbConfig::CONFIG) + $installed_list = open($installed_list, "ab") + $installed_list.sync = true + end + + $rdocdir ||= $mflags.defined?('RDOCOUT') + $htmldir ||= $mflags.defined?('HTMLOUT') + + $dir_mode ||= $prog_mode | 0700 + $script_mode ||= $prog_mode + if $ext_build_dir.nil? + raise OptionParser::MissingArgument.new("--ext-build-dir=DIR") + end +end + +$install_procs = Hash.new {[]} +def install?(*types, &block) + unless types.delete(:nodefault) + $install_procs[:all] <<= block + end + types.each do |type| + $install_procs[type] <<= block + end +end + +def strip_file(files) + if !defined?($strip_command) and (cmd = CONFIG["STRIP"]) + case cmd + when "", "true", ":" then return + else $strip_command = Shellwords.shellwords(cmd) + end + elsif !$strip_command + return + end + system(*($strip_command + [files].flatten)) +end + +def install(src, dest, options = {}) + options = options.clone + strip = options.delete(:strip) + options[:preserve] = true + srcs = Array(src).select {|s| !$installed[$made_dirs[dest] ? File.join(dest, s) : dest]} + return if srcs.empty? + src = srcs if Array === src + d = with_destdir(dest) + super(src, d, **options) + srcs.each {|s| $installed[$made_dirs[dest] ? File.join(dest, s) : dest] = true} + if strip + d = srcs.map {|s| File.join(d, File.basename(s))} if $made_dirs[dest] + strip_file(d) + end + if $installed_list + dest = srcs.map {|s| File.join(dest, File.basename(s))} if $made_dirs[dest] + $installed_list.puts dest + end +end + +def ln_sf(src, dest) + super(src, with_destdir(dest)) + $installed_list.puts dest if $installed_list +end + +$made_dirs = {} +def makedirs(dirs) + dirs = fu_list(dirs) + dirs.collect! do |dir| + realdir = with_destdir(dir) + realdir unless $made_dirs.fetch(dir) do + $made_dirs[dir] = true + $installed_list.puts(File.join(dir, "")) if $installed_list + File.directory?(realdir) + end + end.compact! + super(dirs, :mode => $dir_mode) unless dirs.empty? +end + +FalseProc = proc {false} +def path_matcher(pat) + if pat and !pat.empty? + proc {|f| pat.any? {|n| File.fnmatch?(n, f)}} + else + FalseProc + end +end + +def install_recursive(srcdir, dest, options = {}) + opts = options.clone + noinst = opts.delete(:no_install) + glob = opts.delete(:glob) || "*" + maxdepth = opts.delete(:maxdepth) + subpath = (srcdir.size+1)..-1 + prune = [] + skip = [] + if noinst + if Array === noinst + prune = noinst.grep(/#{File::SEPARATOR}/o).map!{|f| f.chomp(File::SEPARATOR)} + skip = noinst.grep(/\A[^#{File::SEPARATOR}]*\z/o) + else + if noinst.index(File::SEPARATOR) + prune = [noinst] + else + skip = [noinst] + end + end + end + skip |= %w"#*# *~ *.old *.bak *.orig *.rej *.diff *.patch *.core" + prune = path_matcher(prune) + skip = path_matcher(skip) + File.directory?(srcdir) or return rescue return + paths = [[srcdir, dest, 0]] + found = [] + while file = paths.shift + found << file + file, d, dir = *file + if dir + depth = dir + 1 + next if maxdepth and maxdepth < depth + files = [] + Dir.foreach(file) do |f| + src = File.join(file, f) + d = File.join(dest, dir = src[subpath]) + stat = File.lstat(src) rescue next + if stat.directory? + files << [src, d, depth] if maxdepth != depth and /\A\./ !~ f and !prune[dir] + elsif stat.symlink? + # skip + else + files << [src, d, false] if File.fnmatch?(glob, f, File::FNM_EXTGLOB) and !skip[f] + end + end + paths.insert(0, *files) + end + end + for src, d, dir in found + if dir + next + # makedirs(d) + else + makedirs(d[/.*(?=\/)/m]) + if block_given? + yield src, d, opts + else + install src, d, opts + end + end + end +end + +def open_for_install(path, mode) + data = open(realpath = with_destdir(path), "rb") {|f| f.read} rescue nil + newdata = yield + unless $dryrun + unless newdata == data + open(realpath, "wb", mode) {|f| f.write newdata} + end + File.chmod(mode, realpath) + end + $installed_list.puts path if $installed_list +end + +def with_destdir(dir) + return dir if !$destdir or $destdir.empty? + dir = dir.sub(/\A\w:/, '') if File::PATH_SEPARATOR == ';' + $destdir + dir +end + +def without_destdir(dir) + return dir if !$destdir or $destdir.empty? + dir.start_with?($destdir) ? dir[$destdir.size..-1] : dir +end + +def prepare(mesg, basedir, subdirs=nil) + return unless basedir + case + when !subdirs + dirs = basedir + when subdirs.size == 0 + subdirs = nil + when subdirs.size == 1 + dirs = [basedir = File.join(basedir, subdirs)] + subdirs = nil + else + dirs = [basedir, *subdirs.collect {|dir| File.join(basedir, dir)}] + end + printf("%-*s%s%s\n", INDENT.size, "installing #{mesg}:", basedir, + (subdirs ? " (#{subdirs.join(', ')})" : "")) + makedirs(dirs) +end + +def CONFIG.[](name, mandatory = false) + value = super(name) + if mandatory + raise "CONFIG['#{name}'] must be set" if !value or value.empty? + end + value +end + +exeext = CONFIG["EXEEXT"] + +ruby_install_name = CONFIG["ruby_install_name", true] +rubyw_install_name = CONFIG["rubyw_install_name"] +goruby_install_name = "go" + ruby_install_name + +bindir = CONFIG["bindir", true] +libdir = CONFIG[CONFIG.fetch("libdirname", "libdir"), true] +rubyhdrdir = CONFIG["rubyhdrdir", true] +archhdrdir = CONFIG["rubyarchhdrdir"] || (rubyhdrdir + "/" + CONFIG['arch']) +rubylibdir = CONFIG["rubylibdir", true] +archlibdir = CONFIG["rubyarchdir", true] +if CONFIG["sitedir"] + sitelibdir = CONFIG["sitelibdir"] + sitearchlibdir = CONFIG["sitearchdir"] +end +if CONFIG["vendordir"] + vendorlibdir = CONFIG["vendorlibdir"] + vendorarchlibdir = CONFIG["vendorarchdir"] +end +mandir = CONFIG["mandir", true] +docdir = CONFIG["docdir", true] +enable_shared = CONFIG["ENABLE_SHARED"] == 'yes' +dll = CONFIG["LIBRUBY_SO", enable_shared] +lib = CONFIG["LIBRUBY", true] +arc = CONFIG["LIBRUBY_A", true] +load_relative = CONFIG["LIBRUBY_RELATIVE"] == 'yes' + +rdoc_noinst = %w[created.rid] + +install?(:local, :arch, :bin, :'bin-arch') do + prepare "binary commands", bindir + + install ruby_install_name+exeext, bindir, :mode => $prog_mode, :strip => $strip + if rubyw_install_name and !rubyw_install_name.empty? + install rubyw_install_name+exeext, bindir, :mode => $prog_mode, :strip => $strip + end + if File.exist? goruby_install_name+exeext + install goruby_install_name+exeext, bindir, :mode => $prog_mode, :strip => $strip + end + if enable_shared and dll != lib + install dll, bindir, :mode => $prog_mode, :strip => $strip + end +end + +install?(:local, :arch, :lib, :'lib-arch') do + prepare "base libraries", libdir + + install lib, libdir, :mode => $prog_mode, :strip => $strip unless lib == arc + install arc, libdir, :mode => $data_mode unless CONFIG["INSTALL_STATIC_LIBRARY"] == "no" + if dll == lib and dll != arc + for link in CONFIG["LIBRUBY_ALIASES"].split - [File.basename(dll)] + ln_sf(dll, File.join(libdir, link)) + end + end + + prepare "arch files", archlibdir + install "rbconfig.rb", archlibdir, :mode => $data_mode + if CONFIG["ARCHFILE"] + for file in CONFIG["ARCHFILE"].split + install file, archlibdir, :mode => $data_mode + end + end +end + +install?(:local, :arch, :data) do + pc = CONFIG["ruby_pc"] + if pc and File.file?(pc) and File.size?(pc) + prepare "pkgconfig data", pkgconfigdir = File.join(libdir, "pkgconfig") + install pc, pkgconfigdir, :mode => $data_mode + end +end + +install?(:ext, :arch, :'ext-arch') do + prepare "extension objects", archlibdir + noinst = %w[-* -*/] | (CONFIG["no_install_files"] || "").split + install_recursive("#{$extout}/#{CONFIG['arch']}", archlibdir, :no_install => noinst, :mode => $prog_mode, :strip => $strip) + prepare "extension objects", sitearchlibdir + prepare "extension objects", vendorarchlibdir + if extso = File.read("exts.mk")[/^EXTSO[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] and + !(extso = extso.gsub(/\\\n/, '').split).empty? + libpathenv = CONFIG["LIBPATHENV"] + dest = CONFIG[!libpathenv || libpathenv == "PATH" ? "bindir" : "libdir"] + prepare "external libraries", dest + for file in extso + install file, dest, :mode => $prog_mode + end + end +end +install?(:ext, :arch, :hdr, :'arch-hdr', :'hdr-arch') do + prepare "extension headers", archhdrdir + install_recursive("#{$extout}/include/#{CONFIG['arch']}", archhdrdir, :glob => "*.h", :mode => $data_mode) + install_recursive("#{$extout}/include/#{CONFIG['arch']}", archhdrdir, :glob => "rb_mjit_header-*.obj", :mode => $data_mode) + install_recursive("#{$extout}/include/#{CONFIG['arch']}", archhdrdir, :glob => "rb_mjit_header-*.pch", :mode => $data_mode) +end +install?(:ext, :comm, :'ext-comm') do + prepare "extension scripts", rubylibdir + install_recursive("#{$extout}/common", rubylibdir, :mode => $data_mode) + prepare "extension scripts", sitelibdir + prepare "extension scripts", vendorlibdir +end +install?(:ext, :comm, :hdr, :'comm-hdr', :'hdr-comm') do + hdrdir = rubyhdrdir + "/ruby" + prepare "extension headers", hdrdir + install_recursive("#{$extout}/include/ruby", hdrdir, :glob => "*.h", :mode => $data_mode) +end + +install?(:doc, :rdoc) do + if $rdocdir + ridatadir = File.join(CONFIG['ridir'], CONFIG['ruby_version'], "system") + prepare "rdoc", ridatadir + install_recursive($rdocdir, ridatadir, :no_install => rdoc_noinst, :mode => $data_mode) + end +end +install?(:doc, :html) do + if $htmldir + prepare "html-docs", docdir + install_recursive($htmldir, docdir+"/html", :no_install => rdoc_noinst, :mode => $data_mode) + end +end +install?(:doc, :capi) do + prepare "capi-docs", docdir + install_recursive "doc/capi", docdir+"/capi", :mode => $data_mode +end + +prolog_script = <<EOS +bindir="#{load_relative ? '${0%/*}' : bindir.gsub(/\"/, '\\\\"')}" +EOS +if CONFIG["LIBRUBY_RELATIVE"] != 'yes' and libpathenv = CONFIG["LIBPATHENV"] + pathsep = File::PATH_SEPARATOR + prolog_script << <<EOS +libdir="#{load_relative ? '$\{bindir%/bin\}/lib' : libdir.gsub(/\"/, '\\\\"')}" +export #{libpathenv}="$libdir${#{libpathenv}:+#{pathsep}$#{libpathenv}}" +EOS +end +prolog_script << %Q[exec "$bindir/#{ruby_install_name}" "-x" "$0" "$@"\n] +PROLOG_SCRIPT = {} +PROLOG_SCRIPT["exe"] = "#!#{bindir}/#{ruby_install_name}" +PROLOG_SCRIPT["cmd"] = <<EOS +:""||{ ""=> %q<-*- ruby -*- +@"%~dp0#{ruby_install_name}" -x "%~f0" %* +@exit /b %ERRORLEVEL% +};{ #\n#{prolog_script.gsub(/(?=\n)/, ' #')}>,\n} +EOS +PROLOG_SCRIPT.default = (load_relative || /\s/ =~ bindir) ? + <<EOS : PROLOG_SCRIPT["exe"] +#!/bin/sh +# -*- ruby -*- +_=_\\ +=begin +#{prolog_script.chomp} +=end +EOS + +installer = Struct.new(:ruby_shebang, :ruby_bin, :ruby_install_name, :stub, :trans) do + def transform(name) + RbConfig.expand(trans[name]) + end +end + +$script_installer = Class.new(installer) do + ruby_shebang = File.join(bindir, ruby_install_name) + if File::ALT_SEPARATOR + ruby_bin = ruby_shebang.tr(File::SEPARATOR, File::ALT_SEPARATOR) + end + if trans = CONFIG["program_transform_name"] + exp = [] + trans.gsub!(/\$\$/, '$') + trans.scan(%r[\G[\s;]*(/(?:\\.|[^/])*+/)?([sy])(\\?\W)((?:(?!\3)(?:\\.|.))*+)\3((?:(?!\3)(?:\\.|.))*+)\3([gi]*)]) do + |addr, cmd, sep, pat, rep, opt| + addr &&= Regexp.new(addr[/\A\/(.*)\/\z/, 1]) + case cmd + when 's' + next if pat == '^' and rep.empty? + exp << [addr, (opt.include?('g') ? :gsub! : :sub!), + Regexp.new(pat, opt.include?('i')), rep.gsub(/&/){'\&'}] + when 'y' + exp << [addr, :tr!, Regexp.quote(pat), rep] + end + end + trans = proc do |base| + exp.each {|addr, opt, pat, rep| base.__send__(opt, pat, rep) if !addr or addr =~ base} + base + end + elsif /ruby/ =~ ruby_install_name + trans = proc {|base| ruby_install_name.sub(/ruby/, base)} + else + trans = proc {|base| base} + end + + def prolog(shebang) + shebang.sub!(/\r$/, '') + script = PROLOG_SCRIPT[$cmdtype] + shebang.sub!(/\A(\#!.*?ruby\b)?/) do + if script.end_with?("\n") + script + ($1 || "#!ruby\n") + else + $1 ? script : "#{script}\n" + end + end + shebang + end + + def install(src, cmd) + cmd = cmd.sub(/[^\/]*\z/m) {|n| transform(n)} + + shebang, body = open(src, "rb") do |f| + next f.gets, f.read + end + shebang or raise "empty file - #{src}" + shebang = prolog(shebang) + body.gsub!(/\r$/, '') + + cmd << ".#{$cmdtype}" if $cmdtype + open_for_install(cmd, $script_mode) do + case $cmdtype + when "exe" + stub + shebang + body + else + shebang + body + end + end + end + + def self.get_rubystub + stubfile = "rubystub.exe" + stub = File.open(stubfile, "rb") {|f| f.read} << "\n" + rescue => e + abort "No #{stubfile}: #{e}" + else + stub + end + + def stub + super or self.stub = self.class.get_rubystub + end + + break new(ruby_shebang, ruby_bin, ruby_install_name, nil, trans) +end + +install?(:local, :comm, :bin, :'bin-comm') do + prepare "command scripts", bindir + + install_recursive(File.join(srcdir, "bin"), bindir, :maxdepth => 1) do |src, cmd| + $script_installer.install(src, cmd) + end +end + +install?(:local, :comm, :lib) do + prepare "library scripts", rubylibdir + noinst = %w[*.txt *.rdoc *.gemspec] + install_recursive(File.join(srcdir, "lib"), rubylibdir, :no_install => noinst, :mode => $data_mode) +end + +install?(:local, :comm, :hdr, :'comm-hdr') do + prepare "common headers", rubyhdrdir + + noinst = [] + unless RUBY_PLATFORM =~ /mswin|mingw|bccwin/ + noinst << "win32.h" + end + noinst = nil if noinst.empty? + install_recursive(File.join(srcdir, "include"), rubyhdrdir, :no_install => noinst, :glob => "*.{h,hpp}", :mode => $data_mode) +end + +install?(:local, :comm, :man) do + mdocs = Dir["#{srcdir}/man/*.[1-9]"] + prepare "manpages", mandir, ([] | mdocs.collect {|mdoc| mdoc[/\d+$/]}).sort.collect {|sec| "man#{sec}"} + + case $mantype + when /\.(?:(gz)|bz2)\z/ + compress = $1 ? "gzip" : "bzip2" + suffix = $& + end + mandir = File.join(mandir, "man") + has_goruby = File.exist?(goruby_install_name+exeext) + require File.join(srcdir, "tool/mdoc2man.rb") if /\Adoc\b/ !~ $mantype + mdocs.each do |mdoc| + next unless File.file?(mdoc) and open(mdoc){|fh| fh.read(1) == '.'} + base = File.basename(mdoc) + if base == "goruby.1" + next unless has_goruby + end + + destdir = mandir + (section = mdoc[/\d+$/]) + destname = ruby_install_name.sub(/ruby/, base.chomp(".#{section}")) + destfile = File.join(destdir, "#{destname}.#{section}") + + if /\Adoc\b/ =~ $mantype + if compress + w = open(mdoc) {|f| + stdin = STDIN.dup + STDIN.reopen(f) + begin + destfile << suffix + IO.popen(compress, &:read) + ensure + STDIN.reopen(stdin) + stdin.close + end + } + open_for_install(destfile, $data_mode) {w} + else + install mdoc, destfile, :mode => $data_mode + end + else + class << (w = []) + alias print push + end + if File.basename(mdoc).start_with?('bundle') || + File.basename(mdoc).start_with?('gemfile') + w = File.read(mdoc) + else + open(mdoc) {|r| Mdoc2Man.mdoc2man(r, w)} + w = w.join("") + end + if compress + require 'tmpdir' + Dir.mktmpdir("man") {|d| + dest = File.join(d, File.basename(destfile)) + File.open(dest, "wb") {|f| f.write w} + if system(compress, dest) + w = File.open(dest+suffix, "rb") {|f| f.read} + destfile << suffix + end + } + end + open_for_install(destfile, $data_mode) {w} + end + end +end + +install?(:dbg, :nodefault) do + prepare "debugger commands", bindir + prepare "debugger scripts", rubylibdir + conf = RbConfig::MAKEFILE_CONFIG.merge({"prefix"=>"${prefix#/}"}) + Dir.glob(File.join(srcdir, "template/ruby-*db.in")) do |src| + cmd = $script_installer.transform(File.basename(src, ".in")) + open_for_install(File.join(bindir, cmd), $script_mode) { + RbConfig.expand(File.read(src), conf) + } + end + install File.join(srcdir, "misc/lldb_cruby.py"), File.join(rubylibdir, "lldb_cruby.py") + install File.join(srcdir, ".gdbinit"), File.join(rubylibdir, "gdbinit") + if $debug_symbols + { + ruby_install_name => bindir, + rubyw_install_name => bindir, + goruby_install_name => bindir, + dll => libdir, + }.each do |src, dest| + next if src.empty? + src += $debug_symbols + if File.directory?(src) + install_recursive src, File.join(dest, src) + end + end + end +end + +module RbInstall + def self.no_write(options = nil) + u = File.umask(0022) + if $dryrun + fu = ::Object.class_eval do + fu = remove_const(:FileUtils) + const_set(:FileUtils, fu::NoWrite) + fu + end + dir_mode = options.delete(:dir_mode) if options + end + yield + ensure + options[:dir_mode] = dir_mode if dir_mode + if fu + ::Object.class_eval do + remove_const(:FileUtils) + const_set(:FileUtils, fu) + end + end + File.umask(u) + end + + module Specs + class FileCollector + def initialize(gemspec) + @gemspec = gemspec + @base_dir = File.dirname(gemspec) + end + + def collect + (ruby_libraries + built_libraries).sort + end + + def skip_install?(files) + case type + when "ext" + # install ext only when it's configured + !File.exist?("#{$ext_build_dir}/#{relative_base}/Makefile") + when "lib" + files.empty? + end + end + + private + def type + /\/(ext|lib)?\/.*?\z/ =~ @base_dir + $1 + end + + def ruby_libraries + case type + when "ext" + prefix = "#{$extout}/common/" + base = "#{prefix}#{relative_base}" + when "lib" + base = @base_dir + prefix = base.sub(/lib\/.*?\z/, "") + # for lib/net/net-smtp.gemspec + if m = File.basename(@gemspec, ".gemspec").match(/.*\-(.*)\z/) + base = "#{@base_dir}/#{m[1]}" unless remove_prefix(prefix, @base_dir).include?(m[1]) + end + end + + files = if base + Dir.glob("#{base}{.rb,/**/*.rb}").collect do |ruby_source| + remove_prefix(prefix, ruby_source) + end + else + [File.basename(@gemspec, '.gemspec') + '.rb'] + end + + case File.basename(@gemspec, ".gemspec") + when "net-http" + files << "lib/net/https.rb" + when "optparse" + files << "lib/optionparser.rb" + end + + files + end + + def built_libraries + case type + when "ext" + prefix = "#{$extout}/#{CONFIG['arch']}/" + base = "#{prefix}#{relative_base}" + dlext = CONFIG['DLEXT'] + Dir.glob("#{base}{.#{dlext},/**/*.#{dlext}}").collect do |built_library| + remove_prefix(prefix, built_library) + end + when "lib" + [] + else + [] + end + end + + def relative_base + /\/#{Regexp.escape(type)}\/(.*?)\z/ =~ @base_dir + $1 + end + + def remove_prefix(prefix, string) + string.sub(/\A#{Regexp.escape(prefix)}/, "") + end + end + end + + class DirPackage + attr_reader :spec + + attr_accessor :dir_mode + attr_accessor :prog_mode + attr_accessor :data_mode + + def initialize(spec, dir_map = nil) + @spec = spec + @src_dir = File.dirname(@spec.loaded_from) + @dir_map = dir_map + end + + def extract_files(destination_dir, pattern = "*") + return if @src_dir == destination_dir + File.chmod(0700, destination_dir) unless $dryrun + mode = pattern == File.join(spec.bindir, '*') ? prog_mode : data_mode + destdir = without_destdir(destination_dir) + if @dir_map + (dir_map = @dir_map.map {|k, v| Regexp.quote(k) unless k == v}).compact! + dir_map = %r{\A(?:#{dir_map.join('|')})(?=/)} + end + spec.files.each do |f| + next unless File.fnmatch(pattern, f) + src = File.join(@src_dir, dir_map =~ f ? "#{@dir_map[$&]}#{$'}" : f) + dest = File.join(destdir, f) + makedirs(dest[/.*(?=\/)/m]) + install src, dest, :mode => mode + end + File.chmod(dir_mode, destination_dir) unless $dryrun + end + end + + class GemInstaller < Gem::Installer + end + + class UnpackedInstaller < GemInstaller + def write_cache_file + end + + def shebang(bin_file_name) + path = File.join(gem_dir, spec.bindir, bin_file_name) + first_line = File.open(path, "rb") {|file| file.gets} + $script_installer.prolog(first_line).chomp + end + + def app_script_text(bin_file_name) + # move shell script part after comments generated by RubyGems. + super.sub(/\A + (\#!\/bin\/sh\n\#.*-\*-\s*ruby\s*-\*-.*\n) + ((?:.*\n)*?\#!.*ruby.*\n) + \#\n + ((?:\#.*\n)+)/x, '\1\3\2') + end + + def check_executable_overwrite(filename) + return if @wrappers and same_bin_script?(filename, @bin_dir) + super + end + + def generate_bin_script(filename, bindir) + return if same_bin_script?(filename, bindir) + super + end + + def same_bin_script?(filename, bindir) + path = File.join(bindir, formatted_program_filename(filename)) + begin + return true if File.binread(path) == app_script_text(filename) + rescue + end + false + end + + def write_spec + super unless $dryrun + $installed_list.puts(without_destdir(spec_file)) if $installed_list + end + + def write_default_spec + super unless $dryrun + $installed_list.puts(without_destdir(default_spec_file)) if $installed_list + end + end + + class GemInstaller + def install + spec.post_install_message = nil + RbInstall.no_write(options) {super} + end + + if RbConfig::CONFIG["LIBRUBY_RELATIVE"] == "yes" || RbConfig::CONFIG["CROSS_COMPILING"] == "yes" || ENV["DESTDIR"] + # TODO: always build extensions in bundled gems by build-ext and + # install the built binaries. + def build_extensions + end + end + + def generate_bin_script(filename, bindir) + name = formatted_program_filename(filename) + unless $dryrun + super + File.chmod($script_mode, File.join(bindir, name)) + end + $installed_list.puts(File.join(without_destdir(bindir), name)) if $installed_list + end + + def verify_gem_home # :nodoc: + end + + def ensure_writable_dir(dir) + $made_dirs.fetch(d = without_destdir(dir)) do + $made_dirs[d] = true + super unless $dryrun + $installed_list.puts(d+"/") if $installed_list + end + end + end +end + +# :startdoc: + +install?(:ext, :comm, :gem, :'default-gems', :'default-gems-comm') do + install_default_gem('lib', srcdir, bindir) +end +install?(:ext, :arch, :gem, :'default-gems', :'default-gems-arch') do + install_default_gem('ext', srcdir, bindir) +end + +def load_gemspec(file, base = nil) + file = File.realpath(file) + code = File.read(file, encoding: "utf-8:-") + code.gsub!(/(?:`git[^\`]*`|%x\[git[^\]]*\])\.split\([^\)]*\)/m) do + files = [] + if base + Dir.glob("**/*", File::FNM_DOTMATCH, base: base) do |n| + case File.basename(n); when ".", ".."; next; end + next if File.directory?(File.join(base, n)) + files << n.dump + end + end + "[" + files.join(", ") + "]" + end + spec = eval(code, binding, file) + unless Gem::Specification === spec + raise TypeError, "[#{file}] isn't a Gem::Specification (#{spec.class} instead)." + end + spec.loaded_from = base ? File.join(base, File.basename(file)) : file + spec.files.reject! {|n| n.end_with?(".gemspec") or n.start_with?(".git")} + + spec +end + +def install_default_gem(dir, srcdir, bindir) + gem_dir = Gem.default_dir + install_dir = with_destdir(gem_dir) + prepare "default gems from #{dir}", gem_dir + RbInstall.no_write do + makedirs(Gem.ensure_default_gem_subdirectories(install_dir, $dir_mode).map {|d| File.join(gem_dir, d)}) + end + + options = { + :install_dir => with_destdir(gem_dir), + :bin_dir => with_destdir(bindir), + :ignore_dependencies => true, + :dir_mode => $dir_mode, + :data_mode => $data_mode, + :prog_mode => $script_mode, + :wrappers => true, + :format_executable => true, + :install_as_default => true, + } + default_spec_dir = Gem.default_specifications_dir + + gems = Dir.glob("#{srcdir}/#{dir}/**/*.gemspec").map {|src| + spec = load_gemspec(src) + file_collector = RbInstall::Specs::FileCollector.new(src) + files = file_collector.collect + if file_collector.skip_install?(files) + next + end + spec.files = files + spec + } + gems.compact.sort_by(&:name).each do |gemspec| + old_gemspecs = Dir[File.join(with_destdir(default_spec_dir), "#{gemspec.name}-*.gemspec")] + if old_gemspecs.size > 0 + old_gemspecs.each {|spec| rm spec } + end + + full_name = "#{gemspec.name}-#{gemspec.version}" + + gemspec.loaded_from = File.join srcdir, gemspec.spec_name + + package = RbInstall::DirPackage.new gemspec, {gemspec.bindir => 'libexec'} + ins = RbInstall::UnpackedInstaller.new(package, options) + puts "#{INDENT}#{gemspec.name} #{gemspec.version}" + ins.install + end +end + +install?(:ext, :comm, :gem, :'bundled-gems') do + gem_dir = Gem.default_dir + install_dir = with_destdir(gem_dir) + prepare "bundled gems", gem_dir + RbInstall.no_write do + makedirs(Gem.ensure_gem_subdirectories(install_dir, $dir_mode).map {|d| File.join(gem_dir, d)}) + end + + installed_gems = {} + skipped = {} + options = { + :install_dir => install_dir, + :bin_dir => with_destdir(bindir), + :domain => :local, + :ignore_dependencies => true, + :dir_mode => $dir_mode, + :data_mode => $data_mode, + :prog_mode => $script_mode, + :wrappers => true, + :format_executable => true, + } + + extensions_dir = Gem::StubSpecification.gemspec_stub("", gem_dir, gem_dir).extensions_dir + specifications_dir = File.join(gem_dir, "specifications") + build_dir = Gem::StubSpecification.gemspec_stub("", ".bundle", ".bundle").extensions_dir + + # We are about to build extensions, and want to configure extensions with the + # newly installed ruby. + Gem.instance_variable_set(:@ruby, with_destdir(File.join(bindir, ruby_install_name))) + # Prevent fake.rb propagation. It conflicts with the natural mkmf configs of + # the newly installed ruby. + ENV.delete('RUBYOPT') + + File.foreach("#{srcdir}/gems/bundled_gems") do |name| + next if /^\s*(?:#|$)/ =~ name + next unless /^(\S+)\s+(\S+).*/ =~ name + gem = $1 + gem_name = "#$1-#$2" + # Try to find the original gemspec file + path = "#{srcdir}/.bundle/gems/#{gem_name}/#{gem}.gemspec" + unless File.exist?(path) + # Try to find the gemspec file for C ext gems + # ex .bundle/gems/debug-1.7.1/debug-1.7.1.gemspec + # This gemspec keep the original dependencies + path = "#{srcdir}/.bundle/gems/#{gem_name}/#{gem_name}.gemspec" + unless File.exist?(path) + # Try to find the gemspec file for gems that hasn't own gemspec + path = "#{srcdir}/.bundle/specifications/#{gem_name}.gemspec" + unless File.exist?(path) + skipped[gem_name] = "gemspec not found" + next + end + end + end + spec = load_gemspec(path, "#{srcdir}/.bundle/gems/#{gem_name}") + unless spec.platform == Gem::Platform::RUBY + skipped[gem_name] = "not ruby platform (#{spec.platform})" + next + end + unless spec.full_name == gem_name + skipped[gem_name] = "full name unmatch #{spec.full_name}" + next + end + spec.extension_dir = "#{extensions_dir}/#{spec.full_name}" + package = RbInstall::DirPackage.new spec + ins = RbInstall::UnpackedInstaller.new(package, options) + puts "#{INDENT}#{spec.name} #{spec.version}" + ins.install + install_recursive("#{build_dir}/#{gem_name}", "#{extensions_dir}/#{gem_name}") do |src, dest| + # puts "#{INDENT} #{dest[extensions_dir.size+gem_name.size+2..-1]}" + install src, dest, :mode => (File.executable?(src) ? $prog_mode : $data_mode) + end + installed_gems[spec.full_name] = true + end + installed_gems, gems = Dir.glob(srcdir+'/gems/*.gem').partition {|gem| installed_gems.key?(File.basename(gem, '.gem'))} + unless installed_gems.empty? + prepare "bundled gem cache", gem_dir+"/cache" + install installed_gems, gem_dir+"/cache" + end + next if gems.empty? + if defined?(Zlib) + silent = Gem::SilentUI.new + gems.each do |gem| + package = Gem::Package.new(gem) + inst = RbInstall::GemInstaller.new(package, options) + inst.spec.extension_dir = "#{extensions_dir}/#{inst.spec.full_name}" + begin + Gem::DefaultUserInteraction.use_ui(silent) {inst.install} + rescue Gem::InstallError + next + end + gemname = File.basename(gem) + puts "#{INDENT}#{gemname}" + end + # fix directory permissions + # TODO: Gem.install should accept :dir_mode option or something + File.chmod($dir_mode, *Dir.glob(install_dir+"/**/")) + # fix .gemspec permissions + File.chmod($data_mode, *Dir.glob(install_dir+"/specifications/*.gemspec")) + else + puts "skip installing bundled gems because of lacking zlib" + end +end + +parse_args() + +include FileUtils +include FileUtils::NoWrite if $dryrun +@fileutils_output = STDOUT +@fileutils_label = '' + +$install << :all if $install.empty? +installs = $install.map do |inst| + if !(procs = $install_procs[inst]) || procs.empty? + next warn("unknown install target - #{inst}") + end + procs +end +installs.flatten! +installs -= $exclude.map {|exc| $install_procs[exc]}.flatten +puts "Installing to #$destdir" unless installs.empty? +installs.each do |block| + dir = Dir.pwd + begin + block.call + ensure + Dir.chdir(dir) + end +end + +# vi:set sw=2: diff --git a/tool/rbuninstall.rb b/tool/rbuninstall.rb new file mode 100755 index 0000000000..f0c286012c --- /dev/null +++ b/tool/rbuninstall.rb @@ -0,0 +1,73 @@ +#! /usr/bin/ruby -nl + +# Used by the "make uninstall" target to uninstall Ruby. +# See common.mk for more details. + +BEGIN { + $dryrun = false + $tty = STDOUT.tty? + until ARGV.empty? + case ARGV[0] + when /\A--destdir=(.*)/ + $destdir = $1 + when /\A-n\z/ + $dryrun = true + when /\A--(?:no-)?tty\z/ + $tty = !$1 + else + break + end + ARGV.shift + end + $dirs = [] + $files = [] +} +list = ($_.chomp!('/') ? $dirs : $files) +list << $_ +END { + status = true + $\ = nil + ors = (!$dryrun and $tty) ? "\e[K\r" : "\n" + $files.each do |file| + print "rm #{file}#{ors}" + unless $dryrun + file = File.join($destdir, file) if $destdir + begin + File.unlink(file) + rescue Errno::ENOENT + rescue + status = false + puts $! + end + end + end + unlink = {} + $dirs.each do |dir| + unlink[dir] = true + end + while dir = $dirs.pop + dir = File.dirname(dir) while File.basename(dir) == '.' + print "rmdir #{dir}#{ors}" + unless $dryrun + realdir = $destdir ? File.join($destdir, dir) : dir + begin + begin + unlink.delete(dir) + Dir.rmdir(realdir) + rescue Errno::ENOTDIR + raise unless File.symlink?(realdir) + File.unlink(realdir) + end + rescue Errno::ENOENT, Errno::ENOTEMPTY + rescue + status = false + puts $! + else + parent = File.dirname(dir) + $dirs.push(parent) unless parent == dir or unlink[parent] + end + end + end + print ors.chomp + exit(status) +} diff --git a/tool/redmine-backporter.rb b/tool/redmine-backporter.rb new file mode 100755 index 0000000000..843132ab3a --- /dev/null +++ b/tool/redmine-backporter.rb @@ -0,0 +1,507 @@ +#!/usr/bin/env ruby +require 'open-uri' +require 'openssl' +require 'net/http' +require 'json' +require 'io/console' +require 'stringio' +require 'strscan' +require 'optparse' +require 'abbrev' +require 'pp' +require 'shellwords' +require 'reline' + +opts = OptionParser.new +target_version = nil +repo_path = nil +api_key = nil +ssl_verify = true +opts.on('-k REDMINE_API_KEY', '--key=REDMINE_API_KEY', 'specify your REDMINE_API_KEY') {|v| api_key = v} +opts.on('-t TARGET_VERSION', '--target=TARGET_VARSION', /\A\d(?:\.\d)+\z/, 'specify target version (ex: 3.1)') {|v| target_version = v} +opts.on('-r RUBY_REPO_PATH', '--repository=RUBY_REPO_PATH', 'specify repository path') {|v| repo_path = v} +opts.on('--[no-]ssl-verify', TrueClass, 'use / not use SSL verify') {|v| ssl_verify = v} +opts.parse!(ARGV) + +http_options = {use_ssl: true} +http_options[:verify_mode] = OpenSSL::SSL::VERIFY_NONE unless ssl_verify +$openuri_options = {} +$openuri_options[:ssl_verify_mode] = OpenSSL::SSL::VERIFY_NONE unless ssl_verify + +TARGET_VERSION = target_version || ENV['TARGET_VERSION'] || (puts opts.help; raise 'need to specify TARGET_VERSION') +RUBY_REPO_PATH = repo_path || ENV['RUBY_REPO_PATH'] +BACKPORT_CF_KEY = 'cf_5' +STATUS_CLOSE = 5 +REDMINE_API_KEY = api_key || ENV['REDMINE_API_KEY'] || (puts opts.help; raise 'need to specify REDMINE_API_KEY') +REDMINE_BASE = 'https://bugs.ruby-lang.org' + +@query = { + 'f[]' => BACKPORT_CF_KEY, + "op[#{BACKPORT_CF_KEY}]" => '~', + "v[#{BACKPORT_CF_KEY}][]" => "\"#{TARGET_VERSION}: REQUIRED\"", + 'limit' => 40, + 'status_id' => STATUS_CLOSE, + 'sort' => 'updated_on' +} + +PRIORITIES = { + 'Low' => [:white, :blue], + 'Normal' => [], + 'High' => [:red], + 'Urgent' => [:red, :white], + 'Immediate' => [:red, :white, {underscore: true}], +} +COLORS = { + black: 30, + red: 31, + green: 32, + yellow: 33, + blue: 34, + magenta: 35, + cyan: 36, + white: 37, +} + +class String + def color(fore=nil, back=nil, opts={}, bold: false, underscore: false) + seq = "" + if bold || opts[:bold] + seq = seq + "\e[1m" + end + if underscore || opts[:underscore] + seq = seq + "\e[2m" + end + if fore + c = COLORS[fore] + raise "unknown foreground color #{fore}" unless c + seq = seq + "\e[#{c}m" + end + if back + c = COLORS[back] + raise "unknown background color #{back}" unless c + seq = seq + "\e[#{c + 10}m" + end + if seq.empty? + self + else + seq = seq + self + "\e[0m" + end + end +end + +class StringScanner + # lx: limit of x (columns of screen) + # ly: limit of y (rows of screen) + def getrows(lx, ly) + cp1 = charpos + x = 0 + y = 0 + until eos? + case c = getch + when "\r" + x = 0 + when "\n" + x = 0 + y += 1 + when "\t" + x += 8 + when /[\x00-\x7f]/ + # halfwidth + x += 1 + else + # fullwidth + x += 2 + end + + if x > lx + x = 0 + y += 1 + unscan + end + if y >= ly + return string[cp1...charpos] + end + end + string[cp1..-1] + end +end + +def more(sio) + console = IO.console + ly, lx = console.winsize + ly -= 1 + str = sio.string + cls = "\r" + (" " * lx) + "\r" + + ss = StringScanner.new(str) + + rows = ss.getrows(lx, ly) + puts rows + until ss.eos? + print ":" + case c = console.getch + when ' ' + rows = ss.getrows(lx, ly) + puts cls + rows + when 'j', "\r" + rows = ss.getrows(lx, 1) + puts cls + rows + when "q" + print cls + break + else + print "\b" + end + end +end + +def find_git_log(pattern) + `git #{RUBY_REPO_PATH ? "-C #{RUBY_REPO_PATH.shellescape}" : ""} log --grep="#{pattern}"` +end + +def has_commit(commit, branch) + base = RUBY_REPO_PATH ? ["-C", RUBY_REPO_PATH.shellescape] : nil + system("git", *base, "merge-base", "--is-ancestor", commit, branch) +end + +def show_last_journal(http, uri) + res = http.get("#{uri.path}?include=journals") + res.value + h = JSON(res.body) + x = h["issue"] + raise "no issue" unless x + x = x["journals"] + raise "no journals" unless x + x = x.last + puts "== #{x["user"]["name"]} (#{x["created_on"]})" + x["details"].each do |y| + puts JSON(y) + end + puts x["notes"] +end + +def merger_path + RUBY_PLATFORM =~ /mswin|mingw/ ? 'merger' : File.expand_path('../merger.rb', __FILE__) +end + +def backport_command_string + unless @changesets.respond_to?(:validated) + @changesets = @changesets.select do |c| + next false if c.match(/\A\d{1,6}\z/) # skip SVN revision + + # check if the Git revision is included in master + has_commit(c, "master") + end.sort_by do |changeset| + Integer(IO.popen(%W[git show -s --format=%ct #{changeset}], &:read)) + end + @changesets.define_singleton_method(:validated){true} + end + "#{merger_path} --ticket=#{@issue} #{@changesets.join(',')}" +end + +def status_char(obj) + case obj["name"] + when "Closed" + "C".color(bold: true) + else + obj["name"][0] + end +end + +console = IO.console +row, = console.winsize +@query['limit'] = row - 2 +puts "Redmine Backporter".color(bold: true) + " for Ruby #{TARGET_VERSION}" + +class CommandSyntaxError < RuntimeError; end +commands = { + "ls" => proc{|args| + raise CommandSyntaxError unless /\A(\d+)?\z/ =~ args + uri = URI(REDMINE_BASE+'/projects/ruby-master/issues.json?'+URI.encode_www_form(@query.dup.merge('page' => ($1 ? $1.to_i : 1)))) + # puts uri + res = JSON(uri.read($openuri_options)) + @issues = issues = res["issues"] + from = res["offset"] + 1 + total = res["total_count"] + closed = issues.count { |x, _| x["status"]["name"] == "Closed" } + to = from + issues.size - 1 + puts "#{from}-#{to} / #{total} (closed: #{closed})" + issues.each_with_index do |x, i| + id = "##{x["id"]}".color(*PRIORITIES[x["priority"]["name"]], bold: x["status"]["name"] == "Closed") + puts "#{'%2d' % i} #{id} #{x["priority"]["name"][0]} #{status_char(x["status"])} #{x["subject"][0,80]}" + end + }, + + "show" => proc{|args| + if /\A(\d+)\z/ =~ args + id = $1.to_i + id = @issues[id]["id"] if @issues && id < @issues.size + @issue = id + elsif @issue + id = @issue + else + raise CommandSyntaxError + end + uri = "#{REDMINE_BASE}/issues/#{id}" + uri = URI(uri+".json?include=children,attachments,relations,changesets,journals") + res = JSON(uri.read($openuri_options)) + i = res["issue"] + unless i["changesets"] + abort "You don't have view_changesets permission" + end + unless i["custom_fields"] + puts "The specified ticket \##{@issue} seems to be a feature ticket" + @issue = nil + next + end + id = "##{i["id"]}".color(*PRIORITIES[i["priority"]["name"]]) + sio = StringIO.new + sio.set_encoding("utf-8") + sio.puts <<eom +#{i["subject"].color(bold: true, underscore: true)} +#{i["project"]["name"]} [#{i["tracker"]["name"]} #{id}] #{i["status"]["name"]} (#{i["created_on"]}) +author: #{i["author"]["name"]} +assigned: #{i["assigned_to"].to_h["name"]} +eom + i["custom_fields"].each do |x| + sio.puts "%-10s: %s" % [x["name"], x["value"]] + end + #res["attachments"].each do |x| + #end + sio.puts i["description"] + sio.puts + sio.puts "= changesets".color(bold: true, underscore: true) + @changesets = [] + i["changesets"].each do |x| + @changesets << x["revision"] + sio.puts "== #{x["revision"]} #{x["committed_on"]} #{x["user"]["name"] rescue nil}".color(bold: true, underscore: true) + sio.puts x["comments"] + end + @changesets = @changesets.sort.uniq + if i["journals"] && !i["journals"].empty? + sio.puts "= journals".color(bold: true, underscore: true) + i["journals"].each do |x| + sio.puts "== #{x["user"]["name"]} (#{x["created_on"]})".color(bold: true, underscore: true) + x["details"].each do |y| + sio.puts JSON(y) + end + sio.puts x["notes"] + end + end + more(sio) + }, + + "rel" => proc{|args| + # this feature requires custom redmine which allows add_related_issue API + case args + when /\A\h{7,40}\z/ # Git + rev = args + uri = URI("#{REDMINE_BASE}/projects/ruby-master/repository/git/revisions/#{rev}/issues.json") + else + raise CommandSyntaxError + end + unless @issue + puts "ticket not selected" + next + end + + Net::HTTP.start(uri.host, uri.port, http_options) do |http| + res = http.post(uri.path, "issue_id=#@issue", + 'X-Redmine-API-Key' => REDMINE_API_KEY) + begin + res.value + rescue + if $!.respond_to?(:response) && $!.response.is_a?(Net::HTTPConflict) + $stderr.puts "the revision has already related to the ticket" + else + $stderr.puts "#{$!.class}: #{$!.message}\n\ndeployed redmine doesn't have https://github.com/ruby/bugs.ruby-lang.org/commit/01fbba60d68cb916ddbccc8a8710e68c5217171d\nask naruse or hsbt" + end + next + end + puts res.body + @changesets << rev + class << @changesets + remove_method(:validated) rescue nil + end + end + }, + + "backport" => proc{|args| + # this feature implies backport command which wraps tool/merger.rb + raise CommandSyntaxError unless args.empty? + unless @issue + puts "ticket not selected" + next + end + puts backport_command_string + }, + + "done" => proc{|args| + raise CommandSyntaxError unless /\A(\d+)?(?: *by (\h+))?(?:\s*-- +(.*))?\z/ =~ args + notes = $3 + notes.strip! if notes + rev = $2 + if $1 + i = $1.to_i + i = @issues[i]["id"] if @issues && i < @issues.size + @issue = i + end + unless @issue + puts "ticket not selected" + next + end + + if rev.nil? && log = find_git_log("##@issue]") + /^commit (?<rev>\h{40})$/ =~ log + end + if log && rev + str = log[/merge revision\(s\) ([^:]+)(?=:)/] + if str + str.sub!(/\Amerge/, 'merged') + str.gsub!(/\h{40}/, 'commit:\0') + str = "ruby_#{TARGET_VERSION.tr('.','_')} commit:#{rev} #{str}." + else + str = "ruby_#{TARGET_VERSION.tr('.','_')} commit:#{rev}." + end + if notes + str << "\n" + str << notes + end + notes = str + elsif rev && has_commit(rev, "ruby_#{TARGET_VERSION.tr('.','_')}") + # Backport commit's log doesn't have the issue number. + # Instead of that manually it's provided. + notes = "ruby_#{TARGET_VERSION.tr('.','_')} commit:#{rev}." + else + puts "no commit is found whose log include ##@issue" + next + end + puts notes + + uri = URI("#{REDMINE_BASE}/issues/#{@issue}.json") + Net::HTTP.start(uri.host, uri.port, http_options) do |http| + res = http.get(uri.path) + data = JSON(res.body) + h = data["issue"]["custom_fields"].find{|x|x["id"]==5} + if h and val = h["value"] and val != "" + case val[/(?:\A|, )#{Regexp.quote TARGET_VERSION}: ([^,]+)/, 1] + when 'REQUIRED', 'UNKNOWN', 'DONTNEED', 'WONTFIX' + val[$~.offset(1)[0]...$~.offset(1)[1]] = 'DONE' + when 'DONE' # , /\A\d+\z/ + puts 'already backport is done' + next # already done + when nil + val << ", #{TARGET_VERSION}: DONE" + else + raise "unknown status '#$1'" + end + else + val = "#{TARGET_VERSION}: DONE" + end + + data = { "issue" => { "custom_fields" => [ {"id"=>5, "value" => val} ] } } + data['issue']['notes'] = notes if notes + res = http.put(uri.path, JSON(data), + 'X-Redmine-API-Key' => REDMINE_API_KEY, + 'Content-Type' => 'application/json') + res.value + + show_last_journal(http, uri) + end + }, + + "close" => proc{|args| + raise CommandSyntaxError unless /\A(\d+)?\z/ =~ args + if $1 + i = $1.to_i + i = @issues[i]["id"] if @issues && i < @issues.size + @issue = i + end + unless @issue + puts "ticket not selected" + next + end + + uri = URI("#{REDMINE_BASE}/issues/#{@issue}.json") + Net::HTTP.start(uri.host, uri.port, http_options) do |http| + data = { "issue" => { "status_id" => STATUS_CLOSE } } + res = http.put(uri.path, JSON(data), + 'X-Redmine-API-Key' => REDMINE_API_KEY, + 'Content-Type' => 'application/json') + res.value + + show_last_journal(http, uri) + end + }, + + "last" => proc{|args| + raise CommandSyntaxError unless /\A(\d+)?\z/ =~ args + if $1 + i = $1.to_i + i = @issues[i]["id"] if @issues && i < @issues.size + @issue = i + end + unless @issue + puts "ticket not selected" + next + end + + uri = URI("#{REDMINE_BASE}/issues/#{@issue}.json") + Net::HTTP.start(uri.host, uri.port, http_options) do |http| + show_last_journal(http, uri) + end + }, + + "!" => proc{|args| + system(args.strip) + }, + + "quit" => proc{|args| + raise CommandSyntaxError unless args.empty? + exit + }, + "exit" => "quit", + + "help" => proc{|args| + puts 'ls [PAGE] '.color(bold: true) + ' show all required tickets' + puts '[show] TICKET '.color(bold: true) + ' show the detail of the TICKET, and select it' + puts 'backport '.color(bold: true) + ' show the option of selected ticket for merger.rb' + puts 'rel REVISION '.color(bold: true) + ' add the selected ticket as related to the REVISION' + puts 'done [TICKET] [-- NOTE]'.color(bold: true) + ' set Backport field of the TICKET to DONE' + puts 'close [TICKET] '.color(bold: true) + ' close the TICKET' + puts 'last [TICKET] '.color(bold: true) + ' show the last journal of the TICKET' + puts '! COMMAND '.color(bold: true) + ' execute COMMAND' + } +} +list = Abbrev.abbrev(commands.keys) + +@issues = nil +@issue = nil +@changesets = nil +while true + begin + l = Reline.readline "#{('#' + @issue.to_s).color(bold: true) if @issue}> " + rescue Interrupt + break + end + break unless l + cmd, args = l.strip.split(/\s+|\b/, 2) + next unless cmd + if (!args || args.empty?) && /\A\d+\z/ =~ cmd + args = cmd + cmd = "show" + end + cmd = list[cmd] + if commands[cmd].is_a? String + cmd = list[commands[cmd]] + end + begin + if cmd + commands[cmd].call(args) + else + raise CommandSyntaxError + end + rescue CommandSyntaxError + puts "error #{l.inspect}" + end +end diff --git a/tool/release.sh b/tool/release.sh new file mode 100755 index 0000000000..0988dc2a67 --- /dev/null +++ b/tool/release.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Bash version 3.2+ is required for regexp + +EXTS='.tar.gz .tar.bz2 .tar.xz .zip' + +ver=$1 +if [[ $ver =~ ^([1-9]\.[0-9])\.([0-9]|[1-9][0-9]|0-(preview[1-9]|rc[1-9]))$ ]]; then + : +else + echo $ver is not valid release version + exit 1 +fi + +short=${BASH_REMATCH[1]} +echo $ver +echo $short +for ext in $EXTS; do + aws --profile ruby s3 cp s3://ftp.r-l.o/pub/tmp/ruby-$ver-draft$ext s3://ftp.r-l.o/pub/ruby/$short/ruby-$ver$ext +done diff --git a/tool/releng/gen-mail.rb b/tool/releng/gen-mail.rb new file mode 100755 index 0000000000..b958a64e65 --- /dev/null +++ b/tool/releng/gen-mail.rb @@ -0,0 +1,50 @@ +#!/usr/bin/env ruby +require "open-uri" +require "yaml" + +lang = ARGV.shift +unless lang + abort "usage: #$1 {en,ja} | pbcopy" +end + +# Confirm current directory is www.ruby-lang.org's working directory +def confirm_w_r_l_o_wd + File.foreach('.git/config') do |line| + return true if line.include?('git@github.com:ruby/www.ruby-lang.org.git') + end + abort "Run this script in www.ruby-lang.org's working directory" +end +confirm_w_r_l_o_wd + +releases = YAML.load_file('_data/releases.yml') + +url = "https://hackmd.io/@naruse/ruby-relnote-#{lang}/download" +src = URI(url).read +src.gsub!(/[ \t]+$/, "") +src.sub!(/(?<!\n)\z/, "\n") +src.sub!(/^breaks: false\n/, '') + +if /^\{% assign release = site.data.releases \| where: "version", "([^"]+)" \| first %\}/ =~ src + version = $1 +else + abort %[#{url} doesn't include `{% assign release = site.data.releases | where: "version", "<version>" | first %}`] +end + +release = releases.find{|rel|rel['version'] == version} +unless release + abort "#{version} is not found in '_data/releases.yml'" +end + +src.gsub!(/^{% assign .*\n/, '') +src.gsub!(/\{\{(.*?)\}\}/) do + var = $1.strip + case var + when /\Arelease\.(.*)/ + val = release.dig(*$1.split('.')) + raise "invalid variable '#{var}'" unless val + else + raise "unknown variable '#{var}'" + end + val +end +puts src diff --git a/tool/releng/gen-release-note.rb b/tool/releng/gen-release-note.rb new file mode 100755 index 0000000000..5afd11b796 --- /dev/null +++ b/tool/releng/gen-release-note.rb @@ -0,0 +1,36 @@ +#!/usr/bin/env ruby +require 'open-uri' +require 'time' +require 'yaml' + +# Confirm current directory is www.ruby-lang.org's working directory +def confirm_w_r_l_o_wd + File.foreach('.git/config') do |line| + return true if line.include?('git@github.com:ruby/www.ruby-lang.org.git') + end + abort "Run this script in www.ruby-lang.org's working directory" +end +confirm_w_r_l_o_wd + +%w[ + https://hackmd.io/@naruse/ruby-relnote-en/download + https://hackmd.io/@naruse/ruby-relnote-ja/download +].each do |url| + src = URI(url).read + src.gsub!(/[ \t]+$/, "") + src.sub!(/\s+\z/, "\n") + src.sub!(/^breaks: false\n/, '') + if /^\{% assign release = site.data.releases \| where: "version", "([^"]+)" \| first %\}/ =~ src + version = $1 + else + abort %[#{url} doesn't include `{% assign release = site.data.releases | where: "version", "<version>" | first %}`] + end + puts "#{url} -> #{version}" + + + # Write release note article + path = Time.parse(src[/^date: (.*)/, 1]). + strftime("./#{src[/^lang: (\w+)/, 1]}/news/_posts/%Y-%m-%d-ruby-#{version.tr('.', '-')}-released.md") + puts path + File.write(path, src) +end diff --git a/tool/releng/update-www-meta.rb b/tool/releng/update-www-meta.rb new file mode 100755 index 0000000000..8a5651dcd0 --- /dev/null +++ b/tool/releng/update-www-meta.rb @@ -0,0 +1,213 @@ +#!/usr/bin/env ruby +require "open-uri" +require "yaml" + +class Tarball + attr_reader :version, :size, :sha1, :sha256, :sha512 + + def initialize(version, url, size, sha1, sha256, sha512) + @url = url + @size = size + @sha1 = sha1 + @sha256 = sha256 + @sha512 = sha512 + @version = version + @xy = version[/\A\d+\.\d+/] + end + + def gz?; @url.end_with?('.gz'); end + def zip?; @url.end_with?('.zip'); end + def xz?; @url.end_with?('.xz'); end + + def ext; @url[/(?:zip|tar\.(?:gz|xz))\z/]; end + + def to_md + <<eom +* <https://cache.ruby-lang.org/pub/ruby/#{@xy}/ruby-#{@version}.#{ext}> + + SIZE: #{@size} bytes + SHA1: #{@sha1} + SHA256: #{@sha256} + SHA512: #{@sha512} +eom + end + + # * /home/naruse/obj/ruby-trunk/tmp/ruby-2.6.0-preview3.tar.gz + # SIZE: 17116009 bytes + # SHA1: 21f62c369661a2ab1b521fd2fa8191a4273e12a1 + # SHA256: 97cea8aa63dfa250ba6902b658a7aa066daf817b22f82b7ee28f44aec7c2e394 + # SHA512: 1e2042324821bb4e110af7067f52891606dcfc71e640c194ab1c117f0b941550e0b3ac36ad3511214ac80c536b9e5cfaf8789eec74cf56971a832ea8fc4e6d94 + def self.parse(wwwdir, version) + unless /\A(\d+)\.(\d+)\.(\d+)(?:-(?:preview|rc)\d+)?\z/ =~ version + raise "unexpected version string '#{version}'" + end + x = $1.to_i + y = $2.to_i + z = $3.to_i + # previous tag for git diff --shortstat + # It's only for x.y.0 release + if z != 0 + prev_tag = nil + elsif y != 0 + prev_tag = "v#{x}_#{y-1}_0" + prev_ver = "#{x}.#{y-1}.0" + elsif x == 3 && y == 0 && z == 0 + prev_tag = "v2_7_0" + prev_ver = "2.7.0" + else + raise "unexpected version for prev_ver '#{version}'" + end + + uri = "https://cache.ruby-lang.org/pub/tmp/ruby-info-#{version}-draft.yml" + info = YAML.load(URI(uri).read) + if info.size != 1 + raise "unexpected info.yml '#{uri}'" + end + tarballs = [] + info[0]["size"].each_key do |ext| + url = info[0]["url"][ext] + size = info[0]["size"][ext] + sha1 = info[0]["sha1"][ext] + sha256 = info[0]["sha256"][ext] + sha512 = info[0]["sha512"][ext] + tarball = Tarball.new(version, url, size, sha1, sha256, sha512) + tarballs << tarball + end + + if prev_tag + # show diff shortstat + tag = "v#{version.gsub(/[.\-]/, '_')}" + rubydir = File.expand_path(File.join(__FILE__, '../../../')) + puts %`git -C #{rubydir} diff --shortstat #{prev_tag}..#{tag}` + stat = `git -C #{rubydir} diff --shortstat #{prev_tag}..#{tag}` + files_changed, insertions, deletions = stat.scan(/\d+/) + end + + xy = version[/\A\d+\.\d+/] + #puts "## Download\n\n" + #tarballs.each do |tarball| + # puts tarball.to_md + #end + update_branches_yml(version, xy, wwwdir) + update_downloads_yml(version, xy, wwwdir) + update_releases_yml(version, xy, tarballs, wwwdir, files_changed, insertions, deletions) + end + + def self.update_branches_yml(ver, xy, wwwdir) + filename = "_data/branches.yml" + data = File.read(File.join(wwwdir, filename)) + if data.include?("\n- name: #{xy}\n") + data.sub!(/\n- name: #{Regexp.escape(xy)}\n(?: .*\n)*/) do |node| + unless ver.include?("-") + # assume this is X.Y.0 release + node.sub!(/^ status: preview\n/, " status: normal maintenance\n") + node.sub!(/^ date:\n/, " date: #{Time.now.year}-12-25\n") + end + node + end + else + if ver.include?("-") + status = "preview" + year = nil + else + status = "normal maintenance" + year = Time.now.year + end + entry = <<eom +- name: #{xy} + status: #{status} + date:#{ year && " #{year}-12-25" } + eol_date: + +eom + data.sub!(/(?=^- name)/, entry) + end + File.write(File.join(wwwdir, filename), data) + end + + def self.update_downloads_yml(ver, xy, wwwdir) + filename = "_data/downloads.yml" + data = File.read(File.join(wwwdir, filename)) + + if /^preview:\n\n(?: .*\n)* - #{Regexp.escape(xy)}\./ =~ data + if ver.include?("-") + data.sub!(/^ - #{Regexp.escape(xy)}\..*/, " - #{ver}") + else + data.sub!(/^ - #{Regexp.escape(xy)}\..*\n/, "") + data.sub!(/(?<=^stable:\n\n)/, " - #{ver}\n") + end + else + unless data.sub!(/^ - #{Regexp.escape(xy)}\..*/, " - #{ver}") + if ver.include?("-") + data.sub!(/(?<=^preview:\n\n)/, " - #{ver}\n") + else + data.sub!(/(?<=^stable:\n\n)/, " - #{ver}\n") + end + end + end + File.write(File.join(wwwdir, filename), data) + end + + def self.update_releases_yml(ver, xy, ary, wwwdir, files_changed, insertions, deletions) + filename = "_data/releases.yml" + data = File.read(File.join(wwwdir, filename)) + + date = Time.now.utc # use utc to use previous day in midnight + entry = <<eom +- version: #{ver} + tag: v#{ver.tr('-.', '_')} + date: #{date.strftime("%Y-%m-%d")} + post: /en/news/#{date.strftime("%Y/%m/%d")}/ruby-#{ver.tr('.', '-')}-released/ + stats: + files_changed: #{files_changed} + insertions: #{insertions} + deletions: #{deletions} + url: + gz: https://cache.ruby-lang.org/pub/ruby/#{xy}/ruby-#{ver}.tar.gz + zip: https://cache.ruby-lang.org/pub/ruby/#{xy}/ruby-#{ver}.zip + xz: https://cache.ruby-lang.org/pub/ruby/#{xy}/ruby-#{ver}.tar.xz + size: + gz: #{ary.find{|x|x.gz? }.size} + zip: #{ary.find{|x|x.zip?}.size} + xz: #{ary.find{|x|x.xz? }.size} + sha1: + gz: #{ary.find{|x|x.gz? }.sha1} + zip: #{ary.find{|x|x.zip?}.sha1} + xz: #{ary.find{|x|x.xz? }.sha1} + sha256: + gz: #{ary.find{|x|x.gz? }.sha256} + zip: #{ary.find{|x|x.zip?}.sha256} + xz: #{ary.find{|x|x.xz? }.sha256} + sha512: + gz: #{ary.find{|x|x.gz? }.sha512} + zip: #{ary.find{|x|x.zip?}.sha512} + xz: #{ary.find{|x|x.xz? }.sha512} +eom + + if data.include?("\n- version: #{ver}\n") + elsif data.sub!(/\n# #{Regexp.escape(xy)} series\n/, "\\&\n#{entry}") + else + data.sub!(/^$/, "\n# #{xy} series\n\n#{entry}") + end + File.write(File.join(wwwdir, filename), data) + end +end + +# Confirm current directory is www.ruby-lang.org's working directory +def confirm_w_r_l_o_wd + File.foreach('.git/config') do |line| + return true if line.include?('git@github.com:ruby/www.ruby-lang.org.git') + end + abort "Run this script in www.ruby-lang.org's working directory" +end + +def main + if ARGV.size != 1 + abort "usage: #$1 <version>" + end + confirm_w_r_l_o_wd + version = ARGV.shift + Tarball.parse(Dir.pwd, version) +end + +main diff --git a/tool/rmdirs b/tool/rmdirs new file mode 100755 index 0000000000..76c4a39cb1 --- /dev/null +++ b/tool/rmdirs @@ -0,0 +1,14 @@ +#!/bin/sh + +# Script used by configure to delete directories recursively. + +for dir do + while rmdir "$dir" >/dev/null 2>&1 && + parent=`expr "$dir" : '\(.*\)/[^/][^/]*'`; do + case "$parent" in + . | .. | "$dir") break;; + *) dir="$parent";; + esac + done +done +true diff --git a/tool/ruby_vm/controllers/application_controller.rb b/tool/ruby_vm/controllers/application_controller.rb new file mode 100644 index 0000000000..25c10947ed --- /dev/null +++ b/tool/ruby_vm/controllers/application_controller.rb @@ -0,0 +1,25 @@ +#! /your/favourite/path/to/ruby +# -*- Ruby -*- +# -*- frozen_string_literal: true; -*- +# -*- warn_indent: true; -*- +# +# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +# +# This file is a part of the programming language Ruby. Permission is hereby +# granted, to either redistribute and/or modify this file, provided that the +# conditions mentioned in the file COPYING are met. Consult the file for +# details. + +require_relative '../helpers/dumper' +require_relative '../models/instructions' +require_relative '../models/typemap' +require_relative '../loaders/vm_opts_h' + +class ApplicationController + def generate i, destdir + path = Pathname.new i + dst = destdir ? Pathname.new(destdir).join(i) : Pathname.new(i) + dumper = RubyVM::Dumper.new dst + return [path, dumper] + end +end diff --git a/tool/ruby_vm/helpers/c_escape.rb b/tool/ruby_vm/helpers/c_escape.rb new file mode 100644 index 0000000000..34fafd1e34 --- /dev/null +++ b/tool/ruby_vm/helpers/c_escape.rb @@ -0,0 +1,128 @@ +#! /your/favourite/path/to/ruby +# -*- Ruby -*- +# -*- frozen_string_literal: true; -*- +# -*- warn_indent: true; -*- +# +# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +# +# This file is a part of the programming language Ruby. Permission is hereby +# granted, to either redistribute and/or modify this file, provided that the +# conditions mentioned in the file COPYING are met. Consult the file for +# details. + +require 'securerandom' + +module RubyVM::CEscape + module_function + + # generate comment, with escaps. + def commentify str + return "/* #{str.b.gsub('*/', '*\\/').gsub('/*', '/\\*')} */" + end + + # Mimic gensym of CL. + def gensym prefix = 'gensym_' + return as_tr_cpp "#{prefix}#{SecureRandom.uuid}" + end + + # Mimic AS_TR_CPP() of autoconf. + def as_tr_cpp name + q = name.b + q.gsub! %r/[^a-zA-Z0-9_]/m, '_' + q.gsub! %r/_+/, '_' + return q + end + + # Section 6.10.4 of ISO/IEC 9899:1999 specifies that the file name used for + # #line directive shall be a "character string literal". So this is needed. + # + # I'm not sure how many chars are allowed here, though. The standard + # specifies 4095 chars at most, after string concatenation (section 5.2.4.1). + # But it is easy to have a path that is longer than that. + # + # Here we ignore the standard. Just create single string literal of any + # needed length. + def rstring2cstr str + # I believe this is the fastest implementation done in pure-ruby. + # Constants cached, gsub skips block evaluation, string literal optimized. + buf = str.b + buf.gsub! %r/./nm, RString2CStr + return %'"#{buf}"' + end + + RString2CStr = { + "\x00"=> "\\0", "\x01"=> "\\x1", "\x02"=> "\\x2", "\x03"=> "\\x3", + "\x04"=> "\\x4", "\x05"=> "\\x5", "\x06"=> "\\x6", "\a"=> "\\a", + "\b"=> "\\b", "\t"=> "\\t", "\n"=> "\\n", "\v"=> "\\v", + "\f"=> "\\f", "\r"=> "\\r", "\x0E"=> "\\xe", "\x0F"=> "\\xf", + "\x10"=>"\\x10", "\x11"=>"\\x11", "\x12"=>"\\x12", "\x13"=>"\\x13", + "\x14"=>"\\x14", "\x15"=>"\\x15", "\x16"=>"\\x16", "\x17"=>"\\x17", + "\x18"=>"\\x18", "\x19"=>"\\x19", "\x1A"=>"\\x1a", "\e"=>"\\x1b", + "\x1C"=>"\\x1c", "\x1D"=>"\\x1d", "\x1E"=>"\\x1e", "\x1F"=>"\\x1f", + " "=> " ", "!"=> "!", "\""=> "\\\"", "#"=> "#", + "$"=> "$", "%"=> "%", "&"=> "&", "'"=> "'", + "("=> "(", ")"=> ")", "*"=> "*", "+"=> "+", + ","=> ",", "-"=> "-", "."=> ".", "/"=> "/", + "0"=> "0", "1"=> "1", "2"=> "2", "3"=> "3", + "4"=> "4", "5"=> "5", "6"=> "6", "7"=> "7", + "8"=> "8", "9"=> "9", ":"=> ":", ";"=> ";", + "<"=> "<", "="=> "=", ">"=> ">", "?"=> "?", + "@"=> "@", "A"=> "A", "B"=> "B", "C"=> "C", + "D"=> "D", "E"=> "E", "F"=> "F", "G"=> "G", + "H"=> "H", "I"=> "I", "J"=> "J", "K"=> "K", + "L"=> "L", "M"=> "M", "N"=> "N", "O"=> "O", + "P"=> "P", "Q"=> "Q", "R"=> "R", "S"=> "S", + "T"=> "T", "U"=> "U", "V"=> "V", "W"=> "W", + "X"=> "X", "Y"=> "Y", "Z"=> "Z", "["=> "[", + "\\"=> "\\\\", "]"=> "]", "^"=> "^", "_"=> "_", + "`"=> "`", "a"=> "a", "b"=> "b", "c"=> "c", + "d"=> "d", "e"=> "e", "f"=> "f", "g"=> "g", + "h"=> "h", "i"=> "i", "j"=> "j", "k"=> "k", + "l"=> "l", "m"=> "m", "n"=> "n", "o"=> "o", + "p"=> "p", "q"=> "q", "r"=> "r", "s"=> "s", + "t"=> "t", "u"=> "u", "v"=> "v", "w"=> "w", + "x"=> "x", "y"=> "y", "z"=> "z", "{"=> "{", + "|"=> "|", "}"=> "}", "~"=> "~", "\x7F"=>"\\x7f", + "\x80"=>"\\x80", "\x81"=>"\\x81", "\x82"=>"\\x82", "\x83"=>"\\x83", + "\x84"=>"\\x84", "\x85"=>"\\x85", "\x86"=>"\\x86", "\x87"=>"\\x87", + "\x88"=>"\\x88", "\x89"=>"\\x89", "\x8A"=>"\\x8a", "\x8B"=>"\\x8b", + "\x8C"=>"\\x8c", "\x8D"=>"\\x8d", "\x8E"=>"\\x8e", "\x8F"=>"\\x8f", + "\x90"=>"\\x90", "\x91"=>"\\x91", "\x92"=>"\\x92", "\x93"=>"\\x93", + "\x94"=>"\\x94", "\x95"=>"\\x95", "\x96"=>"\\x96", "\x97"=>"\\x97", + "\x98"=>"\\x98", "\x99"=>"\\x99", "\x9A"=>"\\x9a", "\x9B"=>"\\x9b", + "\x9C"=>"\\x9c", "\x9D"=>"\\x9d", "\x9E"=>"\\x9e", "\x9F"=>"\\x9f", + "\xA0"=>"\\xa0", "\xA1"=>"\\xa1", "\xA2"=>"\\xa2", "\xA3"=>"\\xa3", + "\xA4"=>"\\xa4", "\xA5"=>"\\xa5", "\xA6"=>"\\xa6", "\xA7"=>"\\xa7", + "\xA8"=>"\\xa8", "\xA9"=>"\\xa9", "\xAA"=>"\\xaa", "\xAB"=>"\\xab", + "\xAC"=>"\\xac", "\xAD"=>"\\xad", "\xAE"=>"\\xae", "\xAF"=>"\\xaf", + "\xB0"=>"\\xb0", "\xB1"=>"\\xb1", "\xB2"=>"\\xb2", "\xB3"=>"\\xb3", + "\xB4"=>"\\xb4", "\xB5"=>"\\xb5", "\xB6"=>"\\xb6", "\xB7"=>"\\xb7", + "\xB8"=>"\\xb8", "\xB9"=>"\\xb9", "\xBA"=>"\\xba", "\xBB"=>"\\xbb", + "\xBC"=>"\\xbc", "\xBD"=>"\\xbd", "\xBE"=>"\\xbe", "\xBF"=>"\\xbf", + "\xC0"=>"\\xc0", "\xC1"=>"\\xc1", "\xC2"=>"\\xc2", "\xC3"=>"\\xc3", + "\xC4"=>"\\xc4", "\xC5"=>"\\xc5", "\xC6"=>"\\xc6", "\xC7"=>"\\xc7", + "\xC8"=>"\\xc8", "\xC9"=>"\\xc9", "\xCA"=>"\\xca", "\xCB"=>"\\xcb", + "\xCC"=>"\\xcc", "\xCD"=>"\\xcd", "\xCE"=>"\\xce", "\xCF"=>"\\xcf", + "\xD0"=>"\\xd0", "\xD1"=>"\\xd1", "\xD2"=>"\\xd2", "\xD3"=>"\\xd3", + "\xD4"=>"\\xd4", "\xD5"=>"\\xd5", "\xD6"=>"\\xd6", "\xD7"=>"\\xd7", + "\xD8"=>"\\xd8", "\xD9"=>"\\xd9", "\xDA"=>"\\xda", "\xDB"=>"\\xdb", + "\xDC"=>"\\xdc", "\xDD"=>"\\xdd", "\xDE"=>"\\xde", "\xDF"=>"\\xdf", + "\xE0"=>"\\xe0", "\xE1"=>"\\xe1", "\xE2"=>"\\xe2", "\xE3"=>"\\xe3", + "\xE4"=>"\\xe4", "\xE5"=>"\\xe5", "\xE6"=>"\\xe6", "\xE7"=>"\\xe7", + "\xE8"=>"\\xe8", "\xE9"=>"\\xe9", "\xEA"=>"\\xea", "\xEB"=>"\\xeb", + "\xEC"=>"\\xec", "\xED"=>"\\xed", "\xEE"=>"\\xee", "\xEF"=>"\\xef", + "\xF0"=>"\\xf0", "\xF1"=>"\\xf1", "\xF2"=>"\\xf2", "\xF3"=>"\\xf3", + "\xF4"=>"\\xf4", "\xF5"=>"\\xf5", "\xF6"=>"\\xf6", "\xF7"=>"\\xf7", + "\xF8"=>"\\xf8", "\xF9"=>"\\xf9", "\xFA"=>"\\xfa", "\xFB"=>"\\xfb", + "\xFC"=>"\\xfc", "\xFD"=>"\\xfd", "\xFE"=>"\\xfe", "\xFF"=>"\\xff", + }.freeze + private_constant :RString2CStr +end + +unless defined? ''.b + class String + def b + return dup.force_encoding 'binary' + end + end +end diff --git a/tool/ruby_vm/helpers/dumper.rb b/tool/ruby_vm/helpers/dumper.rb new file mode 100644 index 0000000000..98104f4b92 --- /dev/null +++ b/tool/ruby_vm/helpers/dumper.rb @@ -0,0 +1,113 @@ +#! /your/favourite/path/to/ruby +# -*- Ruby -*- +# -*- frozen_string_literal: true; -*- +# -*- warn_indent: true; -*- +# +# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +# +# This file is a part of the programming language Ruby. Permission is hereby +# granted, to either redistribute and/or modify this file, provided that the +# conditions mentioned in the file COPYING are met. Consult the file for +# details. + +require 'pathname' +require 'erb' +require_relative 'c_escape' + +class RubyVM::Dumper + include RubyVM::CEscape + private + + def new_binding + # This `eval 'binding'` does not return the current binding + # but creates one on top of it. + return eval 'binding' + end + + def new_erb spec + path = Pathname.new(__FILE__) + path = (path.relative_path_from(Pathname.pwd) rescue path).dirname + path += '../views' + path += spec + src = path.read mode: 'rt:utf-8:utf-8' + rescue Errno::ENOENT + raise "don't know how to generate #{path}" + else + if ERB.instance_method(:initialize).parameters.assoc(:key) # Ruby 2.6+ + erb = ERB.new(src, trim_mode: '%-') + else + erb = ERB.new(src, nil, '%-') + end + erb.filename = path.to_path + return erb + end + + def finderb spec + return @erb.fetch spec do |k| + erb = new_erb k + @erb[k] = erb + end + end + + def replace_pragma_line str, lineno + if /#(\s*)pragma RubyVM reset source\n/ =~ str then + return "##{$1}line #{lineno + 2} #{@file}\n" + else + return str + end + end + + def local_variable_set bnd, var, val + eval '__locals__ ||= {}', bnd + locals = eval '__locals__', bnd + locals[var] = val + eval "#{var} = __locals__[:#{var}]", bnd + test = eval "#{var}", bnd + raise unless test == val + end + + public + + def do_render source, locals + erb = finderb source + bnd = @empty.dup + locals.each_pair do |k, v| + local_variable_set bnd, k, v + end + return erb.result bnd + end + + def replace_pragma str + return str \ + . each_line \ + . with_index \ + . map {|i, j| replace_pragma_line i, j } \ + . join + end + + def initialize dst + @erb = {} + @empty = new_binding + @file = cstr dst.to_path + end + + def render partial, opts = { :locals => {} } + return do_render "_#{partial}.erb", opts[:locals] + end + + def generate template + str = do_render "#{template}.erb", {} + return replace_pragma str + end + + private + + # view helpers + + alias cstr rstring2cstr + alias comm commentify + + def render_c_expr expr + render 'c_expr', locals: { expr: expr, } + end +end diff --git a/tool/ruby_vm/helpers/scanner.rb b/tool/ruby_vm/helpers/scanner.rb new file mode 100644 index 0000000000..ef6de8120e --- /dev/null +++ b/tool/ruby_vm/helpers/scanner.rb @@ -0,0 +1,53 @@ +#! /your/favourite/path/to/ruby +# -*- Ruby -*- +# -*- frozen_string_literal: true; -*- +# -*- warn_indent: true; -*- +# +# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +# +# This file is a part of the programming language Ruby. Permission is hereby +# granted, to either redistribute and/or modify this file, provided that the +# conditions mentioned in the file COPYING are met. Consult the file for +# details. + +require 'pathname' + +# Poor man's StringScanner. +# Sadly https://bugs.ruby-lang.org/issues/8343 is not backported to 2.0. We +# have to do it by hand. +class RubyVM::Scanner + attr_reader :__FILE__ + attr_reader :__LINE__ + + def initialize path + src = Pathname.new(__FILE__) + src = (src.relative_path_from(Pathname.pwd) rescue src).dirname + src += path + @__LINE__ = 1 + @__FILE__ = src.to_path + @str = src.read mode: 'rt:utf-8:utf-8' + @pos = 0 + end + + def eos? + return @pos >= @str.length + end + + def scan re + ret = @__LINE__ + @last_match = @str.match re, @pos + return unless @last_match + @__LINE__ += @last_match.to_s.count "\n" + @pos = @last_match.end 0 + return ret + end + + def scan! re + scan re or raise sprintf "parse error at %s:%d near:\n %s...", \ + @__FILE__, @__LINE__, @str[@pos, 32] + end + + def [] key + return @last_match[key] + end +end diff --git a/tool/ruby_vm/loaders/insns_def.rb b/tool/ruby_vm/loaders/insns_def.rb new file mode 100644 index 0000000000..034905f74e --- /dev/null +++ b/tool/ruby_vm/loaders/insns_def.rb @@ -0,0 +1,100 @@ +#! /your/favourite/path/to/ruby +# -*- Ruby -*- +# -*- frozen_string_literal: true; -*- +# -*- warn_indent: true; -*- +# +# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +# +# This file is a part of the programming language Ruby. Permission is hereby +# granted, to either redistribute and/or modify this file, provided that the +# conditions mentioned in the file COPYING are met. Consult the file for +# details. + +require_relative '../helpers/scanner' +require_relative './vm_opts_h' + +json = [] +scanner = RubyVM::Scanner.new '../../../insns.def' +path = scanner.__FILE__ +grammar = %r' + (?<comment> /[*] [^*]* [*]+ (?: [^*/] [^*]* [*]+ )* / ){0} + (?<keyword> typedef | extern | static | auto | register | + struct | union | enum ){0} + (?<C> (?: \g<block> | [^{}]+ )* ){0} + (?<block> \{ \g<ws>* \g<C> \g<ws>* \} ){0} + (?<ws> \g<comment> | \s ){0} + (?<ident> [_a-zA-Z] [0-9_a-zA-Z]* ){0} + (?<type> (?: \g<keyword> \g<ws>+ )* \g<ident> ){0} + (?<arg> \g<type> \g<ws>+ \g<ident> ){0} + (?<argv> (?# empty ) | + void | + (?: \.\.\. | \g<arg>) (?: \g<ws>* , \g<ws>* \g<arg> \g<ws>* )* ){0} + (?<pragma> \g<ws>* // \s* attr \g<ws>+ + (?<pragma:type> \g<type> ) \g<ws>+ + (?<pragma:name> \g<ident> ) \g<ws>* + = \g<ws>* + (?<pragma:expr> .+?; ) \g<ws>* ){0} + (?<insn> DEFINE_INSN(_IF\((?<insn:if>\w+)\))? \g<ws>+ + (?<insn:name> \g<ident> ) \g<ws>* + [(] \g<ws>* (?<insn:opes> \g<argv> ) \g<ws>* [)] \g<ws>* + [(] \g<ws>* (?<insn:pops> \g<argv> ) \g<ws>* [)] \g<ws>* + [(] \g<ws>* (?<insn:rets> \g<argv> ) \g<ws>* [)] \g<ws>* ){0} +'x + +until scanner.eos? do + next if scanner.scan(/\G#{grammar}\g<ws>+/o) + split = lambda {|v| + case v when /\Avoid\z/ then + [] + else + v.split(/, */) + end + } + + l1 = scanner.scan!(/\G#{grammar}\g<insn>/o) + name = scanner["insn:name"] + opt = scanner["insn:if"] + ope = split.(scanner["insn:opes"]) + pop = split.(scanner["insn:pops"]) + ret = split.(scanner["insn:rets"]) + if ope.include?("...") + raise sprintf("parse error at %s:%d:%s: operands cannot be variadic", + scanner.__FILE__, scanner.__LINE__, name) + end + + attrs = [] + while l2 = scanner.scan(/\G#{grammar}\g<pragma>/o) do + attrs << { + location: [path, l2], + name: scanner["pragma:name"], + type: scanner["pragma:type"], + expr: scanner["pragma:expr"], + } + end + + l3 = scanner.scan!(/\G#{grammar}\g<block>/o) + if opt.nil? || RubyVM::VmOptsH[opt] + json << { + name: name, + location: [path, l1], + signature: { + name: name, + ope: ope, + pop: pop, + ret: ret, + }, + attributes: attrs, + expr: { + location: [path, l3], + expr: scanner["block"], + }, + } + end +end + +RubyVM::InsnsDef = json + +if __FILE__ == $0 then + require 'json' + JSON.dump RubyVM::InsnsDef, STDOUT +end diff --git a/tool/ruby_vm/loaders/opt_insn_unif_def.rb b/tool/ruby_vm/loaders/opt_insn_unif_def.rb new file mode 100644 index 0000000000..aa6fd79e79 --- /dev/null +++ b/tool/ruby_vm/loaders/opt_insn_unif_def.rb @@ -0,0 +1,34 @@ +#! /your/favourite/path/to/ruby +# -*- Ruby -*- +# -*- frozen_string_literal: true; -*- +# -*- warn_indent: true; -*- +# +# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +# +# This file is a part of the programming language Ruby. Permission is hereby +# granted, to either redistribute and/or modify this file, provided that the +# conditions mentioned in the file COPYING are met. Consult the file for +# details. + +require_relative '../helpers/scanner' + +json = [] +scanner = RubyVM::Scanner.new '../../../defs/opt_insn_unif.def' +path = scanner.__FILE__ +until scanner.eos? do + next if scanner.scan(/\G ^ (?: \#.* )? \n /x) + break if scanner.scan(/\G ^ __END__ $ /x) + + pos = scanner.scan!(/\G (?<series> (?: [\ \t]* \w+ )+ ) \n /mx) + json << { + location: [path, pos], + signature: scanner["series"].strip.split + } +end + +RubyVM::OptInsnUnifDef = json + +if __FILE__ == $0 then + require 'json' + JSON.dump RubyVM::OptInsnUnifDef, STDOUT +end diff --git a/tool/ruby_vm/loaders/opt_operand_def.rb b/tool/ruby_vm/loaders/opt_operand_def.rb new file mode 100644 index 0000000000..29aef8a325 --- /dev/null +++ b/tool/ruby_vm/loaders/opt_operand_def.rb @@ -0,0 +1,56 @@ +#! /your/favourite/path/to/ruby +# -*- Ruby -*- +# -*- frozen_string_literal: true; -*- +# -*- warn_indent: true; -*- +# +# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +# +# This file is a part of the programming language Ruby. Permission is hereby +# granted, to either redistribute and/or modify this file, provided that the +# conditions mentioned in the file COPYING are met. Consult the file for +# details. + +require_relative '../helpers/scanner' + +json = [] +scanner = RubyVM::Scanner.new '../../../defs/opt_operand.def' +path = scanner.__FILE__ +grammar = %r/ + (?<comment> \# .+? \n ){0} + (?<ws> \g<comment> | \s ){0} + (?<insn> \w+ ){0} + (?<paren> \( (?: \g<paren> | [^()]+)* \) ){0} + (?<expr> (?: \g<paren> | [^(),\ \n] )+ ){0} + (?<remain> \g<expr> ){0} + (?<arg> \g<expr> ){0} + (?<extra> , \g<ws>* \g<remain> ){0} + (?<args> \g<arg> \g<extra>* ){0} + (?<decl> \g<insn> \g<ws>+ \g<args> \n ){0} +/mx + +until scanner.eos? do + break if scanner.scan(/\G ^ __END__ $ /x) + next if scanner.scan(/\G#{grammar} \g<ws>+ /ox) + + line = scanner.scan!(/\G#{grammar} \g<decl> /mox) + insn = scanner["insn"] + args = scanner["args"] + ary = [] + until args.strip.empty? do + md = /\G#{grammar} \g<args> /mox.match(args) + ary << md["arg"] + args = md["remain"] + break unless args + end + json << { + location: [path, line], + signature: [insn, ary] + } +end + +RubyVM::OptOperandDef = json + +if __FILE__ == $0 then + require 'json' + JSON.dump RubyVM::OptOperandDef, STDOUT +end diff --git a/tool/ruby_vm/loaders/vm_opts_h.rb b/tool/ruby_vm/loaders/vm_opts_h.rb new file mode 100644 index 0000000000..3f05c270ee --- /dev/null +++ b/tool/ruby_vm/loaders/vm_opts_h.rb @@ -0,0 +1,37 @@ +#! /your/favourite/path/to/ruby +# -*- Ruby -*- +# -*- frozen_string_literal: true; -*- +# -*- warn_indent: true; -*- +# +# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +# +# This file is a part of the programming language Ruby. Permission is hereby +# granted, to either redistribute and/or modify this file, provided that the +# conditions mentioned in the file COPYING are met. Consult the file for +# details. + +require_relative '../helpers/scanner' + +json = {} +scanner = RubyVM::Scanner.new '../../../vm_opts.h' +grammar = %r/ + (?<ws> \u0020 ){0} + (?<key> \w+ ){0} + (?<value> 0|1 ){0} + (?<define> \G \#define \g<ws>+ OPT_\g<key> \g<ws>+ \g<value> \g<ws>*\n ) +/mx + +until scanner.eos? do + if scanner.scan grammar then + json[scanner['key']] = ! scanner['value'].to_i.zero? # not nonzero? + else + scanner.scan(/\G.*\n/) + end +end + +RubyVM::VmOptsH = json + +if __FILE__ == $0 then + require 'json' + JSON.dump RubyVM::VmOptsH, STDOUT +end diff --git a/tool/ruby_vm/models/attribute.rb b/tool/ruby_vm/models/attribute.rb new file mode 100644 index 0000000000..de35e7234a --- /dev/null +++ b/tool/ruby_vm/models/attribute.rb @@ -0,0 +1,59 @@ +#! /your/favourite/path/to/ruby +# -*- Ruby -*- +# -*- frozen_string_literal: true; -*- +# -*- warn_indent: true; -*- +# +# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +# +# This file is a part of the programming language Ruby. Permission is hereby +# granted, to either redistribute and/or modify this file, provided that the +# conditions mentioned in the file COPYING are met. Consult the file for +# details. + +require_relative 'c_expr' + +class RubyVM::Attribute + include RubyVM::CEscape + attr_reader :insn, :key, :type, :expr + + def initialize opts = {} + @insn = opts[:insn] + @key = opts[:name] + @expr = RubyVM::CExpr.new location: opts[:location], expr: opts[:expr] + @type = opts[:type] + @ope_decls = @insn.opes.map do |operand| + decl = operand[:decl] + if @key == 'comptime_sp_inc' && operand[:type] == 'CALL_DATA' + decl = decl.gsub('CALL_DATA', 'CALL_INFO').gsub('cd', 'ci') + end + decl + end + end + + def name + as_tr_cpp "attr #{@key} @ #{@insn.name}" + end + + def pretty_name + "attr #{type} #{key} @ #{insn.pretty_name}" + end + + def declaration + if @ope_decls.empty? + argv = "void" + else + argv = @ope_decls.join(', ') + end + sprintf '%s %s(%s)', @type, name, argv + end + + def definition + if @ope_decls.empty? + argv = "void" + else + argv = @ope_decls.map {|decl| "MAYBE_UNUSED(#{decl})" }.join(",\n ") + argv = "\n #{argv}\n" if @ope_decls.size > 1 + end + sprintf "%s\n%s(%s)", @type, name, argv + end +end diff --git a/tool/ruby_vm/models/bare_instructions.rb b/tool/ruby_vm/models/bare_instructions.rb new file mode 100755 index 0000000000..6b5f1f6cf8 --- /dev/null +++ b/tool/ruby_vm/models/bare_instructions.rb @@ -0,0 +1,240 @@ +#! /your/favourite/path/to/ruby +# -*- Ruby -*- +# -*- frozen_string_literal: true; -*- +# -*- warn_indent: true; -*- +# +# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +# +# This file is a part of the programming language Ruby. Permission is hereby +# granted, to either redistribute and/or modify this file, provided that the +# conditions mentioned in the file COPYING are met. Consult the file for +# details. + +require_relative '../loaders/insns_def' +require_relative 'c_expr' +require_relative 'typemap' +require_relative 'attribute' + +class RubyVM::BareInstructions + attr_reader :template, :name, :opes, :pops, :rets, :decls, :expr + + def initialize opts = {} + @template = opts[:template] + @name = opts[:name] + @loc = opts[:location] + @sig = opts[:signature] + @expr = RubyVM::CExpr.new opts[:expr] + @opes = typesplit @sig[:ope] + @pops = typesplit @sig[:pop].reject {|i| i == '...' } + @rets = typesplit @sig[:ret].reject {|i| i == '...' } + @attrs = opts[:attributes].map {|i| + RubyVM::Attribute.new i.merge(:insn => self) + }.each_with_object({}) {|a, h| + h[a.key] = a + } + @attrs_orig = @attrs.dup + check_attribute_consistency + predefine_attributes + end + + def pretty_name + n = @sig[:name] + o = @sig[:ope].map{|i| i[/\S+$/] }.join ', ' + p = @sig[:pop].map{|i| i[/\S+$/] }.join ', ' + r = @sig[:ret].map{|i| i[/\S+$/] }.join ', ' + return sprintf "%s(%s)(%s)(%s)", n, o, p, r + end + + def bin + return "BIN(#{name})" + end + + def call_attribute name + return sprintf 'attr_%s_%s(%s)', name, @name, \ + @opes.map {|i| i[:name] }.compact.join(', ') + end + + def has_attribute? k + @attrs_orig.has_key? k + end + + def attributes + return @attrs \ + . sort_by {|k, _| k } \ + . map {|_, v| v } + end + + def width + return 1 + opes.size + end + + def declarations + return @variables \ + . values \ + . group_by {|h| h[:type] } \ + . sort_by {|t, v| t } \ + . map {|t, v| [t, v.map {|i| i[:name] }.sort ] } \ + . map {|t, v| + sprintf("MAYBE_UNUSED(%s) %s", t, v.join(', ')) + } + end + + def preamble + # preamble makes sense for operand unifications + return [] + end + + def sc? + # sc stands for stack caching. + return false + end + + def cast_to_VALUE var, expr = var[:name] + RubyVM::Typemap.typecast_to_VALUE var[:type], expr + end + + def cast_from_VALUE var, expr = var[:name] + RubyVM::Typemap.typecast_from_VALUE var[:type], expr + end + + def operands_info + opes.map {|o| + c, _ = RubyVM::Typemap.fetch o[:type] + next c + }.join + end + + def handles_sp? + /\b(false|0)\b/ !~ @attrs.fetch('handles_sp').expr.expr + end + + def always_leaf? + @attrs.fetch('leaf').expr.expr == 'true;' + end + + def leaf_without_check_ints? + @attrs.fetch('leaf').expr.expr == 'leafness_of_check_ints;' + end + + def handle_canary stmt + # Stack canary is basically a good thing that we want to add, however: + # + # - When the instruction returns variadic number of return values, + # it is not easy to tell where is the stack top. We can't but + # skip it. + # + # - When the instruction body is empty (like putobject), we can + # say for 100% sure that canary is a waste of time. + # + # So we skip canary for those cases. + return '' if @sig[:ret].any? {|i| i == '...' } + return '' if @expr.blank? + return " #{stmt};\n" + end + + def inspect + sprintf "#<%s %s@%s:%d>", self.class.name, @name, @loc[0], @loc[1] + end + + def has_ope? var + return @opes.any? {|i| i[:name] == var[:name] } + end + + def has_pop? var + return @pops.any? {|i| i[:name] == var[:name] } + end + + def use_call_data? + @use_call_data ||= + @variables.find { |_, var_info| var_info[:type] == 'CALL_DATA' } + end + + private + + def check_attribute_consistency + if has_attribute?('sp_inc') \ + && use_call_data? \ + && !has_attribute?('comptime_sp_inc') + # As the call cache caches information that can only be obtained at + # runtime, we do not need it when compiling from AST to bytecode. This + # attribute defines an expression that computes the stack pointer + # increase based on just the call info to avoid reserving space for the + # call cache at compile time. In the expression, all call data operands + # are mapped to their call info counterpart. Additionally, all mentions + # of `cd` in the operand name are replaced with `ci`. + raise "Please define attribute `comptime_sp_inc` for `#{@name}`" + end + end + + def generate_attribute t, k, v + @attrs[k] ||= RubyVM::Attribute.new \ + insn: self, \ + name: k, \ + type: t, \ + location: [], \ + expr: v.to_s + ';' + return @attrs[k] ||= attr + end + + def predefine_attributes + # Beware: order matters here because some attribute depends another. + generate_attribute 'const char*', 'name', "insn_name(#{bin})" + generate_attribute 'enum ruby_vminsn_type', 'bin', bin + generate_attribute 'rb_num_t', 'open', opes.size + generate_attribute 'rb_num_t', 'popn', pops.size + generate_attribute 'rb_num_t', 'retn', rets.size + generate_attribute 'rb_num_t', 'width', width + generate_attribute 'rb_snum_t', 'sp_inc', rets.size - pops.size + generate_attribute 'bool', 'handles_sp', default_definition_of_handles_sp + generate_attribute 'bool', 'leaf', default_definition_of_leaf + end + + def default_definition_of_handles_sp + # Insn with ISEQ should yield it; can handle sp. + return opes.any? {|o| o[:type] == 'ISEQ' } + end + + def default_definition_of_leaf + # Insn that handles SP can never be a leaf. + if not has_attribute? 'handles_sp' then + return ! default_definition_of_handles_sp + elsif handles_sp? then + return "! #{call_attribute 'handles_sp'}" + else + return true + end + end + + def typesplit a + @variables ||= {} + a.map do |decl| + md = %r' + (?<comment> /[*] [^*]* [*]+ (?: [^*/] [^*]* [*]+ )* / ){0} + (?<ws> \g<comment> | \s ){0} + (?<ident> [_a-zA-Z] [0-9_a-zA-Z]* ){0} + (?<type> (?: \g<ident> \g<ws>+ )* \g<ident> ){0} + (?<var> \g<ident> ){0} + \G \g<ws>* \g<type> \g<ws>+ \g<var> + 'x.match(decl) + @variables[md['var']] ||= { + decl: decl, + type: md['type'], + name: md['var'], + } + end + end + + @instances = RubyVM::InsnsDef.map {|h| + new h.merge(:template => h) + } + + def self.fetch name + @instances.find do |insn| + insn.name == name + end or raise IndexError, "instruction not found: #{name}" + end + + def self.to_a + @instances + end +end diff --git a/tool/ruby_vm/models/c_expr.rb b/tool/ruby_vm/models/c_expr.rb new file mode 100644 index 0000000000..073112f545 --- /dev/null +++ b/tool/ruby_vm/models/c_expr.rb @@ -0,0 +1,41 @@ +#! /your/favourite/path/to/ruby +# -*- Ruby -*- +# -*- frozen_string_literal: true; -*- +# -*- warn_indent: true; -*- +# +# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +# +# This file is a part of the programming language Ruby. Permission is hereby +# granted, to either redistribute and/or modify this file, provided that the +# conditions mentioned in the file COPYING are met. Consult the file for +# details. + +require_relative '../helpers/c_escape.rb' + +class RubyVM::CExpr + include RubyVM::CEscape + + attr_reader :__FILE__, :__LINE__, :expr + + def initialize opts = {} + @__FILE__ = opts[:location][0] + @__LINE__ = opts[:location][1] + @expr = opts[:expr] + end + + # blank, in sense of C program. + RE = %r'\A{\g<s>*}\z|\A(?<s>\s|/[*][^*]*[*]+([^*/][^*]*[*]+)*/)*\z' + if RUBY_VERSION > '2.4' then + def blank? + RE.match? @expr + end + else + def blank? + RE =~ @expr + end + end + + def inspect + sprintf "#<%s:%d %s>", @__FILE__, @__LINE__, @expr + end +end diff --git a/tool/ruby_vm/models/instructions.rb b/tool/ruby_vm/models/instructions.rb new file mode 100644 index 0000000000..1198c7a4a6 --- /dev/null +++ b/tool/ruby_vm/models/instructions.rb @@ -0,0 +1,22 @@ +#! /your/favourite/path/to/ruby +# -*- Ruby -*- +# -*- frozen_string_literal: true; -*- +# -*- warn_indent: true; -*- +# +# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +# +# This file is a part of the programming language Ruby. Permission is hereby +# granted, to either redistribute and/or modify this file, provided that the +# conditions mentioned in the file COPYING are met. Consult the file for +# details. + +require_relative 'bare_instructions' +require_relative 'operands_unifications' +require_relative 'instructions_unifications' + +RubyVM::Instructions = RubyVM::BareInstructions.to_a + \ + RubyVM::OperandsUnifications.to_a + \ + RubyVM::InstructionsUnifications.to_a + +require_relative 'trace_instructions' +RubyVM::Instructions.freeze diff --git a/tool/ruby_vm/models/instructions_unifications.rb b/tool/ruby_vm/models/instructions_unifications.rb new file mode 100644 index 0000000000..214ba5fcc2 --- /dev/null +++ b/tool/ruby_vm/models/instructions_unifications.rb @@ -0,0 +1,43 @@ +#! /your/favourite/path/to/ruby +# -*- Ruby -*- +# -*- frozen_string_literal: true; -*- +# -*- warn_indent: true; -*- +# +# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +# +# This file is a part of the programming language Ruby. Permission is hereby +# granted, to either redistribute and/or modify this file, provided that the +# conditions mentioned in the file COPYING are met. Consult the file for +# details. + +require_relative '../helpers/c_escape' +require_relative '../loaders/opt_insn_unif_def' +require_relative 'bare_instructions' + +class RubyVM::InstructionsUnifications + include RubyVM::CEscape + + attr_reader :name + + def initialize opts = {} + @location = opts[:location] + @name = namegen opts[:signature] + @series = opts[:signature].map do |i| + RubyVM::BareInstructions.fetch i # Misshit is fatal + end + end + + private + + def namegen signature + as_tr_cpp ['UNIFIED', *signature].join('_') + end + + @instances = RubyVM::OptInsnUnifDef.map do |h| + new h + end + + def self.to_a + @instances + end +end diff --git a/tool/ruby_vm/models/operands_unifications.rb b/tool/ruby_vm/models/operands_unifications.rb new file mode 100644 index 0000000000..ee4e3a695d --- /dev/null +++ b/tool/ruby_vm/models/operands_unifications.rb @@ -0,0 +1,142 @@ +#! /your/favourite/path/to/ruby +# -*- Ruby -*- +# -*- frozen_string_literal: true; -*- +# -*- warn_indent: true; -*- +# +# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +# +# This file is a part of the programming language Ruby. Permission is hereby +# granted, to either redistribute and/or modify this file, provided that the +# conditions mentioned in the file COPYING are met. Consult the file for +# details. + +require_relative '../helpers/c_escape' +require_relative '../loaders/opt_operand_def' +require_relative 'bare_instructions' + +class RubyVM::OperandsUnifications < RubyVM::BareInstructions + include RubyVM::CEscape + + attr_reader :preamble, :original, :spec + + def initialize opts = {} + name = opts[:signature][0] + @original = RubyVM::BareInstructions.fetch name + template = @original.template + parts = compose opts[:location], opts[:signature], template[:signature] + json = template.dup + json[:location] = opts[:location] + json[:signature] = parts[:signature] + json[:name] = parts[:name] + @preamble = parts[:preamble] + @spec = parts[:spec] + super json.merge(:template => template) + @konsts = parts[:vars] + @konsts.each do |v| + @variables[v[:name]] ||= v + end + end + + def operand_shift_of var + before = @original.opes.find_index var + after = @opes.find_index var + raise "no #{var} for #{@name}" unless before and after + return before - after + end + + def condition ptr + # :FIXME: I'm not sure if this method should be in model? + exprs = @spec.each_with_index.map do |(var, val), i| + case val when '*' then + next nil + else + type = @original.opes[i][:type] + expr = RubyVM::Typemap.typecast_to_VALUE type, val + next "#{ptr}[#{i}] == #{expr}" + end + end + exprs.compact! + if exprs.size == 1 then + return exprs[0] + else + exprs.map! {|i| "(#{i})" } + return exprs.join ' && ' + end + end + + def has_ope? var + super or @konsts.any? {|i| i[:name] == var[:name] } + end + + private + + def namegen signature + insn, argv = *signature + wcary = argv.map do |i| + case i when '*' then + 'WC' + else + i + end + end + as_tr_cpp [insn, *wcary].join(', ') + end + + def compose location, spec, template + name = namegen spec + *, argv = *spec + opes = @original.opes + if opes.size != argv.size + raise sprintf("operand size mismatch for %s (%s's: %d, given: %d)", + name, template[:name], opes.size, argv.size) + else + src = [] + mod = [] + spec = [] + vars = [] + argv.each_index do |i| + j = argv[i] + k = opes[i] + spec[i] = [k, j] + case j when '*' then + # operand is from iseq + mod << k[:decl] + else + # operand is inside C + vars << k + src << { + location: location, + expr: " const #{k[:decl]} = #{j};" + } + end + end + src.map! {|i| RubyVM::CExpr.new i } + return { + name: name, + signature: { + name: name, + ope: mod, + pop: template[:pop], + ret: template[:ret], + }, + preamble: src, + vars: vars, + spec: spec + } + end + end + + @instances = RubyVM::OptOperandDef.map do |h| + new h + end + + def self.to_a + @instances + end + + def self.each_group + to_a.group_by(&:original).each_pair do |k, v| + yield k, v + end + end +end diff --git a/tool/ruby_vm/models/trace_instructions.rb b/tool/ruby_vm/models/trace_instructions.rb new file mode 100644 index 0000000000..4ed4c8cb42 --- /dev/null +++ b/tool/ruby_vm/models/trace_instructions.rb @@ -0,0 +1,71 @@ +#! /your/favourite/path/to/ruby +# -*- Ruby -*- +# -*- frozen_string_literal: true; -*- +# -*- warn_indent: true; -*- +# +# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +# +# This file is a part of the programming language Ruby. Permission is hereby +# granted, to either redistribute and/or modify this file, provided that the +# conditions mentioned in the file COPYING are met. Consult the file for +# details. + +require_relative '../helpers/c_escape' +require_relative 'bare_instructions' + +class RubyVM::TraceInstructions + include RubyVM::CEscape + + attr_reader :name + + def initialize orig + @orig = orig + @name = as_tr_cpp "trace @ #{@orig.name}" + end + + def pretty_name + return sprintf "%s(...)(...)(...)", @name + end + + def jump_destination + return @orig.name + end + + def bin + return sprintf "BIN(%s)", @name + end + + def width + return @orig.width + end + + def operands_info + return @orig.operands_info + end + + def rets + return ['...'] + end + + def pops + return ['...'] + end + + def attributes + return [] + end + + def has_attribute? *; + return false + end + + private + + @instances = RubyVM::Instructions.map {|i| new i } + + def self.to_a + @instances + end + + RubyVM::Instructions.push(*to_a) +end diff --git a/tool/ruby_vm/models/typemap.rb b/tool/ruby_vm/models/typemap.rb new file mode 100644 index 0000000000..ed3aea7d2e --- /dev/null +++ b/tool/ruby_vm/models/typemap.rb @@ -0,0 +1,62 @@ +#! /your/favourite/path/to/ruby +# -*- Ruby -*- +# -*- frozen_string_literal: true; -*- +# -*- warn_indent: true; -*- +# +# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +# +# This file is a part of the programming language Ruby. Permission is hereby +# granted, to either redistribute and/or modify this file, provided that the +# conditions mentioned in the file COPYING are met. Consult the file for +# details. + +RubyVM::Typemap = { + "..." => %w[. TS_VARIABLE], + "CALL_DATA" => %w[C TS_CALLDATA], + "CDHASH" => %w[H TS_CDHASH], + "IC" => %w[K TS_IC], + "IVC" => %w[A TS_IVC], + "ID" => %w[I TS_ID], + "ISE" => %w[T TS_ISE], + "ISEQ" => %w[S TS_ISEQ], + "OFFSET" => %w[O TS_OFFSET], + "VALUE" => %w[V TS_VALUE], + "lindex_t" => %w[L TS_LINDEX], + "rb_insn_func_t" => %w[F TS_FUNCPTR], + "rb_num_t" => %w[N TS_NUM], + "RB_BUILTIN" => %w[R TS_BUILTIN], +} + +# :FIXME: should this method be here? +class << RubyVM::Typemap + def typecast_from_VALUE type, val + # see also iseq_set_sequence() + case type + when '...' + raise "cast not possible: #{val}" + when 'VALUE' then + return val + when 'rb_num_t', 'lindex_t' then + return "NUM2LONG(#{val})" + when 'ID' then + return "SYM2ID(#{val})" + else + return "(#{type})(#{val})" + end + end + + def typecast_to_VALUE type, val + case type + when 'VALUE' then + return val + when 'ISEQ', 'rb_insn_func_t' then + return "(VALUE)(#{val})" + when 'rb_num_t', 'lindex_t' + "LONG2NUM(#{val})" + when 'ID' then + return "ID2SYM(#{val})" + else + raise ":FIXME: TBW for #{type}" + end + end +end diff --git a/tool/ruby_vm/scripts/converter.rb b/tool/ruby_vm/scripts/converter.rb new file mode 100644 index 0000000000..4e7c28d67b --- /dev/null +++ b/tool/ruby_vm/scripts/converter.rb @@ -0,0 +1,29 @@ +# This script was needed only once when I converted the old insns.def. +# Consider historical. +# +# ruby converter.rb insns.def | sponge insns.def + +BEGIN { $str = ARGF.read } +END { puts $str } + +# deal with spaces +$str.gsub! %r/\r\n|\r|\n|\z/, "\n" +$str.gsub! %r/([^\t\n]*)\t/ do + x = $1 + y = 8 - x.length % 8 + next x + ' ' * y +end +$str.gsub! %r/\s+$/, "\n" + +# deal with comments +$str.gsub! %r/@c.*?@e/m, '' +$str.gsub! %r/@j.*?\*\//m, '*/' +$str.gsub! %r/\n(\s*\n)+/, "\n\n" +$str.gsub! %r/\/\*\*?\s*\n\s*/, "/* " +$str.gsub! %r/\n\s+\*\//, " */" +$str.gsub! %r/^(?!.*\/\*.+\*\/$)(.+?)\s*\*\//, "\\1\n */" + +# deal with sp_inc +$str.gsub! %r/ \/\/ inc -= (.*)/, ' // inc += -\\1' +$str.gsub! %r/\s+\/\/ inc \+= (.*)/, "\n// attr rb_snum_t sp_inc = \\1;" +$str.gsub! %r/;;$/, ";" diff --git a/tool/ruby_vm/scripts/insns2vm.rb b/tool/ruby_vm/scripts/insns2vm.rb new file mode 100644 index 0000000000..8325dd364f --- /dev/null +++ b/tool/ruby_vm/scripts/insns2vm.rb @@ -0,0 +1,93 @@ +#! /your/favourite/path/to/ruby +# -*- Ruby -*- +# -*- frozen_string_literal: true; -*- +# -*- warn_indent: true; -*- +# +# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +# +# This file is a part of the programming language Ruby. Permission is hereby +# granted, to either redistribute and/or modify this file, provided that the +# conditions mentioned in the file COPYING are met. Consult the file for +# details. + +require 'optparse' +require_relative '../controllers/application_controller.rb' + +module RubyVM::Insns2VM + def self.router argv + options = { destdir: nil } + targets = generate_parser(options).parse argv + return targets.map do |i| + next ApplicationController.new.generate i, options[:destdir] + end + end + + def self.generate_parser(options) + OptionParser.new do |this| + this.on "-I", "--srcdir=DIR", <<-'end' + Historically this option has been passed to the script. This is + supposedly because at the beginning the script was placed + outside of the ruby source tree. Decades passed since the merge + of YARV, now I can safely assume this feature is obsolescent. + Just ignore the passed value here. + end + + this.on "-L", "--vpath=SPEC", <<-'end' + Likewise, this option is no longer supported. + end + + this.on "--path-separator=SEP", /\A(?:\W\z|\.(\W).+)/, <<-'end' + Old script says this option is a "separator for vpath". I am + confident we no longer need this option. + end + + this.on "-Dname", "--enable=name[,name...]", Array, <<-'end' + This option was used to override VM option that is defined in + vm_opts.h. Now it is officially unsupported because vm_opts.h to + remain mismatched with this option must break things. Just edit + vm_opts.h directly. + end + + this.on "-Uname", "--disable=name[,name...]", Array, <<-'end' + This option was used to override VM option that is defined in + vm_opts.h. Now it is officially unsupported because vm_opts.h to + remain mismatched with this option must break things. Just edit + vm_opts.h directly. + end + + this.on "-i", "--insnsdef=FILE", "--instructions-def", <<-'end' + This option was used to specify alternative path to insns.def. For + the same reason to ignore -I, we no longer support this. + end + + this.on "-o", "--opt-operanddef=FILE", "--opt-operand-def", <<-'end' + This option was used to specify alternative path to opt_operand.def. + For the same reason to ignore -I, we no longer support this. + end + + this.on "-u", "--opt-insnunifdef=FILE", "--opt-insn-unif-def", <<-'end' + This option was used to specify alternative path to + opt_insn_unif.def. For the same reason to ignore -I, we no + longer support this. + end + + this.on "-C", "--[no-]use-const", <<-'end' + We use const whenever possible now so this option is ignored. + The author believes that C compilers can constant-fold. + end + + this.on "-d", "--destdir", "--output-directory=DIR", <<-'begin' do |dir| + THIS IS THE ONLY OPTION THAT WORKS today. Change destination + directory from the current working directory to the given path. + begin + raise "directory was not found in '#{dir}'" unless Dir.exist?(dir) + options[:destdir] = dir + end + + this.on "-V", "--[no-]verbose", <<-'end' + Please let us ignore this and be modest. + end + end + end + private_class_method :generate_parser +end diff --git a/tool/ruby_vm/tests/.gitkeep b/tool/ruby_vm/tests/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/tool/ruby_vm/tests/.gitkeep diff --git a/tool/ruby_vm/views/_attributes.erb b/tool/ruby_vm/views/_attributes.erb new file mode 100644 index 0000000000..89a89817af --- /dev/null +++ b/tool/ruby_vm/views/_attributes.erb @@ -0,0 +1,35 @@ +%# -*- C -*- +%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +%# +%# This file is a part of the programming language Ruby. Permission is hereby +%# granted, to either redistribute and/or modify this file, provided that the +%# conditions mentioned in the file COPYING are met. Consult the file for +%# +#ifndef RUBY_VM_EXEC_H +/* can't #include "vm_exec.h" here... */ +typedef long OFFSET; +typedef unsigned long lindex_t; +typedef VALUE GENTRY; +typedef rb_iseq_t *ISEQ; +#endif + +% attrs = RubyVM::Instructions.map(&:attributes).flatten +% +% attrs.each do |a| +PUREFUNC(MAYBE_UNUSED(static <%= a.declaration %>)); +% end +% +% attrs.each do |a| + +/* <%= a.pretty_name %> */ +<%= a.definition %> +{ +% str = render_c_expr a.expr +% case str when /\A#/ then + return +<%= str -%> +% else + return <%= str -%> +% end +} +% end diff --git a/tool/ruby_vm/views/_c_expr.erb b/tool/ruby_vm/views/_c_expr.erb new file mode 100644 index 0000000000..4e1b0ec883 --- /dev/null +++ b/tool/ruby_vm/views/_c_expr.erb @@ -0,0 +1,17 @@ +%# -*- C -*- +%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +%# +%# This file is a part of the programming language Ruby. Permission is hereby +%# granted, to either redistribute and/or modify this file, provided that the +%# conditions mentioned in the file COPYING are met. Consult the file for +%# details. +%; +% if expr.blank? +% # empty +% elsif ! expr.__LINE__ +<%= expr.expr %> +% else +#line <%= expr.__LINE__ %> <%=cstr expr.__FILE__ %> +<%= expr.expr %> +#pragma RubyVM reset source +% end diff --git a/tool/ruby_vm/views/_comptime_insn_stack_increase.erb b/tool/ruby_vm/views/_comptime_insn_stack_increase.erb new file mode 100644 index 0000000000..b633ab4d32 --- /dev/null +++ b/tool/ruby_vm/views/_comptime_insn_stack_increase.erb @@ -0,0 +1,62 @@ +%# -*- C -*- +%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +%# +%# This file is a part of the programming language Ruby. Permission is hereby +%# granted, to either redistribute and/or modify this file, provided that the +%# conditions mentioned in the file COPYING are met. Consult the file for +%# details. +%# +PUREFUNC(MAYBE_UNUSED(static int comptime_insn_stack_increase(int depth, int insn, const VALUE *opes))); +PUREFUNC(static rb_snum_t comptime_insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *opes)); + +rb_snum_t +comptime_insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *opes) +{ + static const signed char t[] = { +% RubyVM::Instructions.each_slice 8 do |a| + <%= a.map { |i| + if i.has_attribute?('sp_inc') + '-127' + else + sprintf("%4d", i.rets.size - i.pops.size) + end + }.join(', ') -%>, +% end + }; + signed char c = t[insn]; + + ASSERT_VM_INSTRUCTION_SIZE(t); + if (c != -127) { + return c; + } + else switch(insn) { + default: + UNREACHABLE; +% RubyVM::Instructions.each do |i| +% next unless i.has_attribute?('sp_inc') +% attr_function = +% if i.has_attribute?('comptime_sp_inc') +% "attr_comptime_sp_inc_#{i.name}" +% else +% "attr_sp_inc_#{i.name}" +% end + case <%= i.bin %>: + return <%= attr_function %>(<%= + i.opes.map.with_index do |v, j| + if v[:type] == 'CALL_DATA' && i.has_attribute?('comptime_sp_inc') + v = v.dup + v[:type] = 'CALL_INFO' + end + i.cast_from_VALUE v, "opes[#{j}]" + end.join(", ") + %>); +% end + } +} + +int +comptime_insn_stack_increase(int depth, int insn, const VALUE *opes) +{ + enum ruby_vminsn_type itype = (enum ruby_vminsn_type)insn; + return depth + (int)comptime_insn_stack_increase_dispatch(itype, opes); +} diff --git a/tool/ruby_vm/views/_copyright.erb b/tool/ruby_vm/views/_copyright.erb new file mode 100644 index 0000000000..2154146f0d --- /dev/null +++ b/tool/ruby_vm/views/_copyright.erb @@ -0,0 +1,31 @@ +%# -*- C -*- +%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +%; +%# This file is a part of the programming language Ruby. Permission is hereby +%# granted, to either redistribute and/or modify this file, provided that the +%# conditions mentioned in the file COPYING are met. Consult the file for +%# details. +%; +%; +%# Below is the licensing term for the generated output, not this erb file. +/* This is an auto-generated file and is a part of the programming language + * Ruby. The person who created a program to generate this file (``I'' + * hereafter) would like to refrain from defining licensing of this generated + * source code. + * + * This file consists of many small parts of codes copyrighted by each author, + * not only the ``I'' person. Those original authors agree with some + * open-source license. I believe that the license we agree is the condition + * mentioned in the file COPYING. It states "4. You may modify and include + * the part of the software into any other software ...". But the problem is, + * the license never makes it clear if such modified parts still remain in the + * same license, or not. The fact that we agree with the source code's + * licensing terms does not automatically define that of generated ones. This + * is the reason why this file is under an unclear situation. All what I know + * is that above provision guarantees this file to exist. + * + * Please let me hesitate to declare something about this nuanced contract. I + * am not in the position to take over other authors' license to merge into my + * one. Changing them to (say) GPLv3 is not doable by myself. Perhaps someday + * it might turn out to be okay to say this file is under a license. I wish + * the situation would become more clear in the future. */ diff --git a/tool/ruby_vm/views/_insn_entry.erb b/tool/ruby_vm/views/_insn_entry.erb new file mode 100644 index 0000000000..f34afddb1f --- /dev/null +++ b/tool/ruby_vm/views/_insn_entry.erb @@ -0,0 +1,76 @@ +%# -*- C -*- +%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +%# +%# This file is a part of the programming language Ruby. Permission is hereby +%# granted, to either redistribute and/or modify this file, provided that the +%# conditions mentioned in the file COPYING are met. Consult the file for +%# details. +%; +% body = render_c_expr(insn.expr).gsub(/^#/, '# ') + +/* insn <%= insn.pretty_name %> */ +INSN_ENTRY(<%= insn.name %>) +{ + /* ### Declare that we have just entered into an instruction. ### */ + START_OF_ORIGINAL_INSN(<%= insn.name %>); + DEBUG_ENTER_INSN(<%=cstr insn.name %>); + + /* ### Declare and assign variables. ### */ +% insn.preamble.each do |konst| +<%= render_c_expr konst -%> +% end +% +% insn.opes.each_with_index do |ope, i| + <%= ope[:decl] %> = (<%= ope[:type] %>)GET_OPERAND(<%= i + 1 %>); +% end +# define INSN_ATTR(x) <%= insn.call_attribute(' ## x ## ') %> + const bool leaf = INSN_ATTR(leaf); +% insn.pops.reverse_each.with_index.reverse_each do |pop, i| + <%= pop[:decl] %> = <%= insn.cast_from_VALUE pop, "TOPN(#{i})"%>; +% end +% +% insn.rets.each do |ret| +% next if insn.has_ope?(ret) or insn.has_pop?(ret) + <%= ret[:decl] %>; +% end + + /* ### Instruction preambles. ### */ + if (! leaf) ADD_PC(INSN_ATTR(width)); +% if insn.handles_sp? + POPN(INSN_ATTR(popn)); +% end +<%= insn.handle_canary "SETUP_CANARY(leaf)" -%> + COLLECT_USAGE_INSN(INSN_ATTR(bin)); +% insn.opes.each_with_index do |ope, i| + COLLECT_USAGE_OPERAND(INSN_ATTR(bin), <%= i %>, <%= ope[:name] %>); +% end +% unless body.empty? + + /* ### Here we do the instruction body. ### */ +%# NAME_OF_CURRENT_INSN is used in vm_exec.h +# define NAME_OF_CURRENT_INSN <%= insn.name %> +<%= body -%> +# undef NAME_OF_CURRENT_INSN +% end + + /* ### Instruction trailers. ### */ + CHECK_VM_STACK_OVERFLOW_FOR_INSN(VM_REG_CFP, INSN_ATTR(retn)); +<%= insn.handle_canary "CHECK_CANARY(leaf, INSN_ATTR(bin))" -%> +% if insn.handles_sp? +% insn.rets.reverse_each do |ret| + PUSH(<%= insn.cast_to_VALUE ret %>); +% end +% else + INC_SP(INSN_ATTR(sp_inc)); +% insn.rets.reverse_each.with_index do |ret, i| + TOPN(<%= i %>) = <%= insn.cast_to_VALUE ret %>; + VM_ASSERT(!RB_TYPE_P(TOPN(<%= i %>), T_NONE)); + VM_ASSERT(!RB_TYPE_P(TOPN(<%= i %>), T_MOVED)); +% end +% end + if (leaf) ADD_PC(INSN_ATTR(width)); +# undef INSN_ATTR + + /* ### Leave the instruction. ### */ + END_INSN(<%= insn.name %>); +} diff --git a/tool/ruby_vm/views/_insn_len_info.erb b/tool/ruby_vm/views/_insn_len_info.erb new file mode 100644 index 0000000000..569dca5845 --- /dev/null +++ b/tool/ruby_vm/views/_insn_len_info.erb @@ -0,0 +1,28 @@ +%# -*- C -*- +%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +%# +%# This file is a part of the programming language Ruby. Permission is hereby +%# granted, to either redistribute and/or modify this file, provided that the +%# conditions mentioned in the file COPYING are met. Consult the file for +%# details. +CONSTFUNC(MAYBE_UNUSED(static int insn_len(VALUE insn))); + +RUBY_SYMBOL_EXPORT_BEGIN /* for debuggers */ +extern const uint8_t rb_vm_insn_len_info[VM_INSTRUCTION_SIZE]; +RUBY_SYMBOL_EXPORT_END + +#ifdef RUBY_VM_INSNS_INFO +const uint8_t rb_vm_insn_len_info[] = { +% RubyVM::Instructions.each_slice 23 do |a| + <%= a.map(&:width).join(', ') -%>, +% end +}; + +ASSERT_VM_INSTRUCTION_SIZE(rb_vm_insn_len_info); +#endif + +int +insn_len(VALUE i) +{ + return rb_vm_insn_len_info[i]; +} diff --git a/tool/ruby_vm/views/_insn_name_info.erb b/tool/ruby_vm/views/_insn_name_info.erb new file mode 100644 index 0000000000..e7ded75e65 --- /dev/null +++ b/tool/ruby_vm/views/_insn_name_info.erb @@ -0,0 +1,44 @@ +%# -*- C -*- +%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +%# +%# This file is a part of the programming language Ruby. Permission is hereby +%# granted, to either redistribute and/or modify this file, provided that the +%# conditions mentioned in the file COPYING are met. Consult the file for +%# details. +% +% a = RubyVM::Instructions.map {|i| i.name } +% b = (0...a.size) +% c = a.inject([0]) {|r, i| r << (r[-1] + i.length + 1) } +% c.pop +% +CONSTFUNC(MAYBE_UNUSED(static const char *insn_name(VALUE insn))); + +RUBY_SYMBOL_EXPORT_BEGIN /* for debuggers */ +extern const int rb_vm_max_insn_name_size; +extern const char rb_vm_insn_name_base[]; +extern const unsigned short rb_vm_insn_name_offset[VM_INSTRUCTION_SIZE]; +RUBY_SYMBOL_EXPORT_END + +#ifdef RUBY_VM_INSNS_INFO +const int rb_vm_max_insn_name_size = <%= a.map(&:size).max %>; + +const char rb_vm_insn_name_base[] = +% a.each do |i| + <%=cstr i%> "\0" +% end + ; + +const unsigned short rb_vm_insn_name_offset[] = { +% c.each_slice 12 do |d| + <%= d.map {|i| sprintf("%4d", i) }.join(', ') %>, +% end +}; + +ASSERT_VM_INSTRUCTION_SIZE(rb_vm_insn_name_offset); +#endif + +const char * +insn_name(VALUE i) +{ + return &rb_vm_insn_name_base[rb_vm_insn_name_offset[i]]; +} diff --git a/tool/ruby_vm/views/_insn_operand_info.erb b/tool/ruby_vm/views/_insn_operand_info.erb new file mode 100644 index 0000000000..996c33e960 --- /dev/null +++ b/tool/ruby_vm/views/_insn_operand_info.erb @@ -0,0 +1,53 @@ +%# -*- C -*- +%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +%# +%# This file is a part of the programming language Ruby. Permission is hereby +%# granted, to either redistribute and/or modify this file, provided that the +%# conditions mentioned in the file COPYING are met. Consult the file for +%# details. +% +% a = RubyVM::Instructions.map {|i| i.operands_info } +% b = (0...a.size) +% c = a.inject([0]) {|r, i| r << (r[-1] + i.length + 1) } +% c.pop +% +CONSTFUNC(MAYBE_UNUSED(static const char *insn_op_types(VALUE insn))); +CONSTFUNC(MAYBE_UNUSED(static int insn_op_type(VALUE insn, long pos))); + +RUBY_SYMBOL_EXPORT_BEGIN /* for debuggers */ +extern const char rb_vm_insn_op_base[]; +extern const unsigned short rb_vm_insn_op_offset[VM_INSTRUCTION_SIZE]; +RUBY_SYMBOL_EXPORT_END + +#ifdef RUBY_VM_INSNS_INFO +const char rb_vm_insn_op_base[] = +% a.each_slice 5 do |d| + <%= d.map {|i| sprintf("%-6s", cstr(i)) }.join(' "\0" ') %> "\0" +% end + ; + +const unsigned short rb_vm_insn_op_offset[] = { +% c.each_slice 12 do |d| + <%= d.map {|i| sprintf("%3d", i) }.join(', ') %>, +% end +}; + +ASSERT_VM_INSTRUCTION_SIZE(rb_vm_insn_op_offset); +#endif + +const char * +insn_op_types(VALUE i) +{ + return &rb_vm_insn_op_base[rb_vm_insn_op_offset[i]]; +} + +int +insn_op_type(VALUE i, long j) +{ + if (j >= insn_len(i)) { + return 0; + } + else { + return insn_op_types(i)[j]; + } +} diff --git a/tool/ruby_vm/views/_insn_sp_pc_dependency.erb b/tool/ruby_vm/views/_insn_sp_pc_dependency.erb new file mode 100644 index 0000000000..95528fbbf4 --- /dev/null +++ b/tool/ruby_vm/views/_insn_sp_pc_dependency.erb @@ -0,0 +1,27 @@ +%# -*- C -*- +%# Copyright (c) 2019 Takashi Kokubun. All rights reserved. +%# +%# This file is a part of the programming language Ruby. Permission is hereby +%# granted, to either redistribute and/or modify this file, provided that the +%# conditions mentioned in the file COPYING are met. Consult the file for +%# details. +%# +PUREFUNC(MAYBE_UNUSED(static bool insn_may_depend_on_sp_or_pc(int insn, const VALUE *opes))); + +static bool +insn_may_depend_on_sp_or_pc(int insn, const VALUE *opes) +{ + switch (insn) { +% RubyVM::Instructions.each do |insn| +% # handles_sp?: If true, it requires to move sp in JIT +% # always_leaf?: If false, it may call an arbitrary method. pc should be moved +% # before the call, and the method may refer to caller's pc (lineno). +% unless !insn.is_a?(RubyVM::TraceInstructions) && !insn.handles_sp? && insn.always_leaf? + case <%= insn.bin %>: +% end +% end + return true; + default: + return false; + } +} diff --git a/tool/ruby_vm/views/_insn_type_chars.erb b/tool/ruby_vm/views/_insn_type_chars.erb new file mode 100644 index 0000000000..4e1f63e660 --- /dev/null +++ b/tool/ruby_vm/views/_insn_type_chars.erb @@ -0,0 +1,13 @@ +%# -*- C -*- +%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +%# +%# This file is a part of the programming language Ruby. Permission is hereby +%# granted, to either redistribute and/or modify this file, provided that the +%# conditions mentioned in the file COPYING are met. Consult the file for +%# details. +% +enum ruby_insn_type_chars { +% RubyVM::Typemap.each_value do |(c, t)| + <%= t %> = '<%= c %>', +% end +}; diff --git a/tool/ruby_vm/views/_leaf_helpers.erb b/tool/ruby_vm/views/_leaf_helpers.erb new file mode 100644 index 0000000000..1735db2196 --- /dev/null +++ b/tool/ruby_vm/views/_leaf_helpers.erb @@ -0,0 +1,54 @@ +%# -*- C -*- +%# Copyright (c) 2018 Urabe, Shyouhei. All rights reserved. +%# +%# This file is a part of the programming language Ruby. Permission is hereby +%# granted, to either redistribute and/or modify this file, provided that the +%# conditions mentioned in the file COPYING are met. Consult the file for +%# details. +%; +#line <%= __LINE__ + 1 %> <%=cstr __FILE__ %> + +#include "iseq.h" + +// This is used to tell MJIT that this insn would be leaf if CHECK_INTS didn't exist. +// It should be used only when RUBY_VM_CHECK_INTS is directly written in insns.def. +static bool leafness_of_check_ints = false; + +static bool +leafness_of_defined(rb_num_t op_type) +{ + /* see also: vm_insnhelper.c:vm_defined() */ + switch (op_type) { + case DEFINED_IVAR: + case DEFINED_GVAR: + case DEFINED_CVAR: + case DEFINED_YIELD: + case DEFINED_REF: + case DEFINED_ZSUPER: + return false; + case DEFINED_CONST: + case DEFINED_CONST_FROM: + /* has rb_autoload_load(); */ + return false; + case DEFINED_FUNC: + case DEFINED_METHOD: + /* calls #respond_to_missing? */ + return false; + default: + rb_bug("unknown operand %ld: blame @shyouhei.", op_type); + } +} + +static bool +leafness_of_checkmatch(rb_num_t flag) +{ + /* see also: vm_insnhelper.c:check_match() */ + if (flag == VM_CHECKMATCH_TYPE_WHEN) { + return true; + } + else { + /* has rb_funcallv() */ + return false; + } +} +#pragma RubyVM reset source diff --git a/tool/ruby_vm/views/_mjit_compile_getinlinecache.erb b/tool/ruby_vm/views/_mjit_compile_getinlinecache.erb new file mode 100644 index 0000000000..d4eb4977a4 --- /dev/null +++ b/tool/ruby_vm/views/_mjit_compile_getinlinecache.erb @@ -0,0 +1,31 @@ +% # -*- C -*- +% # Copyright (c) 2020 Takashi Kokubun. All rights reserved. +% # +% # This file is a part of the programming language Ruby. Permission is hereby +% # granted, to either redistribute and/or modify this file, provided that the +% # conditions mentioned in the file COPYING are met. Consult the file for +% # details. +% +% # compiler: Declare dst and ic +% insn.opes.each_with_index do |ope, i| + <%= ope.fetch(:decl) %> = (<%= ope.fetch(:type) %>)operands[<%= i %>]; +% end + +% # compiler: Capture IC values, locking getinlinecache + struct iseq_inline_constant_cache_entry *ice = ic->entry; + if (ice != NULL && GET_IC_SERIAL(ice) && !status->compile_info->disable_const_cache) { +% # JIT: Inline everything in IC, and cancel the slow path + fprintf(f, " if (vm_inlined_ic_hit_p(0x%"PRIxVALUE", 0x%"PRIxVALUE", (const rb_cref_t *)0x%"PRIxVALUE", %"PRI_SERIALT_PREFIX"u, reg_cfp->ep)) {", ice->flags, ice->value, (VALUE)ice->ic_cref, GET_IC_SERIAL(ice)); + fprintf(f, " stack[%d] = 0x%"PRIxVALUE";\n", b->stack_size, ice->value); + fprintf(f, " goto label_%d;\n", pos + insn_len(insn) + (int)dst); + fprintf(f, " }"); + fprintf(f, " else {"); + fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); + fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); + fprintf(f, " goto const_cancel;\n"); + fprintf(f, " }"); + +% # compiler: Move JIT compiler's internal stack pointer + b->stack_size += <%= insn.call_attribute('sp_inc') %>; + break; + } diff --git a/tool/ruby_vm/views/_mjit_compile_insn.erb b/tool/ruby_vm/views/_mjit_compile_insn.erb new file mode 100644 index 0000000000..f54d1b0e0e --- /dev/null +++ b/tool/ruby_vm/views/_mjit_compile_insn.erb @@ -0,0 +1,92 @@ +% # -*- C -*- +% # Copyright (c) 2018 Takashi Kokubun. All rights reserved. +% # +% # This file is a part of the programming language Ruby. Permission is hereby +% # granted, to either redistribute and/or modify this file, provided that the +% # conditions mentioned in the file COPYING are met. Consult the file for +% # details. + fprintf(f, "{\n"); + { +% # compiler: Prepare operands which may be used by `insn.call_attribute` +% insn.opes.each_with_index do |ope, i| + MAYBE_UNUSED(<%= ope.fetch(:decl) %>) = (<%= ope.fetch(:type) %>)operands[<%= i %>]; +% end +% +% # JIT: Declare stack_size to be used in some macro of _mjit_compile_insn_body.erb + if (status->local_stack_p) { + fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); + } +% +% # JIT: Declare variables for operands, popped values and return values +% insn.declarations.each do |decl| + fprintf(f, " <%= decl %>;\n"); +% end + +% # JIT: Set const expressions for `RubyVM::OperandsUnifications` insn +% insn.preamble.each do |amble| + fprintf(f, "<%= amble.expr.sub(/const \S+\s+/, '') %>\n"); +% end +% +% # JIT: Initialize operands +% insn.opes.each_with_index do |ope, i| + fprintf(f, " <%= ope.fetch(:name) %> = (<%= ope.fetch(:type) %>)0x%"PRIxVALUE";", operands[<%= i %>]); +% case ope.fetch(:type) +% when 'ID' + comment_id(f, (ID)operands[<%= i %>]); +% when 'CALL_DATA' + comment_id(f, vm_ci_mid(((CALL_DATA)operands[<%= i %>])->ci)); +% when 'VALUE' + if (SYMBOL_P((VALUE)operands[<%= i %>])) comment_id(f, SYM2ID((VALUE)operands[<%= i %>])); +% end + fprintf(f, "\n"); +% end +% +% # JIT: Initialize popped values +% insn.pops.reverse_each.with_index.reverse_each do |pop, i| + fprintf(f, " <%= pop.fetch(:name) %> = stack[%d];\n", b->stack_size - <%= i + 1 %>); +% end +% +% # JIT: move sp and pc if necessary +<%= render 'mjit_compile_pc_and_sp', locals: { insn: insn } -%> +% +% # JIT: Print insn body in insns.def +<%= render 'mjit_compile_insn_body', locals: { insn: insn } -%> +% +% # JIT: Set return values +% insn.rets.reverse_each.with_index do |ret, i| +% # TOPN(n) = ... + fprintf(f, " stack[%d] = <%= ret.fetch(:name) %>;\n", b->stack_size + (int)<%= insn.call_attribute('sp_inc') %> - <%= i + 1 %>); +% end +% +% # JIT: We should evaluate ISeq modified for TracePoint if it's enabled. Note: This is slow. +% # leaf insn may not cancel JIT. leaf_without_check_ints is covered in RUBY_VM_CHECK_INTS of _mjit_compile_insn_body.erb. +% unless insn.always_leaf? || insn.leaf_without_check_ints? + fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); + fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)<%= insn.call_attribute('sp_inc') %>); + if (!pc_moved_p) { + fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); + } + fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); + fprintf(f, " goto cancel;\n"); + fprintf(f, " }\n"); +% end +% +% # compiler: Move JIT compiler's internal stack pointer + b->stack_size += <%= insn.call_attribute('sp_inc') %>; + } + fprintf(f, "}\n"); +% +% # compiler: If insn has conditional JUMP, the code should go to the branch not targeted by JUMP next. +% if insn.expr.expr =~ /if\s+\([^{}]+\)\s+\{[^{}]+JUMP\([^)]+\);[^{}]+\}/ + if (ALREADY_COMPILED_P(status, pos + insn_len(insn))) { + fprintf(f, "goto label_%d;\n", pos + insn_len(insn)); + } + else { + compile_insns(f, body, b->stack_size, pos + insn_len(insn), status); + } +% end +% +% # compiler: If insn returns (leave) or does longjmp (throw), the branch should no longer be compiled. TODO: create attr for it? +% if insn.expr.expr =~ /\sTHROW_EXCEPTION\([^)]+\);/ || insn.expr.expr =~ /\bvm_pop_frame\(/ + b->finish_p = TRUE; +% end diff --git a/tool/ruby_vm/views/_mjit_compile_insn_body.erb b/tool/ruby_vm/views/_mjit_compile_insn_body.erb new file mode 100644 index 0000000000..187e043837 --- /dev/null +++ b/tool/ruby_vm/views/_mjit_compile_insn_body.erb @@ -0,0 +1,129 @@ +% # -*- C -*- +% # Copyright (c) 2018 Takashi Kokubun. All rights reserved. +% # +% # This file is a part of the programming language Ruby. Permission is hereby +% # granted, to either redistribute and/or modify this file, provided that the +% # conditions mentioned in the file COPYING are met. Consult the file for +% # details. +% +% to_cstr = lambda do |line| +% normalized = line.gsub(/\t/, ' ' * 8) +% indented = normalized.sub(/\A(?!#)/, ' ') # avoid indenting preprocessor +% rstring2cstr(indented.rstrip).sub(/"\z/, '\\n"') +% end +% +% # +% # Expand simple macro, which doesn't require dynamic C code. +% # +% expand_simple_macros = lambda do |arg_expr| +% arg_expr.dup.tap do |expr| +% # For `leave`. We can't proceed next ISeq in the same JIT function. +% expr.gsub!(/^(?<indent>\s*)RESTORE_REGS\(\);\n/) do +% indent = Regexp.last_match[:indent] +% <<-end.gsub(/^ +/, '') +% #if OPT_CALL_THREADED_CODE +% #{indent}rb_ec_thread_ptr(ec)->retval = val; +% #{indent}return 0; +% #else +% #{indent}return val; +% #endif +% end +% end +% expr.gsub!(/^(?<indent>\s*)NEXT_INSN\(\);\n/) do +% indent = Regexp.last_match[:indent] +% <<-end.gsub(/^ +/, '') +% #{indent}UNREACHABLE_RETURN(Qundef); +% end +% end +% end +% end +% +% # +% # Print a body of insn, but with macro expansion. +% # +% expand_simple_macros.call(insn.expr.expr).each_line do |line| +% # +% # Expand dynamic macro here (only JUMP for now) +% # +% # TODO: support combination of following macros in the same line +% case line +% when /\A\s+RUBY_VM_CHECK_INTS\(ec\);\s+\z/ +% if insn.leaf_without_check_ints? # lazily move PC and optionalize mjit_call_p here + fprintf(f, " if (UNLIKELY(RUBY_VM_INTERRUPTED_ANY(ec))) {\n"); + fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ + fprintf(f, " rb_threadptr_execute_interrupts(rb_ec_thread_ptr(ec), 0);\n"); + fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); + fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); + fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); + fprintf(f, " goto cancel;\n"); + fprintf(f, " }\n"); + fprintf(f, " }\n"); +% else + fprintf(f, <%= to_cstr.call(line) %>); +% end +% when /\A\s+JUMP\((?<dest>[^)]+)\);\s+\z/ +% dest = Regexp.last_match[:dest] +% +% if insn.name == 'opt_case_dispatch' # special case... TODO: use another macro to avoid checking name + { + struct case_dispatch_var arg; + arg.f = f; + arg.base_pos = pos + insn_len(insn); + arg.last_value = Qundef; + + fprintf(f, " switch (<%= dest %>) {\n"); + st_foreach(RHASH_TBL_RAW(hash), compile_case_dispatch_each, (VALUE)&arg); + fprintf(f, " case %lu:\n", else_offset); + fprintf(f, " goto label_%lu;\n", arg.base_pos + else_offset); + fprintf(f, " }\n"); + } +% else +% # Before we `goto` next insn, we need to set return values, especially for getinlinecache +% insn.rets.reverse_each.with_index do |ret, i| +% # TOPN(n) = ... + fprintf(f, " stack[%d] = <%= ret.fetch(:name) %>;\n", b->stack_size + (int)<%= insn.call_attribute('sp_inc') %> - <%= i + 1 %>); +% end +% + next_pos = pos + insn_len(insn) + (unsigned int)<%= dest %>; + fprintf(f, " goto label_%d;\n", next_pos); +% end +% when /\A\s+CALL_SIMPLE_METHOD\(\);\s+\z/ +% # For `opt_xxx`'s fallbacks. + if (status->local_stack_p) { + fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); + } + fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); + fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); + fprintf(f, " goto cancel;\n"); +% when /\A(?<prefix>.+\b)INSN_LABEL\((?<name>[^)]+)\)(?<suffix>.+)\z/m +% prefix, name, suffix = Regexp.last_match[:prefix], Regexp.last_match[:name], Regexp.last_match[:suffix] + fprintf(f, " <%= prefix.gsub(/\t/, ' ' * 8) %>INSN_LABEL(<%= name %>_%d)<%= suffix.sub(/\n/, '\n') %>", pos); +% else +% if insn.handles_sp? +% # If insn.handles_sp? is true, cfp->sp might be changed inside insns (like vm_caller_setup_arg_block) +% # and thus we need to use cfp->sp, even when local_stack_p is TRUE. When insn.handles_sp? is true, +% # cfp->sp should be available too because _mjit_compile_pc_and_sp.erb sets it. + fprintf(f, <%= to_cstr.call(line) %>); +% else +% # If local_stack_p is TRUE and insn.handles_sp? is false, stack values are only available in local variables +% # for stack. So we need to replace those macros if local_stack_p is TRUE here. +% case line +% when /\bGET_SP\(\)/ +% # reg_cfp->sp + fprintf(f, <%= to_cstr.call(line.sub(/\bGET_SP\(\)/, '%s')) %>, (status->local_stack_p ? "(stack + stack_size)" : "GET_SP()")); +% when /\bSTACK_ADDR_FROM_TOP\((?<num>[^)]+)\)/ +% # #define STACK_ADDR_FROM_TOP(n) (GET_SP()-(n)) +% num = Regexp.last_match[:num] + fprintf(f, <%= to_cstr.call(line.sub(/\bSTACK_ADDR_FROM_TOP\(([^)]+)\)/, '%s')) %>, + (status->local_stack_p ? "(stack + (stack_size - (<%= num %>)))" : "STACK_ADDR_FROM_TOP(<%= num %>)")); +% when /\bTOPN\((?<num>[^)]+)\)/ +% # #define TOPN(n) (*(GET_SP()-(n)-1)) +% num = Regexp.last_match[:num] + fprintf(f, <%= to_cstr.call(line.sub(/\bTOPN\(([^)]+)\)/, '%s')) %>, + (status->local_stack_p ? "*(stack + (stack_size - (<%= num %>) - 1))" : "TOPN(<%= num %>)")); +% else + fprintf(f, <%= to_cstr.call(line) %>); +% end +% end +% end +% end diff --git a/tool/ruby_vm/views/_mjit_compile_invokebuiltin.erb b/tool/ruby_vm/views/_mjit_compile_invokebuiltin.erb new file mode 100644 index 0000000000..a3796ffc5e --- /dev/null +++ b/tool/ruby_vm/views/_mjit_compile_invokebuiltin.erb @@ -0,0 +1,29 @@ +% # -*- C -*- +% # Copyright (c) 2020 Urabe, Shyouhei. All rights reserved. +% # +% # This file is a part of the programming language Ruby. Permission is hereby +% # granted, to either redistribute and/or modify this file, provided that the +% # conditions mentioned in the file COPYING are met. Consult the file for +% # details. +% +% insn.opes.each_with_index do |ope, i| + <%= ope.fetch(:decl) %> = (<%= ope.fetch(:type) %>)operands[<%= i %>]; +% end + rb_snum_t sp_inc = <%= insn.call_attribute('sp_inc') %>; + unsigned sp = b->stack_size + (unsigned)sp_inc; + VM_ASSERT(b->stack_size > -sp_inc); + VM_ASSERT(sp_inc < UINT_MAX - b->stack_size); + + if (bf->compiler) { + fprintf(f, "{\n"); + fprintf(f, " VALUE val;\n"); + bf->compiler(f, <%= + insn.name == 'invokebuiltin' ? '-1' : '(rb_num_t)operands[1]' + %>, b->stack_size, body->builtin_inline_p); + fprintf(f, " stack[%u] = val;\n", sp - 1); + fprintf(f, "}\n"); +% if insn.name != 'opt_invokebuiltin_delegate_leave' + b->stack_size = sp; + break; +% end + } diff --git a/tool/ruby_vm/views/_mjit_compile_ivar.erb b/tool/ruby_vm/views/_mjit_compile_ivar.erb new file mode 100644 index 0000000000..5105584ba3 --- /dev/null +++ b/tool/ruby_vm/views/_mjit_compile_ivar.erb @@ -0,0 +1,101 @@ +% # -*- C -*- +% # Copyright (c) 2018 Takashi Kokubun. All rights reserved. +% # +% # This file is a part of the programming language Ruby. Permission is hereby +% # granted, to either redistribute and/or modify this file, provided that the +% # conditions mentioned in the file COPYING are met. Consult the file for +% # details. +% +% # Optimized case of get_instancevariable instruction. +#if OPT_IC_FOR_IVAR +{ +% # compiler: Prepare operands which may be used by `insn.call_attribute` +% insn.opes.each_with_index do |ope, i| + MAYBE_UNUSED(<%= ope.fetch(:decl) %>) = (<%= ope.fetch(:type) %>)operands[<%= i %>]; +% end +% # compiler: Use copied IVC to avoid race condition + IVC ic_copy = &(status->is_entries + ((union iseq_inline_storage_entry *)ic - body->is_entries))->iv_cache; +% + if (!status->compile_info->disable_ivar_cache && ic_copy->entry) { // Only ic_copy is enabled. +% # JIT: optimize away motion of sp and pc. This path does not call rb_warning() and so it's always leaf and not `handles_sp`. +% # <%= render 'mjit_compile_pc_and_sp', locals: { insn: insn } -%> +% +% # JIT: prepare vm_getivar/vm_setivar arguments and variables + fprintf(f, "{\n"); + fprintf(f, " VALUE obj = GET_SELF();\n"); + fprintf(f, " const uint32_t index = %u;\n", (ic_copy->entry->index)); + if (status->merge_ivar_guards_p) { +% # JIT: Access ivar without checking these VM_ASSERTed prerequisites as we checked them in the beginning of `mjit_compile_body` + fprintf(f, " VM_ASSERT(RB_TYPE_P(obj, T_OBJECT));\n"); + fprintf(f, " VM_ASSERT((rb_serial_t)%"PRI_SERIALT_PREFIX"u == RCLASS_SERIAL(RBASIC(obj)->klass));\n", ic_copy->entry->class_serial); + fprintf(f, " VM_ASSERT(index < ROBJECT_NUMIV(obj));\n"); +% if insn.name == 'setinstancevariable' + fprintf(f, " if (LIKELY(!RB_OBJ_FROZEN_RAW(obj) && %s)) {\n", status->max_ivar_index >= ROBJECT_EMBED_LEN_MAX ? "true" : "RB_FL_ANY_RAW(obj, ROBJECT_EMBED)"); + fprintf(f, " RB_OBJ_WRITE(obj, &ROBJECT(obj)->as.%s, stack[%d]);\n", + status->max_ivar_index >= ROBJECT_EMBED_LEN_MAX ? "heap.ivptr[index]" : "ary[index]", b->stack_size - 1); + fprintf(f, " }\n"); +% else + fprintf(f, " VALUE val;\n"); + fprintf(f, " if (LIKELY(%s && (val = ROBJECT(obj)->as.%s) != Qundef)) {\n", + status->max_ivar_index >= ROBJECT_EMBED_LEN_MAX ? "true" : "RB_FL_ANY_RAW(obj, ROBJECT_EMBED)", + status->max_ivar_index >= ROBJECT_EMBED_LEN_MAX ? "heap.ivptr[index]" : "ary[index]"); + fprintf(f, " stack[%d] = val;\n", b->stack_size); + fprintf(f, " }\n"); +%end + } + else { + fprintf(f, " const rb_serial_t ic_serial = (rb_serial_t)%"PRI_SERIALT_PREFIX"u;\n", ic_copy->entry->class_serial); +% # JIT: cache hit path of vm_getivar/vm_setivar, or cancel JIT (recompile it with exivar) +% if insn.name == 'setinstancevariable' + fprintf(f, " if (LIKELY(RB_TYPE_P(obj, T_OBJECT) && ic_serial == RCLASS_SERIAL(RBASIC(obj)->klass) && index < ROBJECT_NUMIV(obj) && !RB_OBJ_FROZEN_RAW(obj))) {\n"); + fprintf(f, " VALUE *ptr = ROBJECT_IVPTR(obj);\n"); + fprintf(f, " RB_OBJ_WRITE(obj, &ptr[index], stack[%d]);\n", b->stack_size - 1); + fprintf(f, " }\n"); +% else + fprintf(f, " VALUE val;\n"); + fprintf(f, " if (LIKELY(RB_TYPE_P(obj, T_OBJECT) && ic_serial == RCLASS_SERIAL(RBASIC(obj)->klass) && index < ROBJECT_NUMIV(obj) && (val = ROBJECT_IVPTR(obj)[index]) != Qundef)) {\n"); + fprintf(f, " stack[%d] = val;\n", b->stack_size); + fprintf(f, " }\n"); +% end + } + fprintf(f, " else {\n"); + fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); + fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); + fprintf(f, " goto ivar_cancel;\n"); + fprintf(f, " }\n"); + +% # compiler: Move JIT compiler's internal stack pointer + b->stack_size += <%= insn.call_attribute('sp_inc') %>; + fprintf(f, "}\n"); + break; + } +% if insn.name == 'getinstancevariable' + else if (!status->compile_info->disable_exivar_cache && ic_copy->entry) { +% # JIT: optimize away motion of sp and pc. This path does not call rb_warning() and so it's always leaf and not `handles_sp`. +% # <%= render 'mjit_compile_pc_and_sp', locals: { insn: insn } -%> +% +% # JIT: prepare vm_getivar's arguments and variables + fprintf(f, "{\n"); + fprintf(f, " VALUE obj = GET_SELF();\n"); + fprintf(f, " const rb_serial_t ic_serial = (rb_serial_t)%"PRI_SERIALT_PREFIX"u;\n", ic_copy->entry->class_serial); + fprintf(f, " const uint32_t index = %u;\n", ic_copy->entry->index); +% # JIT: cache hit path of vm_getivar, or cancel JIT (recompile it without any ivar optimization) + fprintf(f, " struct gen_ivtbl *ivtbl;\n"); + fprintf(f, " VALUE val;\n"); + fprintf(f, " if (LIKELY(FL_TEST_RAW(obj, FL_EXIVAR) && ic_serial == RCLASS_SERIAL(RBASIC(obj)->klass) && rb_ivar_generic_ivtbl_lookup(obj, &ivtbl) && index < ivtbl->numiv && (val = ivtbl->ivptr[index]) != Qundef)) {\n"); + fprintf(f, " stack[%d] = val;\n", b->stack_size); + fprintf(f, " }\n"); + fprintf(f, " else {\n"); + fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); + fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); + fprintf(f, " goto exivar_cancel;\n"); + fprintf(f, " }\n"); + +% # compiler: Move JIT compiler's internal stack pointer + b->stack_size += <%= insn.call_attribute('sp_inc') %>; + fprintf(f, "}\n"); + break; + } +% end +} +#endif // OPT_IC_FOR_IVAR diff --git a/tool/ruby_vm/views/_mjit_compile_pc_and_sp.erb b/tool/ruby_vm/views/_mjit_compile_pc_and_sp.erb new file mode 100644 index 0000000000..390b3ce525 --- /dev/null +++ b/tool/ruby_vm/views/_mjit_compile_pc_and_sp.erb @@ -0,0 +1,38 @@ +% # Copyright (c) 2018 Takashi Kokubun. All rights reserved. +% # +% # This file is a part of the programming language Ruby. Permission is hereby +% # granted, to either redistribute and/or modify this file, provided that the +% # conditions mentioned in the file COPYING are met. Consult the file for +% # details. +% +% # JIT: When an insn is leaf, we don't need to Move pc for a catch table on catch_except_p, #caller_locations, +% # and rb_profile_frames. For check_ints, we lazily move PC when we have interruptions. + MAYBE_UNUSED(bool pc_moved_p) = false; + if (<%= !(insn.always_leaf? || insn.leaf_without_check_ints?) %>) { + fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ + pc_moved_p = true; + } +% +% # JIT: move sp to use or preserve stack variables + if (status->local_stack_p) { +% # sp motion is optimized away for `handles_sp? #=> false` case. +% # Thus sp should be set properly before `goto cancel`. +% if insn.handles_sp? +% # JIT-only behavior (pushing JIT's local variables to VM's stack): + { + rb_snum_t i, push_size; + push_size = -<%= insn.call_attribute('sp_inc') %> + <%= insn.rets.size %> - <%= insn.pops.size %>; + fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ + for (i = 0; i < push_size; i++) { + fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); + } + } +% end + } + else { +% if insn.handles_sp? + fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - <%= insn.pops.size %>); /* POPN(INSN_ATTR(popn)); */ +% else + fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); +% end + } diff --git a/tool/ruby_vm/views/_mjit_compile_send.erb b/tool/ruby_vm/views/_mjit_compile_send.erb new file mode 100644 index 0000000000..28e316a1ef --- /dev/null +++ b/tool/ruby_vm/views/_mjit_compile_send.erb @@ -0,0 +1,119 @@ +% # -*- C -*- +% # Copyright (c) 2018 Takashi Kokubun. All rights reserved. +% # +% # This file is a part of the programming language Ruby. Permission is hereby +% # granted, to either redistribute and/or modify this file, provided that the +% # conditions mentioned in the file COPYING are met. Consult the file for +% # details. +% +% # Optimized case of send / opt_send_without_block instructions. +{ +% # compiler: Prepare operands which may be used by `insn.call_attribute` +% insn.opes.each_with_index do |ope, i| + MAYBE_UNUSED(<%= ope.fetch(:decl) %>) = (<%= ope.fetch(:type) %>)operands[<%= i %>]; +% end +% # compiler: Use captured cc to avoid race condition + size_t cd_index = call_data_index(cd, body); + const struct rb_callcache **cc_entries = captured_cc_entries(status); + const struct rb_callcache *captured_cc = cc_entries[cd_index]; +% +% # compiler: Inline send insn where some supported fastpath is used. + const rb_iseq_t *iseq = NULL; + const CALL_INFO ci = cd->ci; + int kw_splat = IS_ARGS_KW_SPLAT(ci) > 0; + extern bool rb_splat_or_kwargs_p(const struct rb_callinfo *restrict ci); + if (!status->compile_info->disable_send_cache && has_valid_method_type(captured_cc) && ( +% # `CC_SET_FASTPATH(cd->cc, vm_call_cfunc_with_frame, ...)` in `vm_call_cfunc` + (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC + && !rb_splat_or_kwargs_p(ci) && !kw_splat) +% # `CC_SET_FASTPATH(cc, vm_call_iseq_setup_func(...), vm_call_iseq_optimizable_p(...))` in `vm_callee_setup_arg`, +% # and support only non-VM_CALL_TAILCALL path inside it + || (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_ISEQ + && fastpath_applied_iseq_p(ci, captured_cc, iseq = def_iseq_ptr(vm_cc_cme(captured_cc)->def)) + && !(vm_ci_flag(ci) & VM_CALL_TAILCALL)) + )) { + const bool cfunc_debug = false; // Set true when you want to see inlined cfunc + if (cfunc_debug && vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) + fprintf(stderr, " * %s\n", rb_id2name(vm_ci_mid(ci))); + + int sp_inc = (int)sp_inc_of_sendish(ci); + fprintf(f, "{\n"); + +% # JIT: Invalidate call cache if it requires vm_search_method. This allows to inline some of following things. + bool opt_class_of = !maybe_special_const_class_p(captured_cc->klass); // If true, use RBASIC_CLASS instead of CLASS_OF to reduce code size + fprintf(f, " const struct rb_callcache *cc = (const struct rb_callcache *)0x%"PRIxVALUE";\n", (VALUE)captured_cc); + fprintf(f, " const rb_callable_method_entry_t *cc_cme = (const rb_callable_method_entry_t *)0x%"PRIxVALUE";\n", (VALUE)vm_cc_cme(captured_cc)); + fprintf(f, " const VALUE recv = stack[%d];\n", b->stack_size + sp_inc - 1); + fprintf(f, " if (UNLIKELY(%s || !vm_cc_valid_p(cc, cc_cme, %s(recv)))) {\n", opt_class_of ? "RB_SPECIAL_CONST_P(recv)" : "false", opt_class_of ? "RBASIC_CLASS" : "CLASS_OF"); + fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); + fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); + fprintf(f, " goto send_cancel;\n"); + fprintf(f, " }\n"); + +% # JIT: move sp and pc if necessary +<%= render 'mjit_compile_pc_and_sp', locals: { insn: insn } -%> + +% # JIT: If ISeq is inlinable, call the inlined method without pushing a frame. + if (iseq && status->inlined_iseqs != NULL && iseq->body == status->inlined_iseqs[pos]) { + fprintf(f, " {\n"); + fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); + fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size + sp_inc - 1); + fprintf(f, " stack[%d] = _mjit%d_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size + sp_inc - 1, status->compiled_id, pos); + fprintf(f, " reg_cfp->self = orig_self;\n"); + fprintf(f, " }\n"); + } + else { +% # JIT: Forked `vm_sendish` (except method_explorer = vm_search_method_wrap) to inline various things + fprintf(f, " {\n"); + fprintf(f, " VALUE val;\n"); + fprintf(f, " struct rb_calling_info calling;\n"); +% if insn.name == 'send' + fprintf(f, " calling.block_handler = vm_caller_setup_arg_block(ec, reg_cfp, (const struct rb_callinfo *)0x%"PRIxVALUE", (rb_iseq_t *)0x%"PRIxVALUE", FALSE);\n", (VALUE)ci, (VALUE)blockiseq); +% else + fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); +% end + fprintf(f, " calling.kw_splat = %d;\n", kw_splat); + fprintf(f, " calling.recv = stack[%d];\n", b->stack_size + sp_inc - 1); + fprintf(f, " calling.argc = %d;\n", vm_ci_argc(ci)); + + if (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) { +% # TODO: optimize this more + fprintf(f, " calling.ci = (CALL_INFO)0x%"PRIxVALUE";\n", (VALUE)ci); // creating local cd here because operand's cd->cc may not be the same as inlined cc. + fprintf(f, " calling.cc = cc;"); + fprintf(f, " val = vm_call_cfunc_with_frame(ec, reg_cfp, &calling);\n"); + } + else { // VM_METHOD_TYPE_ISEQ +% # fastpath_applied_iseq_p checks rb_simple_iseq_p, which ensures has_opt == FALSE + fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, cc_cme, 0, %d, %d);\n", iseq->body->param.size, iseq->body->local_table_size); + if (iseq->body->catch_except_p) { + fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); + fprintf(f, " val = vm_exec(ec, true);\n"); + } + else { + fprintf(f, " if ((val = mjit_exec(ec)) == Qundef) {\n"); + fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup + fprintf(f, " val = vm_exec(ec, false);\n"); + fprintf(f, " }\n"); + } + } + fprintf(f, " stack[%d] = val;\n", b->stack_size + sp_inc - 1); + fprintf(f, " }\n"); + +% # JIT: We should evaluate ISeq modified for TracePoint if it's enabled. Note: This is slow. + fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); + fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)<%= insn.call_attribute('sp_inc') %>); + if (!pc_moved_p) { + fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); + } + fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); + fprintf(f, " goto cancel;\n"); + fprintf(f, " }\n"); + } + +% # compiler: Move JIT compiler's internal stack pointer + b->stack_size += <%= insn.call_attribute('sp_inc') %>; + + fprintf(f, "}\n"); + break; + } +} diff --git a/tool/ruby_vm/views/_notice.erb b/tool/ruby_vm/views/_notice.erb new file mode 100644 index 0000000000..d17e019727 --- /dev/null +++ b/tool/ruby_vm/views/_notice.erb @@ -0,0 +1,22 @@ +%# -*- C -*- +%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +%; +%# This file is a part of the programming language Ruby. Permission is hereby +%# granted, to either redistribute and/or modify this file, provided that the +%# conditions mentioned in the file COPYING are met. Consult the file for +%# details. +%; +%; +/*******************************************************************/ +/*******************************************************************/ +/*******************************************************************/ +/** + This file <%= this_file %>. + + ---- + This file is auto generated by insns2vm.rb + DO NOT TOUCH! + + If you want to fix something, you must edit <%= cstr edit %> + or tool/insns2vm.rb + */ diff --git a/tool/ruby_vm/views/_sp_inc_helpers.erb b/tool/ruby_vm/views/_sp_inc_helpers.erb new file mode 100644 index 0000000000..d0b0bd79ef --- /dev/null +++ b/tool/ruby_vm/views/_sp_inc_helpers.erb @@ -0,0 +1,37 @@ +%# -*- C -*- +%# Copyright (c) 2018 Urabe, Shyouhei. All rights reserved. +%# +%# This file is a part of the programming language Ruby. Permission is hereby +%# granted, to either redistribute and/or modify this file, provided that the +%# conditions mentioned in the file COPYING are met. Consult the file for +%# details. +%; +#line <%= __LINE__ + 1 %> <%=cstr __FILE__ %> + +static rb_snum_t +sp_inc_of_sendish(const struct rb_callinfo *ci) +{ + /* Send-ish instructions will: + * + * 1. Pop block argument, if any. + * 2. Pop ordinal arguments. + * 3. Pop receiver. + * 4. Push return value. + */ + const int argb = (vm_ci_flag(ci) & VM_CALL_ARGS_BLOCKARG) ? 1 : 0; + const int argc = vm_ci_argc(ci); + const int recv = 1; + const int retn = 1; + + /* 1. 2. 3. 4. */ + return 0 - argb - argc - recv + retn; +} + +static rb_snum_t +sp_inc_of_invokeblock(const struct rb_callinfo *ci) +{ + /* sp_inc of invokeblock is almost identical to that of sendish + * instructions, except that it does not pop receiver. */ + return sp_inc_of_sendish(ci) + 1; +} +#pragma RubyVM reset source diff --git a/tool/ruby_vm/views/_trace_instruction.erb b/tool/ruby_vm/views/_trace_instruction.erb new file mode 100644 index 0000000000..3588207d39 --- /dev/null +++ b/tool/ruby_vm/views/_trace_instruction.erb @@ -0,0 +1,21 @@ +%# -*- C -*- +%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +%# +%# This file is a part of the programming language Ruby. Permission is hereby +%# granted, to either redistribute and/or modify this file, provided that the +%# conditions mentioned in the file COPYING are met. Consult the file for +%# details. +%; + +/* insn <%= insn.pretty_name %> */ +INSN_ENTRY(<%= insn.name %>) +{ + vm_trace(ec, GET_CFP()); +% if insn.name =~ +% /\Atrace_opt_(plus|minus|mult|div|mod|eq|neq|lt|le|gt|ge|ltlt|and|or|aref|aset|length|size|empty_p|nil_p|succ|not|regexpmatch2)\z/ +% jump_dest = "opt_send_without_block" +% end + <%= 'ADD_PC(1);' if insn.name == 'trace_opt_neq' %> + DISPATCH_ORIGINAL_INSN(<%= jump_dest || insn.jump_destination %>); + END_INSN(<%= insn.name %>); +} diff --git a/tool/ruby_vm/views/insns.inc.erb b/tool/ruby_vm/views/insns.inc.erb new file mode 100644 index 0000000000..29981a8a2d --- /dev/null +++ b/tool/ruby_vm/views/insns.inc.erb @@ -0,0 +1,26 @@ +/* -*- C -*- */ + +%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +%# +%# This file is a part of the programming language Ruby. Permission is hereby +%# granted, to either redistribute and/or modify this file, provided that the +%# conditions mentioned in the file COPYING are met. Consult the file for +%# details. +<%= render 'copyright' %> +<%= render 'notice', locals: { + this_file: 'contains YARV instruction list', + edit: __FILE__, +} -%> + +/* BIN : Basic Instruction Name */ +#define BIN(n) YARVINSN_##n + +enum ruby_vminsn_type { +% RubyVM::Instructions.each do |i| + <%= i.bin %>, +% end + VM_INSTRUCTION_SIZE +}; + +#define ASSERT_VM_INSTRUCTION_SIZE(array) \ + STATIC_ASSERT(numberof_##array, numberof(array) == VM_INSTRUCTION_SIZE) diff --git a/tool/ruby_vm/views/insns_info.inc.erb b/tool/ruby_vm/views/insns_info.inc.erb new file mode 100644 index 0000000000..2ca5aca7cf --- /dev/null +++ b/tool/ruby_vm/views/insns_info.inc.erb @@ -0,0 +1,22 @@ +/* -*- C -*- */ + +%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +%# +%# This file is a part of the programming language Ruby. Permission is hereby +%# granted, to either redistribute and/or modify this file, provided that the +%# conditions mentioned in the file COPYING are met. Consult the file for +%# details. +<%= render 'copyright' %> +<%= render 'notice', locals: { + this_file: 'contains instruction information for yarv instruction sequence.', + edit: __FILE__, +} %> +<%= render 'insn_type_chars' %> +<%= render 'insn_name_info' %> +<%= render 'insn_len_info' %> +<%= render 'insn_operand_info' %> +<%= render 'leaf_helpers' %> +<%= render 'sp_inc_helpers' %> +<%= render 'attributes' %> +<%= render 'comptime_insn_stack_increase' %> +<%= render 'insn_sp_pc_dependency' %> diff --git a/tool/ruby_vm/views/mjit_compile.inc.erb b/tool/ruby_vm/views/mjit_compile.inc.erb new file mode 100644 index 0000000000..5820f81770 --- /dev/null +++ b/tool/ruby_vm/views/mjit_compile.inc.erb @@ -0,0 +1,110 @@ +/* -*- C -*- */ + +% # Copyright (c) 2018 Takashi Kokubun. All rights reserved. +% # +% # This file is a part of the programming language Ruby. Permission is hereby +% # granted, to either redistribute and/or modify this file, provided that the +% # conditions mentioned in the file COPYING are met. Consult the file for +% # details. +<%= render 'copyright' %> +% +% # This is an ERB template that generates Ruby code that generates C code that +% # generates JIT-ed C code. +<%= render 'notice', locals: { + this_file: 'is the main part of compile_insn() in mjit_compile.c', + edit: __FILE__, +} -%> +% +% unsupported_insns = [ +% 'defineclass', # low priority +% ] +% +% opt_send_without_block = RubyVM::Instructions.find { |i| i.name == 'opt_send_without_block' } +% if opt_send_without_block.nil? +% raise 'opt_send_without_block not found' +% end +% +% send_compatible_opt_insns = RubyVM::BareInstructions.to_a.select do |insn| +% insn.name.start_with?('opt_') && opt_send_without_block.opes == insn.opes && +% insn.expr.expr.lines.any? { |l| l.match(/\A\s+CALL_SIMPLE_METHOD\(\);\s+\z/) } +% end.map(&:name) +% +% # Available variables and macros in JIT-ed function: +% # ec: the first argument of _mjitXXX +% # reg_cfp: the second argument of _mjitXXX +% # GET_CFP(): refers to `reg_cfp` +% # GET_EP(): refers to `reg_cfp->ep` +% # GET_SP(): refers to `reg_cfp->sp`, or `(stack + stack_size)` if local_stack_p +% # GET_SELF(): refers to `cfp_self` +% # GET_LEP(): refers to `VM_EP_LEP(reg_cfp->ep)` +% # EXEC_EC_CFP(): refers to `val = vm_exec(ec, true)` with frame setup +% # CALL_METHOD(): using `GET_CFP()` and `EXEC_EC_CFP()` +% # TOPN(): refers to `reg_cfp->sp`, or `*(stack + (stack_size - num - 1))` if local_stack_p +% # STACK_ADDR_FROM_TOP(): refers to `reg_cfp->sp`, or `stack + (stack_size - num)` if local_stack_p +% # DISPATCH_ORIGINAL_INSN(): expanded in _mjit_compile_insn.erb +% # THROW_EXCEPTION(): specially defined for JIT +% # RESTORE_REGS(): specially defined for `leave` + +switch (insn) { +% (RubyVM::BareInstructions.to_a + RubyVM::OperandsUnifications.to_a).each do |insn| +% next if unsupported_insns.include?(insn.name) + case BIN(<%= insn.name %>): { +% # Instruction-specific behavior in JIT +% case insn.name +% when 'opt_send_without_block', 'send' +<%= render 'mjit_compile_send', locals: { insn: insn } -%> +% when *send_compatible_opt_insns +% # To avoid cancel, just emit `opt_send_without_block` instead of `opt_*` insn if call cache is populated. +% cd_index = insn.opes.index { |o| o.fetch(:type) == 'CALL_DATA' } + if (has_cache_for_send(captured_cc_entries(status)[call_data_index((CALL_DATA)operands[<%= cd_index %>], body)], BIN(<%= insn.name %>))) { +<%= render 'mjit_compile_send', locals: { insn: opt_send_without_block } -%> +<%= render 'mjit_compile_insn', locals: { insn: opt_send_without_block } -%> + break; + } +% when 'getinstancevariable', 'setinstancevariable' +<%= render 'mjit_compile_ivar', locals: { insn: insn } -%> +% when 'invokebuiltin', 'opt_invokebuiltin_delegate' +<%= render 'mjit_compile_invokebuiltin', locals: { insn: insn } -%> +% when 'opt_getinlinecache' +<%= render 'mjit_compile_getinlinecache', locals: { insn: insn } -%> +% when 'leave', 'opt_invokebuiltin_delegate_leave' +% # opt_invokebuiltin_delegate_leave also implements leave insn. We need to handle it here for inlining. +% if insn.name == 'opt_invokebuiltin_delegate_leave' +<%= render 'mjit_compile_invokebuiltin', locals: { insn: insn } -%> +% else + if (b->stack_size != 1) { + if (mjit_opts.warnings || mjit_opts.verbose) + fprintf(stderr, "MJIT warning: Unexpected JIT stack_size on leave: %d\n", b->stack_size); + status->success = false; + } +% end +% # Skip vm_pop_frame for inlined call + if (status->inlined_iseqs != NULL) { // the current ISeq is NOT being inlined +% # Cancel on interrupts to make leave insn leaf + fprintf(f, " if (UNLIKELY(RUBY_VM_INTERRUPTED_ANY(ec))) {\n"); + fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); + fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); + fprintf(f, " rb_threadptr_execute_interrupts(rb_ec_thread_ptr(ec), 0);\n"); + fprintf(f, " }\n"); + fprintf(f, " ec->cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(reg_cfp);\n"); // vm_pop_frame + } + fprintf(f, " return stack[0];\n"); + b->stack_size += <%= insn.call_attribute('sp_inc') %>; + b->finish_p = TRUE; + break; +% end +% +% # Main insn implementation generated by insns.def +<%= render 'mjit_compile_insn', locals: { insn: insn } -%> + break; + } +% end +% +% # We don't support InstructionsUnifications yet because it's not used for now. +% # We don't support TraceInstructions yet. There is no blocker for it but it's just not implemented. + default: + if (mjit_opts.warnings || mjit_opts.verbose) + fprintf(stderr, "MJIT warning: Skipped to compile unsupported instruction: %s\n", insn_name(insn)); + status->success = false; + break; +} diff --git a/tool/ruby_vm/views/opt_sc.inc.erb b/tool/ruby_vm/views/opt_sc.inc.erb new file mode 100644 index 0000000000..e58c81989f --- /dev/null +++ b/tool/ruby_vm/views/opt_sc.inc.erb @@ -0,0 +1,40 @@ +/* -*- C -*- */ + +%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +%# +%# This file is a part of the programming language Ruby. Permission is hereby +%# granted, to either redistribute and/or modify this file, provided that the +%# conditions mentioned in the file COPYING are met. Consult the file for +%# details. +% raise ':FIXME:TBW' if RubyVM::VmOptsH['STACK_CACHING'] +<%= render 'copyright' %> +<%= render 'notice', locals: { + this_file: 'is for threaded code', + edit: __FILE__, +} -%> + +#define SC_STATE_SIZE 6 + +#define SCS_XX 1 +#define SCS_AX 2 +#define SCS_BX 3 +#define SCS_AB 4 +#define SCS_BA 5 + +#define SC_ERROR 0xffffffff + +static const VALUE sc_insn_info[][SC_STATE_SIZE] = { +#define NO_SC { SC_ERROR, SC_ERROR, SC_ERROR, SC_ERROR, SC_ERROR, SC_ERROR } +% RubyVM::Instructions.each_slice 8 do |a| + <%= a.map{|i| 'NO_SC' }.join(', ') %>, +% end +#undef NO_SC +}; + +static const VALUE sc_insn_next[] = { +% RubyVM::Instructions.each_slice 8 do |a| + <%= a.map{|i| 'SCS_XX' }.join(', ') %>, +% end +}; + +ASSERT_VM_INSTRUCTION_SIZE(sc_insn_next); diff --git a/tool/ruby_vm/views/optinsn.inc.erb b/tool/ruby_vm/views/optinsn.inc.erb new file mode 100644 index 0000000000..676f1edaba --- /dev/null +++ b/tool/ruby_vm/views/optinsn.inc.erb @@ -0,0 +1,71 @@ +/* -*- C -*- */ + +%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +%# +%# This file is a part of the programming language Ruby. Permission is hereby +%# granted, to either redistribute and/or modify this file, provided that the +%# conditions mentioned in the file COPYING are met. Consult the file for +%# details. +<%= render 'copyright' -%> +<%= render 'notice', locals: { + this_file: 'is for threaded code', + edit: __FILE__, +} -%> + +static INSN * +insn_operands_unification(INSN *iobj) +{ +#ifdef OPT_OPERANDS_UNIFICATION + VALUE *op = iobj->operands; + + switch (iobj->insn_id) { + default: + /* do nothing */; + break; + +% RubyVM::OperandsUnifications.each_group do |orig, unifs| + case <%= orig.bin %>: +% unifs.each do |insn| + + /* <%= insn.pretty_name %> */ + if ( <%= insn.condition('op') %> ) { +% insn.opes.each_with_index do |o, x| +% n = insn.operand_shift_of(o) +% if n != 0 then + op[<%= x %>] = op[<%= x + n %>]; +% end +% end + iobj->insn_id = <%= insn.bin %>; + iobj->operand_size = <%= insn.opes.size %>; + break; + } +% end + + break; +% end + } +#endif + return iobj; +} + +int +rb_insn_unified_local_var_level(VALUE insn) +{ +#ifdef OPT_OPERANDS_UNIFICATION + /* optimize rule */ + switch (insn) { + default: + return -1; /* do nothing */; +% RubyVM::OperandsUnifications.each_group do |orig, unifs| +% unifs.each do|insn| + case <%= insn.bin %>: +% insn.spec.map{|(var,val)|val}.reject{|i| i == '*' }.each do |val| + return <%= val %>; +% break +% end +% end +% end + } +#endif + return -1; +} diff --git a/tool/ruby_vm/views/optunifs.inc.erb b/tool/ruby_vm/views/optunifs.inc.erb new file mode 100644 index 0000000000..e92a95beff --- /dev/null +++ b/tool/ruby_vm/views/optunifs.inc.erb @@ -0,0 +1,21 @@ +/* -*- C -*- */ + +%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +%# +%# This file is a part of the programming language Ruby. Permission is hereby +%# granted, to either redistribute and/or modify this file, provided that the +%# conditions mentioned in the file COPYING are met. Consult the file for +%# details. +% raise ':FIXME:TBW' if RubyVM::VmOptsH['INSTRUCTIONS_UNIFICATION'] +% n = RubyVM::Instructions.size +<%= render 'copyright' %> +<%= render 'notice', locals: { + this_file: 'is for threaded code', + edit: __FILE__, +} -%> + +/* Let .bss section automatically initialize this variable */ +/* cf. Section 6.7.8 of ISO/IEC 9899:1999 */ +static const int *const *const unified_insns_data[<%= n %>]; + +ASSERT_VM_INSTRUCTION_SIZE(unified_insns_data); diff --git a/tool/ruby_vm/views/vm.inc.erb b/tool/ruby_vm/views/vm.inc.erb new file mode 100644 index 0000000000..c1a3faf60a --- /dev/null +++ b/tool/ruby_vm/views/vm.inc.erb @@ -0,0 +1,30 @@ +/* -*- C -*- */ + +%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +%# +%# This file is a part of the programming language Ruby. Permission is hereby +%# granted, to either redistribute and/or modify this file, provided that the +%# conditions mentioned in the file COPYING are met. Consult the file for +%# details. +<%= render 'copyright' %> +<%= render 'notice', locals: { + this_file: 'is VM main loop', + edit: __FILE__, +} -%> + +#include "vm_insnhelper.h" +% RubyVM::BareInstructions.to_a.each do |insn| +<%= render 'insn_entry', locals: { insn: insn } -%> +% end +% +% RubyVM::OperandsUnifications.to_a.each do |insn| +<%= render 'insn_entry', locals: { insn: insn } -%> +% end +% +% RubyVM::InstructionsUnifications.to_a.each do |insn| +<%= render 'insn_entry', locals: { insn: insn } -%> +% end +% +% RubyVM::TraceInstructions.to_a.each do |insn| +<%= render 'trace_instruction', locals: { insn: insn } -%> +% end diff --git a/tool/ruby_vm/views/vmtc.inc.erb b/tool/ruby_vm/views/vmtc.inc.erb new file mode 100644 index 0000000000..99cbd92614 --- /dev/null +++ b/tool/ruby_vm/views/vmtc.inc.erb @@ -0,0 +1,21 @@ +/* -*- C -*- */ + +%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +%# +%# This file is a part of the programming language Ruby. Permission is hereby +%# granted, to either redistribute and/or modify this file, provided that the +%# conditions mentioned in the file COPYING are met. Consult the file for +%# details. +<%= render 'copyright' -%> +<%= render 'notice', locals: { + this_file: 'is for threaded code', + edit: __FILE__, +} -%> + +static const void *const insns_address_table[] = { +% RubyVM::Instructions.each do |i| + LABEL_PTR(<%= i.name %>), +% end +}; + +ASSERT_VM_INSTRUCTION_SIZE(insns_address_table); diff --git a/tool/run-gcov.rb b/tool/run-gcov.rb new file mode 100644 index 0000000000..5df7622aa3 --- /dev/null +++ b/tool/run-gcov.rb @@ -0,0 +1,54 @@ +#!ruby +require "pathname" +require "open3" + +Pathname.glob("**/*.gcda").sort.each do |gcda| + if gcda.fnmatch("ext/*") + cwd, gcda = gcda.split.map {|s| s.to_s } + objdir = "." + elsif gcda.fnmatch("rubyspec_temp/*") + next + else + cwd, objdir, gcda = ".", gcda.dirname.to_s, gcda.to_s + end + puts "$ gcov -lpbc -o #{ objdir } #{ gcda }" + out, err, _status = Open3.capture3("gcov", "-lpbc", "-o", objdir, gcda, chdir: cwd) + puts out + puts err + + # a black list of source files that contains wrong #line directives + if err !~ %r( + \A( + Cannot\ open\ source\ file\ ( + defs/keywords + |zonetab\.list + |enc/jis/props\.kwd + |parser\.c + |parser\.rl + )\n + )*\z + )x + raise "Unexpected gcov output" + end + + if out !~ %r( + \A( + File\ .*\nLines\ executed:.*\n + ( + Branches\ executed:.*\n + Taken\ at\ least\ once:.*\n + | + No\ branches\n + )? + ( + Calls\ executed:.*\n + | + No\ calls\n + )? + Creating\ .*\n + \n + )+\z + )x + raise "Unexpected gcov output" + end +end diff --git a/tool/run-lcov.rb b/tool/run-lcov.rb new file mode 100644 index 0000000000..f27578200a --- /dev/null +++ b/tool/run-lcov.rb @@ -0,0 +1,164 @@ +#!ruby +require "pathname" +require "open3" +require "tmpdir" + +def backup_gcda_files(gcda_files) + gcda_files = gcda_files.map do |gcda| + [gcda, gcda.sub_ext(".bak")] + end + begin + gcda_files.each do |before, after| + before.rename(after) + end + yield + ensure + gcda_files.each do |before, after| + after.rename(before) + end + end +end + +def run_lcov(*args) + system("lcov", "--rc", "lcov_branch_coverage=1", *args) +end + +$info_files = [] +def run_lcov_capture(dir, info) + $info_files << info + run_lcov("--capture", "-d", dir, "-o", info) +end + +def run_lcov_merge(files, info) + run_lcov(*files.flat_map {|f| ["--add-tracefile", f] }, "-o", info) +end + +def run_lcov_remove(info_src, info_out) + dirs = %w(/usr/*) + dirs << File.join(Dir.tmpdir, "*") + %w( + test/* + ext/-test-/* + ext/nkf/nkf-utf8/nkf.c + ).each {|f| dirs << File.join(File.dirname(__dir__), f) } + run_lcov("--remove", info_src, *dirs, "-o", info_out) +end + +def run_genhtml(info, out) + system("genhtml", "--branch-coverage", "--ignore-errors", "source", info, "-o", out) +end + +def gen_rb_lcov(file) + res = Marshal.load(File.binread(file)) + + open("lcov-rb-all.info", "w") do |f| + f.puts "TN:" # no test name + base_dir = File.dirname(__dir__) + res.each do |path, cov| + next unless path.start_with?(base_dir) + next if path.start_with?(File.join(base_dir, "test")) + f.puts "SF:#{ path }" + + total = covered = 0 + cov.each_with_index do |count, lineno| + next unless count + f.puts "DA:#{ lineno + 1 },#{ count }" + total += 1 + covered += 1 if count > 0 + end + f.puts "LF:#{ total }" + f.puts "LH:#{ covered }" + + f.puts "end_of_record" + end + end +end + +def gen_rb_lcov(file) + res = Marshal.load(File.binread(file)) + + open("lcov-rb-all.info", "w") do |f| + f.puts "TN:" # no test name + base_dir = File.dirname(File.dirname(__dir__)) + res.each do |path, cov| + next unless path.start_with?(base_dir) + next if path.start_with?(File.join(base_dir, "test")) + f.puts "SF:#{ path }" + + # function coverage + total = covered = 0 + cov[:methods].each do |(klass, name, lineno), count| + f.puts "FN:#{ lineno },#{ klass }##{ name }" + total += 1 + covered += 1 if count > 0 + end + f.puts "FNF:#{ total }" + f.puts "FNF:#{ covered }" + cov[:methods].each do |(klass, name, _), count| + f.puts "FNDA:#{ count },#{ klass }##{ name }" + end + + # line coverage + total = covered = 0 + cov[:lines].each_with_index do |count, lineno| + next unless count + f.puts "DA:#{ lineno + 1 },#{ count }" + total += 1 + covered += 1 if count > 0 + end + f.puts "LF:#{ total }" + f.puts "LH:#{ covered }" + + # branch coverage + total = covered = 0 + id = 0 + cov[:branches].each do |(_base_type, _, base_lineno), targets| + i = 0 + targets.each do |(_target_type, _target_lineno), count| + f.puts "BRDA:#{ base_lineno },#{ id },#{ i },#{ count }" + total += 1 + covered += 1 if count > 0 + i += 1 + end + id += 1 + end + f.puts "BRF:#{ total }" + f.puts "BRH:#{ covered }" + f.puts "end_of_record" + end + end +end + +gcda_files = Pathname.glob("**/*.gcda") +ext_gcda_files = gcda_files.select {|f| f.fnmatch("ext/*") } +rubyspec_temp_gcda_files = gcda_files.select {|f| f.fnmatch("rubyspec_temp/*") } + +backup_gcda_files(rubyspec_temp_gcda_files) do + if ext_gcda_files != [] + backup_gcda_files(ext_gcda_files) do + info = "lcov-root.info" + run_lcov_capture(".", info) + end + end + ext_gcda_files.group_by {|f| f.descend.to_a[1] }.each do |key, files| + info = "lcov-#{ key.to_s.gsub(File::Separator, "-") }.info" + run_lcov_capture(key.to_s, info) + end +end +if $info_files != [] + run_lcov_merge($info_files, "lcov-c-all.info") + run_lcov_remove("lcov-c-all.info", "lcov-c-all-filtered.info") + run_genhtml("lcov-c-all-filtered.info", "lcov-c-out") +end + +if File.readable?("test-coverage.dat") + gen_rb_lcov("test-coverage.dat") + run_lcov_remove("lcov-rb-all.info", "lcov-rb-all-filtered.info") + run_genhtml("lcov-rb-all-filtered.info", "lcov-rb-out") +end + +if File.readable?("lcov-c-all.info") && File.readable?("lcov-rb-all.info") + run_lcov_merge(%w(lcov-c-all.info lcov-rb-all.info), "lcov-all.info") + run_lcov_remove("lcov-all.info", "lcov-all-filtered.info") + run_genhtml("lcov-all-filtered.info", "lcov-out") +end diff --git a/tool/runruby.rb b/tool/runruby.rb new file mode 100755 index 0000000000..1efe38fd13 --- /dev/null +++ b/tool/runruby.rb @@ -0,0 +1,178 @@ +#!./miniruby + +# Used by "make runruby", configure, and by hand to run a locally-built Ruby +# with correct environment variables and arguments. + +show = false +precommand = [] +srcdir = File.realpath('..', File.dirname(__FILE__)) +case +when ENV['RUNRUBY_USE_GDB'] == 'true' + debugger = :gdb +when ENV['RUNRUBY_USE_LLDB'] == 'true' + debugger = :lldb +when ENV['RUNRUBY_YJIT_STATS'] + use_yjit_stat = true +end +while arg = ARGV[0] + break ARGV.shift if arg == '--' + case arg + when '-C', /\A-C(.+)/m + ARGV.shift + Dir.chdir($1 || ARGV.shift) + next + end + /\A--([-\w]+)(?:=(.*))?\z/ =~ arg or break + arg, value = $1, $2 + re = Regexp.new('\A'+arg.gsub(/\w+\b/, '\&\\w*')+'\z', "i") + case + when re =~ "srcdir" + srcdir = value + when re =~ "archdir" + archdir = value + when re =~ "cpu" + precommand << "arch" << "-arch" << value + when re =~ "extout" + extout = value + when re =~ "pure" + # obsolete switch do nothing + when re =~ "debugger" + require 'shellwords' + case value + when nil + debugger = :gdb + when "lldb" + debugger = :lldb + when "no" + else + debugger = Shellwords.shellwords(value) + end and precommand |= [:debugger] + when re =~ "precommand" + require 'shellwords' + precommand.concat(Shellwords.shellwords(value)) + when re =~ "show" + show = true + when re =~ "chdir" + Dir.chdir(value) + else + break + end + ARGV.shift +end + +unless defined?(File.realpath) + def File.realpath(*args) + path = expand_path(*args) + if File.stat(path).directory? + Dir.chdir(path) {Dir.pwd} + else + dir, base = File.split(path) + File.join(Dir.chdir(dir) {Dir.pwd}, base) + end + end +end + +begin + conffile = File.realpath('rbconfig.rb', archdir) +rescue Errno::ENOENT => e + # retry if !archdir and ARGV[0] and File.directory?(archdir = ARGV.shift) + abort "#$0: rbconfig.rb not found, use --archdir option" +end + +abs_archdir = File.dirname(conffile) +archdir ||= abs_archdir +$:.unshift(abs_archdir) + +config = File.read(conffile) +config.sub!(/^(\s*)RUBY_VERSION\b.*(\sor\s*)\n.*\n/, '') +config = Module.new {module_eval(config, conffile)}::RbConfig::CONFIG + +install_name = config["RUBY_INSTALL_NAME"]+config['EXEEXT'] +ruby = File.join(archdir, install_name) +unless File.exist?(ruby) + abort "#{ruby} is not found.\nTry `make' first, then `make test', please.\n" +end + +libs = [abs_archdir] +extout ||= config["EXTOUT"] +if extout + abs_extout = File.expand_path(extout, abs_archdir) + libs << File.expand_path("common", abs_extout) << File.expand_path(config['arch'], abs_extout) +end +libs << File.expand_path("lib", srcdir) +config["bindir"] = abs_archdir + +env = { + # Test with the smallest possible machine stack sizes. + # These values are clamped to machine-dependent minimum values in vm_core.h + 'RUBY_THREAD_MACHINE_STACK_SIZE' => '1', + 'RUBY_FIBER_MACHINE_STACK_SIZE' => '1', +} + +runner = File.join(abs_archdir, "exe/#{install_name}") +runner = nil unless File.exist?(runner) +abs_ruby = runner || File.expand_path(ruby) +env["RUBY"] = abs_ruby +env["GEM_PATH"] = env["GEM_HOME"] = File.expand_path(".bundle", srcdir) +env["GEM_COMMAND"] = "#{abs_ruby} -rrubygems #{srcdir}/bin/gem --backtrace" +env["PATH"] = [File.dirname(abs_ruby), abs_archdir, ENV["PATH"]].compact.join(File::PATH_SEPARATOR) + +if e = ENV["RUBYLIB"] + libs |= e.split(File::PATH_SEPARATOR) +end +env["RUBYLIB"] = $:.replace(libs).join(File::PATH_SEPARATOR) + +gem_path = [abs_archdir, srcdir].map {|d| File.realdirpath(".bundle", d)} +if e = ENV["GEM_PATH"] + gem_path |= e.split(File::PATH_SEPARATOR) +end +env["GEM_PATH"] = gem_path.join(File::PATH_SEPARATOR) + +libruby_so = File.join(abs_archdir, config['LIBRUBY_SO']) +if File.file?(libruby_so) + if e = config['LIBPATHENV'] and !e.empty? + env[e] = [abs_archdir, ENV[e]].compact.join(File::PATH_SEPARATOR) + end +end + +ENV.update env + +if debugger + case debugger + when :gdb, nil + debugger = %W'gdb -x #{srcdir}/.gdbinit' + if File.exist?(gdb = 'run.gdb') or + File.exist?(gdb = File.join(abs_archdir, 'run.gdb')) + debugger.push('-x', gdb) + end + debugger << '--args' + when :lldb + debugger = ['lldb', '-O', "command script import #{srcdir}/misc/lldb_cruby.py"] + if File.exist?(lldb = 'run.lldb') or + File.exist?(lldb = File.join(abs_archdir, 'run.lldb')) + debugger.push('-s', lldb) + end + debugger << '--' + end + + if idx = precommand.index(:debugger) + precommand[idx, 1] = debugger + else + precommand.concat(debugger) + end +end + +cmd = [runner || ruby] +if use_yjit_stat + cmd << '--yjit-stats' +end +cmd.concat(ARGV) +cmd.unshift(*precommand) unless precommand.empty? + +if show + require 'shellwords' + env.each {|k,v| puts "#{k}=#{v}"} + puts Shellwords.join(cmd) +end + +exec(*cmd, close_others: false) diff --git a/tool/search-cgvars.rb b/tool/search-cgvars.rb new file mode 100644 index 0000000000..c62641a3ff --- /dev/null +++ b/tool/search-cgvars.rb @@ -0,0 +1,55 @@ +# +# Listing C's global variables in .so or .o, or .bundle on Mac OS using "objdump -t" (elf64-x86-64) +# to check ractor-safety. +# +# Usage: ruby search-cgvars.rb foo.so bar.o .ext/x86_64-darwin18/psych.bundle +# +MAC_OS = RbConfig::CONFIG['host_os'].match? /darwin|mac os/ + +def gvars file + # '0000000000031ac8 g O .bss 0000000000000008 rb_cSockIfaddr' + # On mac, with .bundle files: + # '0000000000004258 l O __DATA,__bss _passwd_blocking' + + strs = `objdump -t #{file}` + found = {} + strs.each_line{|line| + if /[\da-f]{16} / =~ line + addr = line[0...16] + flags = line[17...24].tr(' ', '').split(//).sort.uniq + rest = line[25..] + if MAC_OS + seg, name = rest.split(/\s+/) + else + seg, size, name = rest.split(/\s+/) + end + if flags.include?('O') + # p [addr, flags, seg, size, name] + found[name] = [flags, seg, *size] + end + end + } + puts "## #{file}:" + found.sort_by{|name, (flags, *)| + [flags, name] + }.each{|name, rest| + flags, seg, size = *rest + next if (size.to_i == 0 && !MAC_OS) && seg != '*UND*' + case seg + when ".rodata", ".data.rel.ro", ".got.plt", ".eh_frame", ".fini_array" + next + when /,__const$/ # Mac OS + next + end + case name + when /^id_/, /^rbimpl_id/, /^sym_/, /^rb_[cme]/, /\Acompleted\.\d+\z/ + next + when /^_id_/, /\.rbimpl_id(\.\d+)?$/ # Mac OS + next + end + puts " %40s %s" % [name, rest.inspect] + } +end +ARGV.each{|file| + gvars file +} diff --git a/tool/strip-rdoc.rb b/tool/strip-rdoc.rb new file mode 100755 index 0000000000..d8e311cdbf --- /dev/null +++ b/tool/strip-rdoc.rb @@ -0,0 +1,14 @@ +#!ruby +# frozen_string_literal: true + +# Filter for preventing Doxygen from processing RDoc comments. +# Used by the Doxygen template. + +print ARGF.binmode.read.tap {|src| + src.gsub!(%r|(/\*[!*])(?:(?!\*/).)+?^\s*\*\s?\-\-\s*$(.+?\*/)|m) { + marker = $1 + comment = $2 + comment.sub!(%r|^\s*\*\s?\+\+\s*$.+?(\s*\*/)\z|m, '\\1') + marker + comment + } +} diff --git a/tool/sync_default_gems.rb b/tool/sync_default_gems.rb new file mode 100755 index 0000000000..564877a26b --- /dev/null +++ b/tool/sync_default_gems.rb @@ -0,0 +1,638 @@ +#!/usr/bin/env ruby +# sync upstream github repositories to ruby repository + +require 'fileutils' +include FileUtils + +REPOSITORIES = { + rubygems: 'rubygems/rubygems', + rdoc: 'ruby/rdoc', + reline: 'ruby/reline', + json: 'flori/json', + psych: 'ruby/psych', + fileutils: 'ruby/fileutils', + fiddle: 'ruby/fiddle', + stringio: 'ruby/stringio', + "io-console": 'ruby/io-console', + "io-nonblock": 'ruby/io-nonblock', + "io-wait": 'ruby/io-wait', + csv: 'ruby/csv', + etc: 'ruby/etc', + date: 'ruby/date', + zlib: 'ruby/zlib', + fcntl: 'ruby/fcntl', + strscan: 'ruby/strscan', + ipaddr: 'ruby/ipaddr', + logger: 'ruby/logger', + ostruct: 'ruby/ostruct', + irb: 'ruby/irb', + forwardable: "ruby/forwardable", + mutex_m: "ruby/mutex_m", + racc: "ruby/racc", + singleton: "ruby/singleton", + open3: "ruby/open3", + getoptlong: "ruby/getoptlong", + pstore: "ruby/pstore", + delegate: "ruby/delegate", + benchmark: "ruby/benchmark", + cgi: "ruby/cgi", + readline: "ruby/readline", + "readline-ext": "ruby/readline-ext", + observer: "ruby/observer", + timeout: "ruby/timeout", + yaml: "ruby/yaml", + uri: "ruby/uri", + openssl: "ruby/openssl", + did_you_mean: "ruby/did_you_mean", + weakref: "ruby/weakref", + tempfile: "ruby/tempfile", + tmpdir: "ruby/tmpdir", + English: "ruby/English", + "net-protocol": "ruby/net-protocol", + "net-http": "ruby/net-http", + bigdecimal: "ruby/bigdecimal", + optparse: "ruby/optparse", + set: "ruby/set", + find: "ruby/find", + rinda: "ruby/rinda", + erb: "ruby/erb", + nkf: "ruby/nkf", + tsort: "ruby/tsort", + abbrev: "ruby/abbrev", + shellwords: "ruby/shellwords", + base64: "ruby/base64", + syslog: "ruby/syslog", + "open-uri": "ruby/open-uri", + securerandom: "ruby/securerandom", + resolv: "ruby/resolv", + "resolv-replace": "ruby/resolv-replace", + time: "ruby/time", + pp: "ruby/pp", + prettyprint: "ruby/prettyprint", + drb: "ruby/drb", + pathname: "ruby/pathname", + digest: "ruby/digest", + error_highlight: "ruby/error_highlight", + un: "ruby/un", + win32ole: "ruby/win32ole", +} + +# We usually don't use this. Please consider using #sync_default_gems_with_commits instead. +def sync_default_gems(gem) + repo = REPOSITORIES[gem.to_sym] + puts "Sync #{repo}" + + upstream = File.join("..", "..", repo) + + case gem + when "rubygems" + rm_rf(%w[lib/rubygems lib/rubygems.rb test/rubygems]) + cp_r(Dir.glob("#{upstream}/lib/rubygems*"), "lib") + cp_r("#{upstream}/test/rubygems", "test") + rm_rf(%w[lib/bundler lib/bundler.rb libexec/bundler libexec/bundle spec/bundler tool/bundler/*]) + cp_r(Dir.glob("#{upstream}/bundler/lib/bundler*"), "lib") + cp_r(Dir.glob("#{upstream}/bundler/exe/bundle*"), "libexec") + + gemspec_content = File.readlines("#{upstream}/bundler/bundler.gemspec").map do |line| + next if line =~ /LICENSE\.md/ + + line.gsub("bundler.gemspec", "lib/bundler/bundler.gemspec").gsub('"exe"', '"libexec"') + end.compact.join + File.write("lib/bundler/bundler.gemspec", gemspec_content) + + cp_r("#{upstream}/bundler/spec", "spec/bundler") + cp_r(Dir.glob("#{upstream}/bundler/tool/bundler/dev_gems*"), "tool/bundler") + cp_r(Dir.glob("#{upstream}/bundler/tool/bundler/test_gems*"), "tool/bundler") + cp_r(Dir.glob("#{upstream}/bundler/tool/bundler/rubocop_gems*"), "tool/bundler") + cp_r(Dir.glob("#{upstream}/bundler/tool/bundler/standard_gems*"), "tool/bundler") + rm_rf(%w[spec/bundler/support/artifice/vcr_cassettes]) + license_files = %w[ + lib/bundler/vendor/thor/LICENSE.md + lib/rubygems/resolver/molinillo/LICENSE + lib/bundler/vendor/molinillo/LICENSE + lib/bundler/vendor/connection_pool/LICENSE + lib/bundler/vendor/net-http-persistent/README.rdoc + lib/bundler/vendor/fileutils/LICENSE.txt + lib/bundler/vendor/tsort/LICENSE.txt + lib/bundler/vendor/uri/LICENSE.txt + lib/rubygems/optparse/COPYING + lib/rubygems/tsort/LICENSE.txt + ] + rm_rf license_files + when "rdoc" + rm_rf(%w[lib/rdoc lib/rdoc.rb test/rdoc libexec/rdoc libexec/ri]) + cp_r(Dir.glob("#{upstream}/lib/rdoc*"), "lib") + cp_r("#{upstream}/test/rdoc", "test") + cp_r("#{upstream}/rdoc.gemspec", "lib/rdoc") + cp_r("#{upstream}/Gemfile", "lib/rdoc") + cp_r("#{upstream}/Rakefile", "lib/rdoc") + cp_r("#{upstream}/exe/rdoc", "libexec") + cp_r("#{upstream}/exe/ri", "libexec") + parser_files = { + 'lib/rdoc/markdown.kpeg' => 'lib/rdoc/markdown.rb', + 'lib/rdoc/markdown/literals.kpeg' => 'lib/rdoc/markdown/literals.rb', + 'lib/rdoc/rd/block_parser.ry' => 'lib/rdoc/rd/block_parser.rb', + 'lib/rdoc/rd/inline_parser.ry' => 'lib/rdoc/rd/inline_parser.rb' + } + Dir.chdir(upstream) do + `bundle install` + parser_files.each_value do |dst| + `bundle exec rake #{dst}` + end + end + parser_files.each_pair do |src, dst| + rm_rf(src) + cp_r("#{upstream}/#{dst}", dst) + end + `git checkout lib/rdoc/.document` + rm_rf(%w[lib/rdoc/Gemfile lib/rdoc/Rakefile]) + when "reline" + rm_rf(%w[lib/reline lib/reline.rb test/reline]) + cp_r(Dir.glob("#{upstream}/lib/reline*"), "lib") + cp_r("#{upstream}/test/reline", "test") + cp_r("#{upstream}/reline.gemspec", "lib/reline") + when "irb" + rm_rf(%w[lib/irb lib/irb.rb test/irb]) + cp_r(Dir.glob("#{upstream}/lib/irb*"), "lib") + cp_r("#{upstream}/test/irb", "test") + cp_r("#{upstream}/irb.gemspec", "lib/irb") + cp_r("#{upstream}/man/irb.1", "man/irb.1") + cp_r("#{upstream}/doc/irb", "doc") + when "json" + rm_rf(%w[ext/json test/json]) + cp_r("#{upstream}/ext/json/ext", "ext/json") + cp_r("#{upstream}/tests", "test/json") + rm_rf("test/json/lib") + cp_r("#{upstream}/lib", "ext/json") + cp_r("#{upstream}/json.gemspec", "ext/json") + cp_r("#{upstream}/VERSION", "ext/json") + rm_rf(%w[ext/json/lib/json/ext ext/json/lib/json/pure.rb ext/json/lib/json/pure]) + `git checkout ext/json/extconf.rb ext/json/parser/prereq.mk ext/json/generator/depend ext/json/parser/depend ext/json/depend` + when "psych" + rm_rf(%w[ext/psych test/psych]) + cp_r("#{upstream}/ext/psych", "ext") + cp_r("#{upstream}/lib", "ext/psych") + cp_r("#{upstream}/test/psych", "test") + rm_rf(%w[ext/psych/lib/org ext/psych/lib/psych.jar ext/psych/lib/psych_jars.rb]) + rm_rf(%w[ext/psych/lib/psych.{bundle,so} ext/psych/lib/2.*]) + rm_rf(["ext/psych/yaml/LICENSE"]) + cp_r("#{upstream}/psych.gemspec", "ext/psych") + `git checkout ext/psych/depend` + when "fiddle" + rm_rf(%w[ext/fiddle test/fiddle]) + cp_r("#{upstream}/ext/fiddle", "ext") + cp_r("#{upstream}/lib", "ext/fiddle") + cp_r("#{upstream}/test/fiddle", "test") + cp_r("#{upstream}/fiddle.gemspec", "ext/fiddle") + `git checkout ext/fiddle/depend` + rm_rf(%w[ext/fiddle/lib/fiddle.{bundle,so}]) + when "stringio" + rm_rf(%w[ext/stringio test/stringio]) + cp_r("#{upstream}/ext/stringio", "ext") + cp_r("#{upstream}/test/stringio", "test") + cp_r("#{upstream}/stringio.gemspec", "ext/stringio") + `git checkout ext/stringio/depend ext/stringio/README.md` + when "io-console" + rm_rf(%w[ext/io/console test/io/console]) + cp_r("#{upstream}/ext/io/console", "ext/io") + cp_r("#{upstream}/test/io/console", "test/io") + mkdir_p("ext/io/console/lib") + cp_r("#{upstream}/lib/io/console", "ext/io/console/lib") + rm_rf("ext/io/console/lib/console/ffi") + cp_r("#{upstream}/io-console.gemspec", "ext/io/console") + `git checkout ext/io/console/depend` + when "io-nonblock" + rm_rf(%w[ext/io/nonblock test/io/nonblock]) + cp_r("#{upstream}/ext/io/nonblock", "ext/io") + cp_r("#{upstream}/test/io/nonblock", "test/io") + cp_r("#{upstream}/io-nonblock.gemspec", "ext/io/nonblock") + `git checkout ext/io/nonblock/depend` + when "io-wait" + rm_rf(%w[ext/io/wait test/io/wait]) + cp_r("#{upstream}/ext/io/wait", "ext/io") + cp_r("#{upstream}/test/io/wait", "test/io") + cp_r("#{upstream}/io-wait.gemspec", "ext/io/wait") + `git checkout ext/io/wait/depend` + when "etc" + rm_rf(%w[ext/etc test/etc]) + cp_r("#{upstream}/ext/etc", "ext") + cp_r("#{upstream}/test/etc", "test") + cp_r("#{upstream}/etc.gemspec", "ext/etc") + `git checkout ext/etc/depend` + when "date" + rm_rf(%w[ext/date test/date]) + cp_r("#{upstream}/ext/date", "ext") + cp_r("#{upstream}/lib", "ext/date") + cp_r("#{upstream}/test/date", "test") + cp_r("#{upstream}/date.gemspec", "ext/date") + `git checkout ext/date/depend` + rm_rf(["ext/date/lib/date_core.bundle"]) + when "zlib" + rm_rf(%w[ext/zlib test/zlib]) + cp_r("#{upstream}/ext/zlib", "ext") + cp_r("#{upstream}/test/zlib", "test") + cp_r("#{upstream}/zlib.gemspec", "ext/zlib") + `git checkout ext/zlib/depend` + when "fcntl" + rm_rf(%w[ext/fcntl]) + cp_r("#{upstream}/ext/fcntl", "ext") + cp_r("#{upstream}/fcntl.gemspec", "ext/fcntl") + `git checkout ext/fcntl/depend` + when "strscan" + rm_rf(%w[ext/strscan test/strscan]) + cp_r("#{upstream}/ext/strscan", "ext") + cp_r("#{upstream}/test/strscan", "test") + cp_r("#{upstream}/strscan.gemspec", "ext/strscan") + rm_rf(%w["ext/strscan/regenc.h ext/strscan/regint.h"]) + `git checkout ext/strscan/depend` + when "racc" + rm_rf(%w[lib/racc lib/racc.rb ext/racc test/racc]) + cp_r(Dir.glob("#{upstream}/lib/racc*"), "lib") + mkdir_p("ext/racc/cparse") + cp_r(Dir.glob("#{upstream}/ext/racc/cparse/*"), "ext/racc/cparse") + cp_r("#{upstream}/test", "test/racc") + cp_r("#{upstream}/racc.gemspec", "lib/racc") + rm_rf("test/racc/lib") + rm_rf("lib/racc/cparse-jruby.jar") + `git checkout ext/racc/cparse/README ext/racc/cparse/depend` + when "cgi" + rm_rf(%w[lib/cgi.rb lib/cgi ext/cgi test/cgi]) + cp_r("#{upstream}/ext/cgi", "ext") + cp_r("#{upstream}/lib", ".") + cp_r("#{upstream}/test/cgi", "test") + cp_r("#{upstream}/cgi.gemspec", "lib/cgi") + `git checkout ext/cgi/escape/depend` + when "openssl" + rm_rf(%w[ext/openssl test/openssl]) + cp_r("#{upstream}/ext/openssl", "ext") + cp_r("#{upstream}/lib", "ext/openssl") + cp_r("#{upstream}/test/openssl", "test") + rm_rf("test/openssl/envutil.rb") + cp_r("#{upstream}/openssl.gemspec", "ext/openssl") + cp_r("#{upstream}/History.md", "ext/openssl") + `git checkout ext/openssl/depend` + when "net-protocol" + rm_rf(%w[lib/net/protocol.rb lib/net/net-protocol.gemspec test/net/protocol]) + cp_r("#{upstream}/lib/net/protocol.rb", "lib/net") + cp_r("#{upstream}/test/net/protocol", "test/net") + cp_r("#{upstream}/net-protocol.gemspec", "lib/net") + when "net-http" + rm_rf(%w[lib/net/http.rb lib/net/http test/net/http]) + cp_r("#{upstream}/lib/net/http.rb", "lib/net") + cp_r("#{upstream}/lib/net/http", "lib/net") + cp_r("#{upstream}/test/net/http", "test/net") + cp_r("#{upstream}/net-http.gemspec", "lib/net/http") + when "readline-ext" + rm_rf(%w[ext/readline test/readline]) + cp_r("#{upstream}/ext/readline", "ext") + cp_r("#{upstream}/test/readline", "test") + cp_r("#{upstream}/readline-ext.gemspec", "ext/readline") + `git checkout ext/readline/depend` + when "did_you_mean" + rm_rf(%w[lib/did_you_mean lib/did_you_mean.rb test/did_you_mean]) + cp_r(Dir.glob("#{upstream}/lib/did_you_mean*"), "lib") + cp_r("#{upstream}/did_you_mean.gemspec", "lib/did_you_mean") + cp_r("#{upstream}/test", "test/did_you_mean") + rm_rf(%w[test/did_you_mean/tree_spell/test_explore.rb]) + when "erb" + rm_rf(%w[lib/erb* test/erb libexec/erb]) + cp_r("#{upstream}/lib/erb.rb", "lib") + cp_r("#{upstream}/test/erb", "test") + cp_r("#{upstream}/erb.gemspec", "lib") + cp_r("#{upstream}/libexec/erb", "libexec") + when "nkf" + rm_rf(%w[ext/nkf test/nkf]) + cp_r("#{upstream}/ext/nkf", "ext") + cp_r("#{upstream}/lib", "ext/nkf") + cp_r("#{upstream}/test/nkf", "test") + cp_r("#{upstream}/nkf.gemspec", "ext/nkf") + `git checkout ext/nkf/depend` + when "syslog" + rm_rf(%w[ext/syslog test/syslog test/test_syslog.rb]) + cp_r("#{upstream}/ext/syslog", "ext") + cp_r("#{upstream}/lib", "ext/syslog") + cp_r("#{upstream}/test/syslog", "test") + cp_r("#{upstream}/test/test_syslog.rb", "test") + cp_r("#{upstream}/syslog.gemspec", "ext/syslog") + `git checkout ext/syslog/depend` + when "bigdecimal" + rm_rf(%w[ext/bigdecimal test/bigdecimal]) + cp_r("#{upstream}/ext/bigdecimal", "ext") + cp_r("#{upstream}/sample", "ext/bigdecimal") + cp_r("#{upstream}/lib", "ext/bigdecimal") + cp_r("#{upstream}/test/bigdecimal", "test") + cp_r("#{upstream}/bigdecimal.gemspec", "ext/bigdecimal") + `git checkout ext/bigdecimal/depend` + when "pathname" + rm_rf(%w[ext/pathname test/pathname]) + cp_r("#{upstream}/ext/pathname", "ext") + cp_r("#{upstream}/test/pathname", "test") + cp_r("#{upstream}/lib", "ext/pathname") + cp_r("#{upstream}/pathname.gemspec", "ext/pathname") + `git checkout ext/pathname/depend` + when "digest" + rm_rf(%w[ext/digest test/digest]) + cp_r("#{upstream}/ext/digest", "ext") + mkdir_p("ext/digest/lib/digest") + cp_r("#{upstream}/lib/digest.rb", "ext/digest/lib/") + cp_r("#{upstream}/lib/digest/version.rb", "ext/digest/lib/digest/") + mkdir_p("ext/digest/sha2/lib") + cp_r("#{upstream}/lib/digest/sha2.rb", "ext/digest/sha2/lib") + move("ext/digest/lib/digest/sha2", "ext/digest/sha2/lib") + cp_r("#{upstream}/test/digest", "test") + cp_r("#{upstream}/digest.gemspec", "ext/digest") + `git checkout ext/digest/depend ext/digest/*/depend` + when "set" + sync_lib gem, upstream + cp_r("#{upstream}/test", ".") + when "optparse" + sync_lib gem, upstream + rm_rf(%w[doc/optparse]) + mkdir_p("doc/optparse") + cp_r("#{upstream}/doc/optparse", "doc") + when "error_highlight" + rm_rf(%w[lib/error_highlight lib/error_highlight.rb test/error_highlight]) + cp_r(Dir.glob("#{upstream}/lib/error_highlight*"), "lib") + cp_r("#{upstream}/error_highlight.gemspec", "lib/error_highlight") + cp_r("#{upstream}/test", "test/error_highlight") + when "win32ole" + sync_lib gem, upstream + rm_rf(%w[ext/win32ole/lib]) + Dir.mkdir(*%w[ext/win32ole/lib]) + move("lib/win32ole/win32ole.gemspec", "ext/win32ole") + move(Dir.glob("lib/win32ole*"), "ext/win32ole/lib") + when "open3" + sync_lib gem, upstream + rm_rf("lib/open3/jruby_windows.rb") + else + sync_lib gem, upstream + end +end + +IGNORE_FILE_PATTERN = + /\A(?:[A-Z]\w*\.(?:md|txt) + |[^\/]+\.yml + |\.git.* + |[A-Z]\w+file + |COPYING + |rakelib\/ + )\z/x + +def message_filter(repo, sha) + log = STDIN.read + log.delete!("\r") + url = "https://github.com/#{repo}" + print "[#{repo}] ", log.gsub(/\b(?i:fix) +\K#(?=\d+\b)|\(\K#(?=\d+\))|\bGH-(?=\d+\b)/) { + "#{url}/pull/" + }.gsub(%r{(?<![-\[\](){}\w@/])(?:(\w+(?:-\w+)*/\w+(?:-\w+)*)@)?(\h{10,40})\b}) {|c| + "https://github.com/#{$1 || repo}/commit/#{$2[0,12]}" + }.sub(/\s*(?=(?i:\nCo-authored-by:.*)*\Z)/) { + "\n\n" "#{url}/commit/#{sha[0,10]}\n" + } +end + +# NOTE: This method is also used by ruby-commit-hook/bin/update-default-gem.sh +# @param gem [String] A gem name, also used as a git remote name. REPOSITORIES converts it to the appropriate GitHub repository. +# @param ranges [Array<String>] "before..after". Note that it will NOT sync "before" (but commits after that). +# @param edit [TrueClass] Set true if you want to resolve conflicts. Obviously, update-default-gem.sh doesn't use this. +def sync_default_gems_with_commits(gem, ranges, edit: nil) + repo = REPOSITORIES[gem.to_sym] + puts "Sync #{repo} with commit history." + + IO.popen(%W"git remote") do |f| + unless f.read.split.include?(gem) + `git remote add #{gem} git@github.com:#{repo}.git` + end + end + system(*%W"git fetch --no-tags #{gem}") + + if ranges == true + pattern = "https://github\.com/#{Regexp.quote(repo)}/commit/([0-9a-f]+)$" + log = IO.popen(%W"git log -E --grep=#{pattern} -n1 --format=%B", &:read) + ranges = ["#{log[%r[#{pattern}\n\s*(?i:co-authored-by:.*)*\s*\Z], 1]}..#{gem}/master"] + end + + commits = ranges.flat_map do |range| + unless range.include?("..") + range = "#{range}~1..#{range}" + end + + IO.popen(%W"git log --format=%H,%s #{range} --") do |f| + f.read.split("\n").reverse.map{|commit| commit.split(',', 2)} + end + end + + # Ignore Merge commit and insufficiency commit for ruby core repository. + commits.delete_if do |sha, subject| + files = IO.popen(%W"git diff-tree --no-commit-id --name-only -r #{sha}", &:readlines) + subject =~ /^Merge/ || subject =~ /^Auto Merge/ || files.all?{|file| file =~ IGNORE_FILE_PATTERN} + end + + if commits.empty? + puts "No commits to pick" + return true + end + + puts "Try to pick these commits:" + puts commits.map{|commit| commit.join(": ")} + puts "----" + + failed_commits = [] + + ENV["FILTER_BRANCH_SQUELCH_WARNING"] = "1" + + require 'shellwords' + filter = [ + ENV.fetch('RUBY', 'ruby').shellescape, + File.realpath(__FILE__).shellescape, + "--message-filter", + ] + commits.each do |sha, subject| + puts "Pick #{sha} from #{repo}." + + skipped = false + result = IO.popen(%W"git cherry-pick #{sha}", &:read) + if result =~ /nothing\ to\ commit/ + `git reset` + skipped = true + puts "Skip empty commit #{sha}" + end + next if skipped + + if result.empty? + skipped = true + elsif /^CONFLICT/ =~ result + result = IO.popen(%W"git status --porcelain", &:readlines).each(&:chomp!) + result.map! {|line| line[/^.U (.*)/, 1]} + result.compact! + ignore, conflict = result.partition {|name| IGNORE_FILE_PATTERN =~ name} + unless ignore.empty? + system(*%W"git reset HEAD --", *ignore) + File.unlink(*ignore) + ignore = IO.popen(%W"git status --porcelain" + ignore, &:readlines).map! {|line| line[/^.. (.*)/, 1]} + system(*%W"git checkout HEAD --", *ignore) unless ignore.empty? + end + unless conflict.empty? + if edit + case + when (editor = ENV["GIT_EDITOR"] and !editor.empty?) + when (editor = `git config core.editor` and (editor.chomp!; !editor.empty?)) + end + if editor + system([editor, conflict].join(' ')) + end + end + end + skipped = !system({"GIT_EDITOR"=>"true"}, *%W"git cherry-pick --no-edit --continue") + end + + if skipped + failed_commits << sha + `git reset` && `git checkout .` && `git clean -fd` + puts "Failed to pick #{sha}" + next + end + + puts "Update commit message: #{sha}" + + IO.popen(%W[git filter-branch -f --msg-filter #{[filter, repo, sha].join(' ')} -- HEAD~1..HEAD], &:read) + unless $?.success? + puts "Failed to modify commit message of #{sha}" + break + end + end + + unless failed_commits.empty? + puts "---- failed commits ----" + puts failed_commits + return false + end + return true +end + +def sync_lib(repo, upstream = nil) + unless upstream and File.directory?(upstream) or File.directory?(upstream = "../#{repo}") + abort %[Expected '#{upstream}' \(#{File.expand_path("#{upstream}")}\) to be a directory, but it wasn't.] + end + rm_rf(["lib/#{repo}.rb", "lib/#{repo}/*", "test/test_#{repo}.rb"]) + cp_r(Dir.glob("#{upstream}/lib/*"), "lib") + tests = if File.directory?("test/#{repo}") + "test/#{repo}" + else + "test/test_#{repo}.rb" + end + cp_r("#{upstream}/#{tests}", "test") if File.exist?("#{upstream}/#{tests}") + gemspec = if File.directory?("lib/#{repo}") + "lib/#{repo}/#{repo}.gemspec" + else + "lib/#{repo}.gemspec" + end + cp_r("#{upstream}/#{repo}.gemspec", "#{gemspec}") +end + +def update_default_gems(gem, release: false) + + author, repository = REPOSITORIES[gem.to_sym].split('/') + + puts "Update #{author}/#{repository}" + + unless File.exist?("../../#{author}/#{repository}") + mkdir_p("../../#{author}") + `git clone git@github.com:#{author}/#{repository}.git ../../#{author}/#{repository}` + end + + Dir.chdir("../../#{author}/#{repository}") do + unless `git remote`.match(/ruby\-core/) + `git remote add ruby-core git@github.com:ruby/ruby.git` + end + `git fetch ruby-core master --no-tags` + unless `git branch`.match(/ruby\-core/) + `git checkout ruby-core/master` + `git branch ruby-core` + end + `git checkout ruby-core` + `git rebase ruby-core/master` + `git fetch origin --tags` + + if release + last_release = `git tag`.chomp.split.delete_if{|v| v =~ /pre|beta/ }.last + `git checkout #{last_release}` + else + `git checkout master` + `git rebase origin/master` + end + end +end + +case ARGV[0] +when "up" + if ARGV[1] + update_default_gems(ARGV[1]) + else + REPOSITORIES.keys.each{|gem| update_default_gems(gem.to_s)} + end +when "all" + if ARGV[1] == "release" + REPOSITORIES.keys.each do |gem| + update_default_gems(gem.to_s, release: true) + sync_default_gems(gem.to_s) + end + else + REPOSITORIES.keys.each{|gem| sync_default_gems(gem.to_s)} + end +when "list" + ARGV.shift + pattern = Regexp.new(ARGV.join('|')) + REPOSITORIES.each_pair do |name, gem| + next unless pattern =~ name or pattern =~ gem + printf "%-15s https://github.com/%s\n", name, gem + end +when "--message-filter" + ARGV.shift + abort unless ARGV.size == 2 + message_filter(*ARGV) + exit +when nil, "-h", "--help" + puts <<-HELP +\e[1mSync with upstream code of default libraries\e[0m + +\e[1mImport a default library through `git clone` and `cp -rf` (git commits are lost)\e[0m + ruby #$0 rubygems + +\e[1mPick a single commit from the upstream repository\e[0m + ruby #$0 rubygems 97e9768612 + +\e[1mPick a commit range from the upstream repository\e[0m + ruby #$0 rubygems 97e9768612..9e53702832 + +\e[1mList known libraries\e[0m + ruby #$0 list + +\e[1mList known libraries matching with patterns\e[0m + ruby #$0 list read + HELP + + exit +else + while /\A-/ =~ ARGV[0] + case ARGV[0] + when "-e" + edit = true + ARGV.shift + when "-a" + auto = true + ARGV.shift + else + $stderr.puts "Unknown command line option: #{ARGV[0]}" + exit 1 + end + end + gem = ARGV.shift + if ARGV[0] + exit sync_default_gems_with_commits(gem, ARGV, edit: edit) + elsif auto + exit sync_default_gems_with_commits(gem, true, edit: edit) + else + sync_default_gems(gem) + end +end diff --git a/tool/test-bundled-gems.rb b/tool/test-bundled-gems.rb new file mode 100644 index 0000000000..79c6b61493 --- /dev/null +++ b/tool/test-bundled-gems.rb @@ -0,0 +1,116 @@ +require 'rbconfig' +require 'timeout' +require 'fileutils' + +ENV.delete("GNUMAKEFLAGS") + +github_actions = ENV["GITHUB_ACTIONS"] == "true" + +allowed_failures = ENV['TEST_BUNDLED_GEMS_ALLOW_FAILURES'] || '' +allowed_failures = allowed_failures.split(',').reject(&:empty?) + +ENV["GEM_PATH"] = [File.realpath('.bundle'), File.realpath('../.bundle', __dir__)].join(File::PATH_SEPARATOR) + +rake = File.realpath("../../.bundle/bin/rake", __FILE__) +gem_dir = File.realpath('../../gems', __FILE__) +rubylib = [gem_dir+'/lib', ENV["RUBYLIB"]].compact.join(File::PATH_SEPARATOR) +exit_code = 0 +ruby = ENV['RUBY'] || RbConfig.ruby +failed = [] +File.foreach("#{gem_dir}/bundled_gems") do |line| + next if /^\s*(?:#|$)/ =~ line + gem = line.split.first + next if ARGV.any? {|pat| !File.fnmatch?(pat, gem)} + puts "#{github_actions ? "##[group]" : "\n"}Testing the #{gem} gem" + + test_command = "#{ruby} -C #{gem_dir}/src/#{gem} #{rake} test" + envs = {} + first_timeout = 600 # 10min + + toplib = gem + case gem + when "typeprof" + + when "rbs" + # TODO: We should skip test file instead of test class/methods + skip_test_files = %w[ + test/stdlib/Prime_test.rb + ] + + skip_test_files.each do |file| + path = "#{gem_dir}/src/#{gem}/#{file}" + File.unlink(path) if File.exist?(path) + end + + test_command << " stdlib_test validate" + first_timeout *= 3 + + when "minitest" + # Tentatively exclude some tests that conflict with error_highlight + # https://github.com/seattlerb/minitest/pull/880 + test_command << " 'TESTOPTS=-e /test_stub_value_block_args_5__break_if_not_passed|test_no_method_error_on_unexpected_methods/'" + + when "debug" + # Since debug gem requires debug.so in child processes without + # acitvating the gem, we preset necessary paths in RUBYLIB + # environment variable. + load_path = true + + # disable remote test in debug.gem on macOS + if /darwin/ =~ RUBY_PLATFORM + envs["RUBY_DEBUG_TEST_NO_REMOTE"] = "1" + end + + when "test-unit" + test_command = "#{ruby} -C #{gem_dir}/src/#{gem} test/run-test.rb" + + when /\Anet-/ + toplib = gem.tr("-", "/") + + end + + if load_path + libs = IO.popen([ruby, "-e", "old = $:.dup; require '#{toplib}'; puts $:-old"], &:read) + next unless $?.success? + puts libs + ENV["RUBYLIB"] = [libs.split("\n"), rubylib].join(File::PATH_SEPARATOR) + else + ENV["RUBYLIB"] = rubylib + end + + print "[command]" if github_actions + puts test_command + pid = Process.spawn(envs, test_command, "#{/mingw|mswin/ =~ RUBY_PLATFORM ? 'new_' : ''}pgroup": true) + {nil => first_timeout, INT: 30, TERM: 10, KILL: nil}.each do |sig, sec| + if sig + puts "Sending #{sig} signal" + Process.kill("-#{sig}", pid) + end + begin + break Timeout.timeout(sec) {Process.wait(pid)} + rescue Timeout::Error + end + rescue Interrupt + exit_code = Signal.list["INT"] + Process.kill("-KILL", pid) + Process.wait(pid) + break + end + + unless $?.success? + + puts "Tests failed " + + ($?.signaled? ? "by SIG#{Signal.signame($?.termsig)}" : + "with exit code #{$?.exitstatus}") + if allowed_failures.include?(gem) + puts "Ignoring test failures for #{gem} due to \$TEST_BUNDLED_GEMS_ALLOW_FAILURES" + else + failed << gem + exit_code = $?.exitstatus if $?.exitstatus + end + end + print "##[endgroup]\n" if github_actions +end + +puts "Failed gems: #{failed.join(', ')}" unless failed.empty? +exit exit_code diff --git a/tool/test-coverage.rb b/tool/test-coverage.rb new file mode 100644 index 0000000000..4950bc65de --- /dev/null +++ b/tool/test-coverage.rb @@ -0,0 +1,118 @@ +require "coverage" + +Coverage.start(lines: true, branches: true, methods: true) + +TEST_COVERAGE_DATA_FILE = "test-coverage.dat" + +def merge_coverage_data(res1, res2) + res1.each do |path, cov1| + cov2 = res2[path] + if cov2 + cov1[:lines].each_with_index do |count1, i| + next unless count1 + add_count(cov2[:lines], i, count1) + end + cov1[:branches].each do |base_key, targets1| + if cov2[:branches][base_key] + targets1.each do |target_key, count1| + add_count(cov2[:branches][base_key], target_key, count1) + end + else + cov2[:branches][base_key] = targets1 + end + end + cov1[:methods].each do |key, count1| + add_count(cov2[:methods], key, count1) + end + else + res2[path] = cov1 + end + end + res2 +end + +def add_count(h, key, count) + if h[key] + h[key] += count + else + h[key] = count + end +end + +def save_coverage_data(res1) + res1.each do |_path, cov| + if cov[:methods] + h = {} + cov[:methods].each do |(klass, *key), count| + h[[klass.name, *key]] = count + end + cov[:methods].replace h + end + end + File.open(TEST_COVERAGE_DATA_FILE, File::RDWR | File::CREAT | File::BINARY) do |f| + f.flock(File::LOCK_EX) + s = f.read + res2 = s.size > 0 ? Marshal.load(s) : {} + res1 = merge_coverage_data(res1, res2) + f.rewind + f << Marshal.dump(res2) + f.flush + f.truncate(f.pos) + end +end + +def invoke_simplecov_formatter + %w[doclie simplecov-html simplecov].each do |f| + $LOAD_PATH.unshift "#{__dir__}/../coverage/#{f}/lib" + end + + require "simplecov" + res = Marshal.load(File.binread(TEST_COVERAGE_DATA_FILE)) + simplecov_result = {} + base_dir = File.dirname(__dir__) + cur_dir = Dir.pwd + + res.each do |path, cov| + next unless path.start_with?(base_dir) || path.start_with?(cur_dir) + next if path.start_with?(File.join(base_dir, "test")) + simplecov_result[path] = cov[:lines] + end + + a, b = base_dir, cur_dir + until a == b + if a.size > b.size + a = File.dirname(a) + else + b = File.dirname(b) + end + end + root_dir = a + + SimpleCov.configure do + root(root_dir) + coverage_dir(File.join(cur_dir, "coverage")) + end + res = SimpleCov::Result.new(simplecov_result) + res.command_name = "Ruby's `make test-all`" + SimpleCov::Formatter::HTMLFormatter.new.format(res) +end + +pid = $$ +pwd = Dir.pwd + +at_exit do + exit_exc = $! + + Dir.chdir(pwd) do + save_coverage_data(Coverage.result) + if pid == $$ + begin + nil while Process.waitpid(-1) + rescue Errno::ECHILD + invoke_simplecov_formatter + end + end + end + + raise exit_exc if exit_exc +end diff --git a/tool/test/runner.rb b/tool/test/runner.rb new file mode 100644 index 0000000000..c629943090 --- /dev/null +++ b/tool/test/runner.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true +require 'rbconfig' + +$LOAD_PATH.unshift File.expand_path("../lib", __dir__) + +require 'test/unit' + +require "profile_test_all" if ENV.key?('RUBY_TEST_ALL_PROFILE') +require "tracepointchecker" +require "zombie_hunter" +require "iseq_loader_checker" +require "gc_checker" +require_relative "../test-coverage.rb" if ENV.key?('COVERAGE') + +case $0 +when __FILE__ + dir = __dir__ +when "-e" + # No default directory +else + dir = File.expand_path("..", $0) +end +exit Test::Unit::AutoRunner.run(true, dir) diff --git a/tool/test/test_jisx0208.rb b/tool/test/test_jisx0208.rb new file mode 100644 index 0000000000..98f216ff4f --- /dev/null +++ b/tool/test/test_jisx0208.rb @@ -0,0 +1,40 @@ +require 'test/unit' + +require_relative '../lib/jisx0208' + +class Test_JISX0208_Char < Test::Unit::TestCase + def test_create_with_row_cell + assert_equal JISX0208::Char.new(0x2121), JISX0208::Char.new(1, 1) + end + + def test_succ + assert_equal JISX0208::Char.new(0x2221), JISX0208::Char.new(0x217e).succ + assert_equal JISX0208::Char.new(2, 1), JISX0208::Char.new(1, 94).succ + assert_equal JISX0208::Char.new(0x7f21), JISX0208::Char.new(0x7e7e).succ + end + + def test_to_shift_jis + assert_equal 0x895C, JISX0208::Char.new(0x313D).to_sjis + assert_equal 0x895C, JISX0208::Char.from_sjis(0x895C).to_sjis + assert_equal 0xF3DE, JISX0208::Char.from_sjis(0xF3DE).to_sjis + assert_equal 0xFC40, JISX0208::Char.from_sjis(0xFC40).to_sjis + end + + def test_from_sjis + assert_raise(ArgumentError) { JISX0208::Char.from_sjis(-1) } + assert_raise(ArgumentError) { JISX0208::Char.from_sjis(0x10000) } + assert_nothing_raised { JISX0208::Char.from_sjis(0x8140) } + assert_nothing_raised { JISX0208::Char.from_sjis(0xFCFC) } + assert_equal JISX0208::Char.new(0x313D), JISX0208::Char.from_sjis(0x895C) + end + + def test_row + assert_equal 1, JISX0208::Char.new(0x2121).row + assert_equal 94, JISX0208::Char.new(0x7E7E).row + end + + def test_cell + assert_equal 1, JISX0208::Char.new(0x2121).cell + assert_equal 94, JISX0208::Char.new(0x7E7E).cell + end +end diff --git a/tool/test/testunit/metametameta.rb b/tool/test/testunit/metametameta.rb new file mode 100644 index 0000000000..e494038939 --- /dev/null +++ b/tool/test/testunit/metametameta.rb @@ -0,0 +1,70 @@ +# encoding: utf-8 +# frozen_string_literal: false + +require 'tempfile' +require 'stringio' + +class Test::Unit::TestCase + def clean s + s.gsub(/^ {6}/, '') + end +end + +class MetaMetaMetaTestCase < Test::Unit::TestCase + def assert_report expected, flags = %w[--seed 42] + header = clean <<-EOM + Run options: #{flags.map { |s| s =~ /\|/ ? s.inspect : s }.join " "} + + # Running tests: + + EOM + + with_output do + @tu.run flags + end + + output = @output.string.dup + output.sub!(/Finished tests in .*/, "Finished tests in 0.00") + output.sub!(/Loaded suite .*/, 'Loaded suite blah') + + output.gsub!(/ = \d+.\d\d s = /, ' = 0.00 s = ') + output.gsub!(/0x[A-Fa-f0-9]+/, '0xXXX') + + if windows? then + output.gsub!(/\[(?:[A-Za-z]:)?[^\]:]+:\d+\]/, '[FILE:LINE]') + output.gsub!(/^(\s+)(?:[A-Za-z]:)?[^:]+:\d+:in/, '\1FILE:LINE:in') + else + output.gsub!(/\[[^\]:]+:\d+\]/, '[FILE:LINE]') + output.gsub!(/^(\s+)[^:]+:\d+:in/, '\1FILE:LINE:in') + end + + assert_equal header + expected, output + end + + def setup + super + srand 42 + Test::Unit::TestCase.reset + @tu = Test::Unit::Runner.new + + Test::Unit::Runner.runner = nil # protect the outer runner from the inner tests + end + + def teardown + super + end + + def with_output + synchronize do + begin + save = Test::Unit::Runner.output + @output = StringIO.new("") + Test::Unit::Runner.output = @output + + yield + ensure + Test::Unit::Runner.output = save + end + end + end +end diff --git a/tool/test/testunit/test4test_hideskip.rb b/tool/test/testunit/test4test_hideskip.rb new file mode 100644 index 0000000000..410bffc13c --- /dev/null +++ b/tool/test/testunit/test4test_hideskip.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: false +$LOAD_PATH.unshift "#{File.dirname(__FILE__)}/../../lib" + +require 'test/unit' + +class TestForTestHideSkip < Test::Unit::TestCase + def test_skip + skip "do nothing" + end +end diff --git a/tool/test/testunit/test4test_redefinition.rb b/tool/test/testunit/test4test_redefinition.rb new file mode 100644 index 0000000000..ad3c5e7113 --- /dev/null +++ b/tool/test/testunit/test4test_redefinition.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: false +$LOAD_PATH.unshift "#{File.dirname(__FILE__)}/../../lib" + +require 'test/unit' + +class TestForTestRedefinition < Test::Unit::TestCase + def test_redefinition + skip "do nothing (1)" + end + + def test_redefinition + skip "do nothing (2)" + end +end diff --git a/tool/test/testunit/test4test_sorting.rb b/tool/test/testunit/test4test_sorting.rb new file mode 100644 index 0000000000..698c875b79 --- /dev/null +++ b/tool/test/testunit/test4test_sorting.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: false +$LOAD_PATH.unshift "#{File.dirname(__FILE__)}/../../lib" + +require 'test/unit' + +class TestForTestHideSkip < Test::Unit::TestCase + def test_c + skip "do nothing" + end + + def test_b + assert_equal true, false + end + + def test_a + raise + end +end diff --git a/tool/test/testunit/test_assertion.rb b/tool/test/testunit/test_assertion.rb new file mode 100644 index 0000000000..8c83b447a7 --- /dev/null +++ b/tool/test/testunit/test_assertion.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: false +require 'test/unit' +class TestAssertion < Test::Unit::TestCase + def test_wrong_assertion + error, line = assert_raise(ArgumentError) {assert(true, true)}, __LINE__ + assert_match(/assertion message must be String or Proc, but TrueClass was given/, error.message) + assert_match(/\A#{Regexp.quote(__FILE__)}:#{line}:/, error.backtrace[0]) + end + + def test_timeout_separately + assert_raise(Timeout::Error) do + assert_separately([], <<~"end;", timeout: 0.1) + sleep + end; + end + end + + def return_in_assert_raise + assert_raise(RuntimeError) do + return + end + end + + def test_assert_raise + assert_raise(Test::Unit::AssertionFailedError) do + return_in_assert_raise + end + end +end diff --git a/tool/test/testunit/test_hideskip.rb b/tool/test/testunit/test_hideskip.rb new file mode 100644 index 0000000000..13d887189e --- /dev/null +++ b/tool/test/testunit/test_hideskip.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: false +require 'test/unit' + +class TestHideSkip < Test::Unit::TestCase + def test_hideskip + assert_not_match(/^ *1\) Skipped/, hideskip) + assert_match(/^ *1\) Skipped/, hideskip("--show-skip")) + output = hideskip("--hide-skip") + output.gsub!(/Successful MJIT finish\n/, '') if defined?(RubyVM::MJIT) && RubyVM::MJIT.enabled? + assert_match(/assertions\/s.\n+1 tests, 0 assertions, 0 failures, 0 errors, 1 skips/, output) + end + + private + + def hideskip(*args) + IO.popen([*@options[:ruby], "#{File.dirname(__FILE__)}/test4test_hideskip.rb", + "--verbose", *args], err: [:child, :out]) {|f| + f.read + } + end +end diff --git a/tool/test/testunit/test_minitest_unit.rb b/tool/test/testunit/test_minitest_unit.rb new file mode 100644 index 0000000000..5941392fa0 --- /dev/null +++ b/tool/test/testunit/test_minitest_unit.rb @@ -0,0 +1,1474 @@ +# encoding: utf-8 +# frozen_string_literal: false + +require 'pathname' +require_relative 'metametameta' + +module MyModule; end +class AnError < StandardError; include MyModule; end +class ImmutableString < String; def inspect; super.freeze; end; end + +class TestMiniTestUnit < MetaMetaMetaTestCase + pwd = Pathname.new File.expand_path Dir.pwd + basedir = Pathname.new(File.expand_path "lib/test") + basedir = basedir.relative_path_from(pwd).to_s + MINITEST_BASE_DIR = basedir[/\A\./] ? basedir : "./#{basedir}" + BT_MIDDLE = ["#{MINITEST_BASE_DIR}/test.rb:161:in `each'", + "#{MINITEST_BASE_DIR}/test.rb:158:in `each'", + "#{MINITEST_BASE_DIR}/test.rb:139:in `run'", + "#{MINITEST_BASE_DIR}/test.rb:106:in `run'"] + + def test_class_puke_with_assertion_failed + exception = Test::Unit::AssertionFailedError.new "Oh no!" + exception.set_backtrace ["unhappy"] + assert_equal 'F', @tu.puke('SomeClass', 'method_name', exception) + assert_equal 1, @tu.failures + assert_match(/^Failure.*Oh no!/m, @tu.report.first) + assert_match("SomeClass#method_name [unhappy]", @tu.report.first) + end + + def test_class_puke_with_assertion_failed_and_long_backtrace + bt = (["test/test_some_class.rb:615:in `method_name'", + "#{MINITEST_BASE_DIR}/unit.rb:140:in `assert_raise'", + "test/test_some_class.rb:615:in `each'", + "test/test_some_class.rb:614:in `test_method_name'", + "#{MINITEST_BASE_DIR}/test.rb:165:in `__send__'"] + + BT_MIDDLE + + ["#{MINITEST_BASE_DIR}/test.rb:29"]) + bt = util_expand_bt bt + + ex_location = util_expand_bt(["test/test_some_class.rb:615"]).first + + exception = Test::Unit::AssertionFailedError.new "Oh no!" + exception.set_backtrace bt + assert_equal 'F', @tu.puke('TestSomeClass', 'test_method_name', exception) + assert_equal 1, @tu.failures + assert_match(/^Failure.*Oh no!/m, @tu.report.first) + assert_match("TestSomeClass#test_method_name [#{ex_location}]", @tu.report.first) + end + + def test_class_puke_with_assertion_failed_and_user_defined_assertions + bt = (["lib/test/my/util.rb:16:in `another_method_name'", + "#{MINITEST_BASE_DIR}/unit.rb:140:in `assert_raise'", + "lib/test/my/util.rb:15:in `block in assert_something'", + "lib/test/my/util.rb:14:in `each'", + "lib/test/my/util.rb:14:in `assert_something'", + "test/test_some_class.rb:615:in `each'", + "test/test_some_class.rb:614:in `test_method_name'", + "#{MINITEST_BASE_DIR}/test.rb:165:in `__send__'"] + + BT_MIDDLE + + ["#{MINITEST_BASE_DIR}/test.rb:29"]) + bt = util_expand_bt bt + + ex_location = util_expand_bt(["test/test_some_class.rb:615"]).first + + exception = Test::Unit::AssertionFailedError.new "Oh no!" + exception.set_backtrace bt + assert_equal 'F', @tu.puke('TestSomeClass', 'test_method_name', exception) + assert_equal 1, @tu.failures + assert_match(/^Failure.*Oh no!/m, @tu.report.first) + assert_match("TestSomeClass#test_method_name [#{ex_location}]", @tu.report.first) + end + + def test_class_puke_with_failure_and_flunk_in_backtrace + exception = begin + Test::Unit::TestCase.new('fake tc').flunk + rescue Test::Unit::AssertionFailedError => failure + failure + end + assert_equal 'F', @tu.puke('SomeClass', 'method_name', exception) + refute @tu.report.any?{|line| line =~ /in .flunk/} + end + + def test_class_puke_with_flunk_and_user_defined_assertions + bt = (["lib/test/my/util.rb:16:in `flunk'", + "#{MINITEST_BASE_DIR}/unit.rb:140:in `assert_raise'", + "lib/test/my/util.rb:15:in `block in assert_something'", + "lib/test/my/util.rb:14:in `each'", + "lib/test/my/util.rb:14:in `assert_something'", + "test/test_some_class.rb:615:in `each'", + "test/test_some_class.rb:614:in `test_method_name'", + "#{MINITEST_BASE_DIR}/test.rb:165:in `__send__'"] + + BT_MIDDLE + + ["#{MINITEST_BASE_DIR}/test.rb:29"]) + bt = util_expand_bt bt + + ex_location = util_expand_bt(["test/test_some_class.rb:615"]).first + + exception = Test::Unit::AssertionFailedError.new "Oh no!" + exception.set_backtrace bt + assert_equal 'F', @tu.puke('TestSomeClass', 'test_method_name', exception) + assert_equal 1, @tu.failures + assert_match(/^Failure.*Oh no!/m, @tu.report.first) + assert_match("TestSomeClass#test_method_name [#{ex_location}]", @tu.report.first) + end + + def test_class_puke_with_non_failure_exception + exception = Exception.new("Oh no again!") + assert_equal 'E', @tu.puke('SomeClass', 'method_name', exception) + assert_equal 1, @tu.errors + assert_match(/^Exception.*Oh no again!/m, @tu.report.first) + end + + def test_filter_backtrace + # this is a semi-lame mix of relative paths. + # I cheated by making the autotest parts not have ./ + bt = (["lib/autotest.rb:571:in `add_exception'", + "test/test_autotest.rb:62:in `test_add_exception'", + "#{MINITEST_BASE_DIR}/test.rb:165:in `__send__'"] + + BT_MIDDLE + + ["#{MINITEST_BASE_DIR}/test.rb:29", + "test/test_autotest.rb:422"]) + bt = util_expand_bt bt + + ex = ["lib/autotest.rb:571:in `add_exception'", + "test/test_autotest.rb:62:in `test_add_exception'"] + ex = util_expand_bt ex + + fu = Test::filter_backtrace(bt) + + assert_equal ex, fu + end + + def test_filter_backtrace_all_unit + bt = (["#{MINITEST_BASE_DIR}/test.rb:165:in `__send__'"] + + BT_MIDDLE + + ["#{MINITEST_BASE_DIR}/test.rb:29"]) + ex = bt.clone + fu = Test::filter_backtrace(bt) + assert_equal ex, fu + end + + def test_filter_backtrace_unit_starts + bt = (["#{MINITEST_BASE_DIR}/test.rb:165:in `__send__'"] + + BT_MIDDLE + + ["#{MINITEST_BASE_DIR}/mini/test.rb:29", + "-e:1"]) + + bt = util_expand_bt bt + + ex = ["-e:1"] + fu = Test::filter_backtrace bt + assert_equal ex, fu + end + + def test_default_runner_is_minitest_unit + assert_instance_of Test::Unit::Runner, Test::Unit::Runner.runner + end + + + def test_passed_eh_teardown_good + test_class = Class.new Test::Unit::TestCase do + def teardown; assert true; end + def test_omg; assert true; end + end + + test = test_class.new :test_omg + test.run @tu + assert test.passed? + end + + def test_passed_eh_teardown_skipped + test_class = Class.new Test::Unit::TestCase do + def teardown; assert true; end + def test_omg; skip "bork"; end + end + + test = test_class.new :test_omg + test.run @tu + assert test.passed? + end + + def test_passed_eh_teardown_flunked + test_class = Class.new Test::Unit::TestCase do + def teardown; flunk; end + def test_omg; assert true; end + end + + test = test_class.new :test_omg + test.run @tu + refute test.passed? + end + + def util_expand_bt bt + bt.map { |f| (f =~ /^\./) ? File.expand_path(f) : f } + end +end + +class TestMiniTestUnitInherited < MetaMetaMetaTestCase + def with_overridden_include + Class.class_eval do + def inherited_with_hacks klass + throw :inherited_hook + end + + alias inherited_without_hacks inherited + alias inherited inherited_with_hacks + alias IGNORE_ME! inherited # 1.8 bug. god I love venture bros + end + + yield + ensure + Class.class_eval do + alias inherited inherited_without_hacks + + undef_method :inherited_with_hacks + undef_method :inherited_without_hacks + end + + refute_respond_to Class, :inherited_with_hacks + refute_respond_to Class, :inherited_without_hacks + end + + def test_inherited_hook_plays_nice_with_others + with_overridden_include do + assert_throws :inherited_hook do + Class.new Test::Unit::TestCase + end + end + end +end + +class TestMiniTestRunner < MetaMetaMetaTestCase + # do not parallelize this suite... it just can't handle it. + + def test_class_test_suites + @assertion_count = 0 + + tc = Class.new(Test::Unit::TestCase) + + assert_equal 2, Test::Unit::TestCase.test_suites.size + assert_equal [tc, Test::Unit::TestCase], Test::Unit::TestCase.test_suites.sort_by {|ts| ts.name.to_s} + end + + def assert_filtering name, expected, a = false + args = %W[--name #{name} --seed 42] + + alpha = Class.new Test::Unit::TestCase do + define_method :test_something do + assert a + end + end + Object.const_set(:Alpha, alpha) + + beta = Class.new Test::Unit::TestCase do + define_method :test_something do + assert true + end + end + Object.const_set(:Beta, beta) + + assert_report expected, args + ensure + Object.send :remove_const, :Alpha + Object.send :remove_const, :Beta + end + + def test_run_with_other_runner + pend "We don't imagine to replace the default runner with ruby/ruby test suite." + Test::Unit::Runner.runner = Class.new Test::Unit::Runner do + def _run_suite suite, type + suite.before_suite # Run once before each suite + super suite, type + end + end.new + + Class.new Test::Unit::TestCase do + def self.name; "wacky!" end + + def self.before_suite + Test::Unit::Runner.output.puts "Running #{self.name} tests" + @@foo = 1 + end + + def test_something + assert_equal 1, @@foo + end + + def test_something_else + assert_equal 1, @@foo + end + end + + expected = clean <<-EOM + Running wacky! tests + .. + + Finished tests in 0.00 + + 2 tests, 2 assertions, 0 failures, 0 errors, 0 skips + EOM + + assert_report expected + end + + require 'monitor' + + class Latch + def initialize count = 1 + @count = count + @lock = Monitor.new + @cv = @lock.new_cond + end + + def release + @lock.synchronize do + @count -= 1 if @count > 0 + @cv.broadcast if @count == 0 + end + end + + def await + @lock.synchronize { @cv.wait_while { @count > 0 } } + end + end +end + +class TestMiniTestUnitOrder < MetaMetaMetaTestCase + # do not parallelize this suite... it just can't handle it. + + def test_before_setup + pend "Surpressing the raise message when running with tests" + + call_order = [] + Class.new Test::Unit::TestCase do + define_method :setup do + super() + call_order << :setup + end + + define_method :before_setup do + call_order << :before_setup + end + + def test_omg; assert true; end + end + + with_output do + @tu.run %w[--seed 42] + end + + expected = [:before_setup, :setup] + assert_equal expected, call_order + end + + def test_after_teardown + pend "Surpressing the result message of this tests" + + call_order = [] + Class.new Test::Unit::TestCase do + define_method :teardown do + super() + call_order << :teardown + end + + define_method :after_teardown do + call_order << :after_teardown + end + + def test_omg; assert true; end + end + + with_output do + @tu.run %w[--seed 42] + end + + expected = [:teardown, :after_teardown] + assert_equal expected, call_order + end + + def test_all_teardowns_are_guaranteed_to_run + pend "Surpressing the raise message when running with tests" + + call_order = [] + Class.new Test::Unit::TestCase do + define_method :after_teardown do + super() + call_order << :after_teardown + raise + end + + define_method :teardown do + super() + call_order << :teardown + raise + end + + define_method :before_teardown do + super() + call_order << :before_teardown + raise + end + + def test_omg; assert true; end + end + + with_output do + @tu.run %w[--seed 42] + end + + expected = [:before_teardown, :teardown, :after_teardown] + assert_equal expected, call_order + end + + def test_setup_and_teardown_survive_inheritance + pend "Surpressing the result message of this tests" + + call_order = [] + + parent = Class.new Test::Unit::TestCase do + define_method :setup do + call_order << :setup_method + end + + define_method :teardown do + call_order << :teardown_method + end + + define_method :test_something do + call_order << :test + end + end + + _ = Class.new parent + + with_output do + @tu.run %w[--seed 42] + end + + # Once for the parent class, once for the child + expected = [:setup_method, :test, :teardown_method] * 2 + + assert_equal expected, call_order + end +end + +class TestMiniTestUnitTestCase < Test::Unit::TestCase + # do not call parallelize_me! - teardown accesses @tc._assertions + # which is not threadsafe. Nearly every method in here is an + # assertion test so it isn't worth splitting it out further. + + def setup + super + + Test::Unit::TestCase.reset + + @tc = Test::Unit::TestCase.new 'fake tc' + @zomg = "zomg ponies!" + @assertion_count = 1 + end + + def teardown + assert_equal(@assertion_count, @tc._assertions, + "expected #{@assertion_count} assertions to be fired during the test, not #{@tc._assertions}") if @tc.passed? + end + + def non_verbose + orig_verbose = $VERBOSE + $VERBOSE = false + + yield + ensure + $VERBOSE = orig_verbose + end + + def test_assert + @assertion_count = 2 + + @tc.assert_equal true, @tc.assert(true), "returns true on success" + end + + def test_assert__triggered + util_assert_triggered "Failed assertion, no message given." do + @tc.assert false + end + end + + def test_assert__triggered_message + util_assert_triggered @zomg do + @tc.assert false, @zomg + end + end + + def test_assert_empty + @assertion_count = 2 + + @tc.assert_empty [] + end + + def test_assert_empty_triggered + @assertion_count = 2 + + util_assert_triggered "Expected [1] to be empty." do + @tc.assert_empty [1] + end + end + + def test_assert_equal + @tc.assert_equal 1, 1 + end + + def test_assert_equal_different_collection_array_hex_invisible + object1 = Object.new + object2 = Object.new + msg = "<[#{object1.inspect}]> expected but was + <[#{object2.inspect}]>.".gsub(/^ +/, "") + util_assert_triggered msg do + @tc.assert_equal [object1], [object2] + end + end + + def test_assert_equal_different_collection_hash_hex_invisible + h1, h2 = {}, {} + h1[1] = Object.new + h2[1] = Object.new + msg = "<#{h1.inspect}> expected but was + <#{h2.inspect}>.".gsub(/^ +/, "") + + util_assert_triggered msg do + @tc.assert_equal h1, h2 + end + end + + def test_assert_equal_different_diff_deactivated + without_diff do + util_assert_triggered util_msg("haha" * 10, "blah" * 10) do + o1 = "haha" * 10 + o2 = "blah" * 10 + + @tc.assert_equal o1, o2 + end + end + end + + def test_assert_equal_different_hex + c = Class.new do + def initialize s; @name = s; end + end + + o1 = c.new "a" + o2 = c.new "b" + msg = "<#{o1.inspect}> expected but was + <#{o2.inspect}>.".gsub(/^ +/, "") + + util_assert_triggered msg do + @tc.assert_equal o1, o2 + end + end + + def test_assert_equal_different_hex_invisible + o1 = Object.new + o2 = Object.new + + msg = "<#{o1.inspect}> expected but was + <#{o2.inspect}>.".gsub(/^ +/, "") + + util_assert_triggered msg do + @tc.assert_equal o1, o2 + end + end + + def test_assert_equal_different_long + msg = "<\"hahahahahahahahahahahahahahahahahahahaha\"> expected but was + <\"blahblahblahblahblahblahblahblahblahblah\">.".gsub(/^ +/, "") + + util_assert_triggered msg do + o1 = "haha" * 10 + o2 = "blah" * 10 + + @tc.assert_equal o1, o2 + end + end + + def test_assert_equal_different_long_invisible + msg = "<\"blahblahblahblahblahblahblahblahblahblah\"> (UTF-8) expected but was + <\"blahblahblahblahblahblahblahblahblahblah\"> (UTF-8).".gsub(/^ +/, "") + + util_assert_triggered msg do + o1 = "blah" * 10 + o2 = "blah" * 10 + def o1.== o + false + end + @tc.assert_equal o1, o2 + end + end + + def test_assert_equal_different_long_msg + msg = "message. + <\"hahahahahahahahahahahahahahahahahahahaha\"> expected but was + <\"blahblahblahblahblahblahblahblahblahblah\">.".gsub(/^ +/, "") + + util_assert_triggered msg do + o1 = "haha" * 10 + o2 = "blah" * 10 + @tc.assert_equal o1, o2, "message" + end + end + + def test_assert_equal_different_short + util_assert_triggered util_msg(1, 2) do + @tc.assert_equal 1, 2 + end + end + + def test_assert_equal_different_short_msg + util_assert_triggered util_msg(1, 2, "message") do + @tc.assert_equal 1, 2, "message" + end + end + + def test_assert_equal_different_short_multiline + msg = "<\"a\\n\" + \"b\"> expected but was\n<\"a\\n\" + \"c\">." + util_assert_triggered msg do + @tc.assert_equal "a\nb", "a\nc" + end + end + + def test_assert_equal_different_escaped_newline + msg = "<\"xxx\\n\" + \"a\\\\nb\"> expected but was\n<\"xxx\\n\" + \"a\\\\nc\">." + util_assert_triggered msg do + @tc.assert_equal "xxx\na\\nb", "xxx\na\\nc" + end + end + + def test_assert_in_delta + @tc.assert_in_delta 0.0, 1.0 / 1000, 0.1 + end + + def test_delta_consistency + @tc.assert_in_delta 0, 1, 1 + + util_assert_triggered "Expected |0 - 1| (1) to not be <= 1." do + @tc.refute_in_delta 0, 1, 1 + end + end + + def test_assert_in_delta_triggered + x = "1.0e-06" + util_assert_triggered "Expected |0.0 - 0.001| (0.001) to be <= #{x}." do + @tc.assert_in_delta 0.0, 1.0 / 1000, 0.000001 + end + end + + def test_assert_in_epsilon + @assertion_count = 10 + + @tc.assert_in_epsilon 10000, 9991 + @tc.assert_in_epsilon 9991, 10000 + @tc.assert_in_epsilon 1.0, 1.001 + @tc.assert_in_epsilon 1.001, 1.0 + + @tc.assert_in_epsilon 10000, 9999.1, 0.0001 + @tc.assert_in_epsilon 9999.1, 10000, 0.0001 + @tc.assert_in_epsilon 1.0, 1.0001, 0.0001 + @tc.assert_in_epsilon 1.0001, 1.0, 0.0001 + + @tc.assert_in_epsilon(-1, -1) + @tc.assert_in_epsilon(-10000, -9991) + end + + def test_epsilon_consistency + @tc.assert_in_epsilon 1.0, 1.001 + + msg = "Expected |1.0 - 1.001| (0.000999xxx) to not be <= 0.001." + util_assert_triggered msg do + @tc.refute_in_epsilon 1.0, 1.001 + end + end + + def test_assert_in_epsilon_triggered + util_assert_triggered 'Expected |10000 - 9990| (10) to be <= 9.99.' do + @tc.assert_in_epsilon 10000, 9990 + end + end + + def test_assert_in_epsilon_triggered_negative_case + x = "0.100000xxx" + y = "0.1" + util_assert_triggered "Expected |-1.1 - -1| (#{x}) to be <= #{y}." do + @tc.assert_in_epsilon(-1.1, -1, 0.1) + end + end + + def test_assert_includes + @assertion_count = 2 + + @tc.assert_includes [true], true + end + + def test_assert_includes_triggered + @assertion_count = 3 + + e = @tc.assert_raise Test::Unit::AssertionFailedError do + @tc.assert_includes [true], false + end + + expected = "Expected [true] to include false." + assert_equal expected, e.message + end + + def test_assert_instance_of + @tc.assert_instance_of String, "blah" + end + + def test_assert_instance_of_triggered + util_assert_triggered 'Expected "blah" to be an instance of Array, not String.' do + @tc.assert_instance_of Array, "blah" + end + end + + def test_assert_kind_of + @tc.assert_kind_of String, "blah" + end + + def test_assert_kind_of_triggered + util_assert_triggered 'Expected "blah" to be a kind of Array, not String.' do + @tc.assert_kind_of Array, "blah" + end + end + + def test_assert_match + @assertion_count = 2 + @tc.assert_match(/\w+/, "blah blah blah") + end + + def test_assert_match_matcher_object + @assertion_count = 2 + + pattern = Object.new + def pattern.=~(other) true end + + @tc.assert_match pattern, 5 + end + + def test_assert_match_matchee_to_str + @assertion_count = 2 + + obj = Object.new + def obj.to_str; "blah" end + + @tc.assert_match "blah", obj + end + + def test_assert_match_object_triggered + @assertion_count = 2 + + pattern = Object.new + def pattern.=~(other) false end + def pattern.inspect; "[Object]" end + + util_assert_triggered 'Expected [Object] to match 5.' do + @tc.assert_match pattern, 5 + end + end + + def test_assert_match_triggered + @assertion_count = 2 + util_assert_triggered 'Expected /\d+/ to match "blah blah blah".' do + @tc.assert_match(/\d+/, "blah blah blah") + end + end + + def test_assert_nil + @tc.assert_nil nil + end + + def test_assert_nil_triggered + util_assert_triggered 'Expected 42 to be nil.' do + @tc.assert_nil 42 + end + end + + def test_assert_operator + @tc.assert_operator 2, :>, 1 + end + + def test_assert_operator_bad_object + bad = Object.new + def bad.==(other) true end + + @tc.assert_operator bad, :equal?, bad + end + + def test_assert_operator_triggered + util_assert_triggered "Expected 2 to be < 1." do + @tc.assert_operator 2, :<, 1 + end + end + + def test_assert_output_both + @assertion_count = 2 + + @tc.assert_output "yay", "blah" do + print "yay" + $stderr.print "blah" + end + end + + def test_assert_output_both_regexps + @assertion_count = 4 + + @tc.assert_output(/y.y/, /bl.h/) do + print "yay" + $stderr.print "blah" + end + end + + def test_assert_output_err + @tc.assert_output nil, "blah" do + $stderr.print "blah" + end + end + + def test_assert_output_neither + @assertion_count = 0 + + @tc.assert_output do + # do nothing + end + end + + def test_assert_output_out + @tc.assert_output "blah" do + print "blah" + end + end + + def test_assert_output_triggered_both + util_assert_triggered util_msg("blah", "blah blah", "In stderr") do + @tc.assert_output "yay", "blah" do + print "boo" + $stderr.print "blah blah" + end + end + end + + def test_assert_output_triggered_err + util_assert_triggered util_msg("blah", "blah blah", "In stderr") do + @tc.assert_output nil, "blah" do + $stderr.print "blah blah" + end + end + end + + def test_assert_output_triggered_out + util_assert_triggered util_msg("blah", "blah blah", "In stdout") do + @tc.assert_output "blah" do + print "blah blah" + end + end + end + + def test_assert_predicate + @tc.assert_predicate "", :empty? + end + + def test_assert_predicate_triggered + util_assert_triggered 'Expected "blah" to be empty?.' do + @tc.assert_predicate "blah", :empty? + end + end + + def test_assert_raise + @tc.assert_raise RuntimeError do + raise "blah" + end + end + + def test_assert_raise_module + @tc.assert_raise MyModule do + raise AnError + end + end + + ## + # *sigh* This is quite an odd scenario, but it is from real (albeit + # ugly) test code in ruby-core: + # + # https://github.com/ruby/ruby/commit/6bab4ea9917dc05cd2c94aead2e96eb7df7d4be1 + + def test_assert_raise_skip + @assertion_count = 0 + + util_assert_triggered "skipped", Test::Unit::PendedError do + @tc.assert_raise ArgumentError do + begin + raise "blah" + rescue + skip "skipped" + end + end + end + end + + def test_assert_raise_triggered_different + e = assert_raise Test::Unit::AssertionFailedError do + @tc.assert_raise RuntimeError do + raise SyntaxError, "icky" + end + end + + expected = clean <<-EOM.chomp + [RuntimeError] exception expected, not #<SyntaxError: icky>. + EOM + + actual = e.message.gsub(/^.+:\d+/, 'FILE:LINE') + actual.gsub!(/block \(\d+ levels\) in /, '') if RUBY_VERSION >= '1.9.0' + + assert_equal expected, actual + end + + def test_assert_raise_triggered_different_msg + e = assert_raise Test::Unit::AssertionFailedError do + @tc.assert_raise RuntimeError, "XXX" do + raise SyntaxError, "icky" + end + end + + expected = clean <<-EOM + XXX. + [RuntimeError] exception expected, not #<SyntaxError: icky>. + EOM + + actual = e.message.gsub(/^.+:\d+/, 'FILE:LINE') + actual.gsub!(/block \(\d+ levels\) in /, '') if RUBY_VERSION >= '1.9.0' + + assert_equal expected.chomp, actual + end + + def test_assert_raise_triggered_none + e = assert_raise Test::Unit::AssertionFailedError do + @tc.assert_raise Test::Unit::AssertionFailedError do + # do nothing + end + end + + expected = "Test::Unit::AssertionFailedError expected but nothing was raised." + + assert_equal expected, e.message + end + + def test_assert_raise_triggered_none_msg + e = assert_raise Test::Unit::AssertionFailedError do + @tc.assert_raise Test::Unit::AssertionFailedError, "XXX" do + # do nothing + end + end + + expected = "XXX.\nTest::Unit::AssertionFailedError expected but nothing was raised." + + assert_equal expected, e.message + end + + def test_assert_raise_triggered_subclass + e = assert_raise Test::Unit::AssertionFailedError do + @tc.assert_raise StandardError do + raise AnError + end + end + + expected = clean <<-EOM.chomp + [StandardError] exception expected, not #<AnError: AnError>. + EOM + + actual = e.message.gsub(/^.+:\d+/, 'FILE:LINE') + actual.gsub!(/block \(\d+ levels\) in /, '') if RUBY_VERSION >= '1.9.0' + + assert_equal expected, actual + end + + def test_assert_respond_to + @tc.assert_respond_to "blah", :empty? + end + + def test_assert_respond_to_triggered + util_assert_triggered 'Expected "blah" (String) to respond to #rawr!.' do + @tc.assert_respond_to "blah", :rawr! + end + end + + def test_assert_same + @assertion_count = 3 + + o = "blah" + @tc.assert_same 1, 1 + @tc.assert_same :blah, :blah + @tc.assert_same o, o + end + + def test_assert_same_triggered + @assertion_count = 2 + + util_assert_triggered 'Expected 2 (oid=N) to be the same as 1 (oid=N).' do + @tc.assert_same 1, 2 + end + + s1 = "blah" + s2 = "blah" + + util_assert_triggered 'Expected "blah" (oid=N) to be the same as "blah" (oid=N).' do + @tc.assert_same s1, s2 + end + end + + def test_assert_send + @tc.assert_send [1, :<, 2] + end + + def test_assert_send_bad + util_assert_triggered "Expected 1.>(2) to return true." do + @tc.assert_send [1, :>, 2] + end + end + + def test_assert_silent + @assertion_count = 2 + + @tc.assert_silent do + # do nothing + end + end + + def test_assert_silent_triggered_err + util_assert_triggered util_msg("", "blah blah", "In stderr") do + @tc.assert_silent do + $stderr.print "blah blah" + end + end + end + + def test_assert_silent_triggered_out + @assertion_count = 2 + + util_assert_triggered util_msg("", "blah blah", "In stdout") do + @tc.assert_silent do + print "blah blah" + end + end + end + + def test_assert_throws + @tc.assert_throws :blah do + throw :blah + end + end + + def test_assert_throws_different + util_assert_triggered 'Expected :blah to have been thrown, not :not_blah.' do + @tc.assert_throws :blah do + throw :not_blah + end + end + end + + def test_assert_throws_unthrown + util_assert_triggered 'Expected :blah to have been thrown.' do + @tc.assert_throws :blah do + # do nothing + end + end + end + + def test_capture_output + @assertion_count = 0 + + non_verbose do + out, err = capture_output do + puts 'hi' + $stderr.puts 'bye!' + end + + assert_equal "hi\n", out + assert_equal "bye!\n", err + end + end + + def test_flunk + util_assert_triggered 'Epic Fail!' do + @tc.flunk + end + end + + def test_flunk_message + util_assert_triggered @zomg do + @tc.flunk @zomg + end + end + + def test_message + @assertion_count = 0 + + assert_equal "blah2.", @tc.message { "blah2" }.call + assert_equal "blah2.", @tc.message("") { "blah2" }.call + assert_equal "blah1.\nblah2.", @tc.message(:blah1) { "blah2" }.call + assert_equal "blah1.\nblah2.", @tc.message("blah1") { "blah2" }.call + + message = proc { "blah1" } + assert_equal "blah1.\nblah2.", @tc.message(message) { "blah2" }.call + + message = @tc.message { "blah1" } + assert_equal "blah1.\nblah2.", @tc.message(message) { "blah2" }.call + end + + def test_message_message + util_assert_triggered "whoops.\n<1> expected but was\n<2>." do + @tc.assert_equal 1, 2, message { "whoops" } + end + end + + def test_message_lambda + util_assert_triggered "whoops.\n<1> expected but was\n<2>." do + @tc.assert_equal 1, 2, lambda { "whoops" } + end + end + + def test_message_deferred + @assertion_count, var = 0, nil + + msg = message { var = "blah" } + + assert_nil var + + msg.call + + assert_equal "blah", var + end + + def test_pass + @tc.pass + end + + def test_prints + printer = Class.new { extend Test::Unit::CoreAssertions } + @tc.assert_equal '"test"', printer.mu_pp(ImmutableString.new 'test') + end + + def test_refute + @assertion_count = 2 + + @tc.assert_equal false, @tc.refute(false), "returns false on success" + end + + def test_refute_empty + @assertion_count = 2 + + @tc.refute_empty [1] + end + + def test_refute_empty_triggered + @assertion_count = 2 + + util_assert_triggered "Expected [] to not be empty." do + @tc.refute_empty [] + end + end + + def test_refute_equal + @tc.refute_equal "blah", "yay" + end + + def test_refute_equal_triggered + util_assert_triggered 'Expected "blah" to not be equal to "blah".' do + @tc.refute_equal "blah", "blah" + end + end + + def test_refute_in_delta + @tc.refute_in_delta 0.0, 1.0 / 1000, 0.000001 + end + + def test_refute_in_delta_triggered + x = "0.1" + util_assert_triggered "Expected |0.0 - 0.001| (0.001) to not be <= #{x}." do + @tc.refute_in_delta 0.0, 1.0 / 1000, 0.1 + end + end + + def test_refute_in_epsilon + @tc.refute_in_epsilon 10000, 9990-1 + end + + def test_refute_in_epsilon_triggered + util_assert_triggered 'Expected |10000 - 9990| (10) to not be <= 10.0.' do + @tc.refute_in_epsilon 10000, 9990 + fail + end + end + + def test_refute_includes + @assertion_count = 2 + + @tc.refute_includes [true], false + end + + def test_refute_includes_triggered + @assertion_count = 3 + + e = @tc.assert_raise Test::Unit::AssertionFailedError do + @tc.refute_includes [true], true + end + + expected = "Expected [true] to not include true." + assert_equal expected, e.message + end + + def test_refute_instance_of + @tc.refute_instance_of Array, "blah" + end + + def test_refute_instance_of_triggered + util_assert_triggered 'Expected "blah" to not be an instance of String.' do + @tc.refute_instance_of String, "blah" + end + end + + def test_refute_kind_of + @tc.refute_kind_of Array, "blah" + end + + def test_refute_kind_of_triggered + util_assert_triggered 'Expected "blah" to not be a kind of String.' do + @tc.refute_kind_of String, "blah" + end + end + + def test_refute_match + @assertion_count = 2 + @tc.refute_match(/\d+/, "blah blah blah") + end + + def test_refute_match_matcher_object + @assertion_count = 2 + non_verbose do + obj = Object.new + def obj.=~(other); false; end + @tc.refute_match obj, 5 + end + end + + def test_refute_match_object_triggered + @assertion_count = 2 + + pattern = Object.new + def pattern.=~(other) true end + def pattern.inspect; "[Object]" end + + util_assert_triggered 'Expected [Object] to not match 5.' do + @tc.refute_match pattern, 5 + end + end + + def test_refute_match_triggered + @assertion_count = 2 + util_assert_triggered 'Expected /\w+/ to not match "blah blah blah".' do + @tc.refute_match(/\w+/, "blah blah blah") + end + end + + def test_refute_nil + @tc.refute_nil 42 + end + + def test_refute_nil_triggered + util_assert_triggered 'Expected nil to not be nil.' do + @tc.refute_nil nil + end + end + + def test_refute_predicate + @tc.refute_predicate "42", :empty? + end + + def test_refute_predicate_triggered + util_assert_triggered 'Expected "" to not be empty?.' do + @tc.refute_predicate "", :empty? + end + end + + def test_refute_operator + @tc.refute_operator 2, :<, 1 + end + + def test_refute_operator_bad_object + bad = Object.new + def bad.==(other) true end + + @tc.refute_operator true, :equal?, bad + end + + def test_refute_operator_triggered + util_assert_triggered "Expected 2 to not be > 1." do + @tc.refute_operator 2, :>, 1 + end + end + + def test_refute_respond_to + @tc.refute_respond_to "blah", :rawr! + end + + def test_refute_respond_to_triggered + util_assert_triggered 'Expected "blah" to not respond to empty?.' do + @tc.refute_respond_to "blah", :empty? + end + end + + def test_refute_same + @tc.refute_same 1, 2 + end + + def test_refute_same_triggered + util_assert_triggered 'Expected 1 (oid=N) to not be the same as 1 (oid=N).' do + @tc.refute_same 1, 1 + end + end + + def test_skip + @assertion_count = 0 + + util_assert_triggered "haha!", Test::Unit::PendedError do + @tc.skip "haha!" + end + end + + def test_test_methods + @assertion_count = 0 + + sample_test_case = Class.new Test::Unit::TestCase do + def test_test1; assert "does not matter" end + def test_test2; assert "does not matter" end + def test_test3; assert "does not matter" end + end + + expected = %i(test_test1 test_test2 test_test3) + assert_equal expected, sample_test_case.test_methods.sort + end + + def assert_triggered expected, klass = Test::Unit::AssertionFailedError + e = assert_raise klass do + yield + end + + msg = e.message.sub(/(---Backtrace---).*/m, '\1') + msg.gsub!(/\(oid=[-0-9]+\)/, '(oid=N)') + msg.gsub!(/(\d\.\d{6})\d+/, '\1xxx') # normalize: ruby version, impl, platform + + assert_equal expected, msg + end + alias util_assert_triggered assert_triggered + + def util_msg exp, act, msg = nil + s = "<#{exp.inspect}> expected but was\n<#{act.inspect}>." + s = "#{msg}.\n#{s}" if msg + s + end + + def without_diff + old_diff = Test::Unit::Assertions.diff + Test::Unit::Assertions.diff = nil + + yield + ensure + Test::Unit::Assertions.diff = old_diff + end +end + +class TestMiniTestGuard < Test::Unit::TestCase + def test_mri_eh + assert self.class.mri? "ruby blah" + assert self.mri? "ruby blah" + end + + def test_jruby_eh + assert self.class.jruby? "java" + assert self.jruby? "java" + end + + def test_windows_eh + assert self.class.windows? "mswin" + assert self.windows? "mswin" + end +end + +class TestMiniTestUnitRecording < MetaMetaMetaTestCase + # do not parallelize this suite... it just can't handle it. + + def assert_run_record(*expected, &block) + pend "Test::Unit::Runner#run was changed about recoding feature. We should fix it." + def @tu.record suite, method, assertions, time, error + recording[method] << error + end + + def @tu.recording + @recording ||= Hash.new { |h,k| h[k] = [] } + end + + Test::Unit::Runner.runner = @tu + + Class.new Test::Unit::TestCase, &block + + with_output do + @tu.run + end + + recorded = @tu.recording.fetch("test_method").map(&:class) + + assert_equal expected, recorded + end + + def test_record_passing + assert_run_record NilClass do + def test_method + assert true + end + end + end + + def test_record_failing + assert_run_record Test::Unit::AssertionFailedError do + def test_method + assert false + end + end + end + + def test_record_error + assert_run_record RuntimeError do + def test_method + raise "unhandled exception" + end + end + end + + def test_record_error_teardown + assert_run_record NilClass, RuntimeError do + def test_method + assert true + end + + def teardown + raise "unhandled exception" + end + end + end + + def test_record_error_in_test_and_teardown + assert_run_record AnError, RuntimeError do + def test_method + raise AnError + end + + def teardown + raise "unhandled exception" + end + end + end + + def test_record_skip + assert_run_record Test::Unit::PendedError do + def test_method + skip "not yet" + end + end + end +end diff --git a/tool/test/testunit/test_parallel.rb b/tool/test/testunit/test_parallel.rb new file mode 100644 index 0000000000..006354aee2 --- /dev/null +++ b/tool/test/testunit/test_parallel.rb @@ -0,0 +1,219 @@ +# frozen_string_literal: false +require 'test/unit' +require 'timeout' + +module TestParallel + PARALLEL_RB = "#{__dir__}/../../lib/test/unit/parallel.rb" + TESTS = "#{__dir__}/tests_for_parallel" + # use large timeout for --jit-wait + TIMEOUT = EnvUtil.apply_timeout_scale(defined?(RubyVM::MJIT) && RubyVM::MJIT.enabled? ? 100 : 30) + + class TestParallelWorker < Test::Unit::TestCase + def setup + i, @worker_in = IO.pipe + @worker_out, o = IO.pipe + @worker_pid = spawn(*@options[:ruby], PARALLEL_RB, + "--ruby", @options[:ruby].join(" "), + "-j", "t1", "-v", out: o, in: i) + [i,o].each(&:close) + end + + def teardown + if @worker_pid && @worker_in + begin + begin + @worker_in.puts "quit" + rescue IOError, Errno::EPIPE + end + Timeout.timeout(2) do + Process.waitpid(@worker_pid) + end + rescue Timeout::Error + begin + Process.kill(:KILL, @worker_pid) + rescue Errno::ESRCH + end + end + end + ensure + begin + @worker_in.close + @worker_out.close + rescue Errno::EPIPE + # may already broken and rescue'ed in above code + end + end + + def test_run + Timeout.timeout(TIMEOUT) do + assert_match(/^ready/,@worker_out.gets) + @worker_in.puts "run #{TESTS}/ptest_first.rb test" + assert_match(/^okay/,@worker_out.gets) + assert_match(/^start/,@worker_out.gets) + assert_match(/^record/,@worker_out.gets) + assert_match(/^p/,@worker_out.gets) + assert_match(/^done/,@worker_out.gets) + assert_match(/^ready/,@worker_out.gets) + end + end + + def test_run_multiple_testcase_in_one_file + Timeout.timeout(TIMEOUT) do + assert_match(/^ready/,@worker_out.gets) + @worker_in.puts "run #{TESTS}/ptest_second.rb test" + assert_match(/^okay/,@worker_out.gets) + assert_match(/^start/,@worker_out.gets) + assert_match(/^record/,@worker_out.gets) + assert_match(/^p/,@worker_out.gets) + assert_match(/^done/,@worker_out.gets) + assert_match(/^start/,@worker_out.gets) + assert_match(/^record/,@worker_out.gets) + assert_match(/^p/,@worker_out.gets) + assert_match(/^done/,@worker_out.gets) + assert_match(/^ready/,@worker_out.gets) + end + end + + def test_accept_run_command_multiple_times + Timeout.timeout(TIMEOUT) do + assert_match(/^ready/,@worker_out.gets) + @worker_in.puts "run #{TESTS}/ptest_first.rb test" + assert_match(/^okay/,@worker_out.gets) + assert_match(/^start/,@worker_out.gets) + assert_match(/^record/,@worker_out.gets) + assert_match(/^p/,@worker_out.gets) + assert_match(/^done/,@worker_out.gets) + assert_match(/^ready/,@worker_out.gets) + @worker_in.puts "run #{TESTS}/ptest_second.rb test" + assert_match(/^okay/,@worker_out.gets) + assert_match(/^start/,@worker_out.gets) + assert_match(/^record/,@worker_out.gets) + assert_match(/^p/,@worker_out.gets) + assert_match(/^done/,@worker_out.gets) + assert_match(/^start/,@worker_out.gets) + assert_match(/^record/,@worker_out.gets) + assert_match(/^p/,@worker_out.gets) + assert_match(/^done/,@worker_out.gets) + assert_match(/^ready/,@worker_out.gets) + end + end + + def test_p + Timeout.timeout(TIMEOUT) do + @worker_in.puts "run #{TESTS}/ptest_first.rb test" + while buf = @worker_out.gets + break if /^p (.+?)$/ =~ buf + end + assert_not_nil($1, "'p' was not found") + assert_match(/TestA#test_nothing_test = \d+\.\d+ s = \.\n/, $1.chomp.unpack1("m")) + end + end + + def test_done + Timeout.timeout(TIMEOUT) do + @worker_in.puts "run #{TESTS}/ptest_forth.rb test" + while buf = @worker_out.gets + break if /^done (.+?)$/ =~ buf + end + assert_not_nil($1, "'done' was not found") + + result = Marshal.load($1.chomp.unpack1("m")) + assert_equal(5, result[0]) + pend "TODO: result[1] returns 17. We should investigate it" do + assert_equal(12, result[1]) + end + assert_kind_of(Array,result[2]) + assert_kind_of(Array,result[3]) + assert_kind_of(Array,result[4]) + assert_kind_of(Array,result[2][1]) + assert_kind_of(Test::Unit::AssertionFailedError,result[2][0][2]) + assert_kind_of(Test::Unit::PendedError,result[2][1][2]) + assert_kind_of(Test::Unit::PendedError,result[2][2][2]) + assert_kind_of(Exception, result[2][3][2]) + assert_equal(result[5], "TestE") + end + end + + def test_quit + Timeout.timeout(TIMEOUT) do + @worker_in.puts "quit" + assert_match(/^bye$/m,@worker_out.read) + end + end + end + + class TestParallel < Test::Unit::TestCase + def spawn_runner(*opt_args) + @test_out, o = IO.pipe + @test_pid = spawn(*@options[:ruby], TESTS+"/runner.rb", + "--ruby", @options[:ruby].join(" "), + "-j","t1",*opt_args, out: o, err: o) + o.close + end + + def teardown + begin + if @test_pid + Timeout.timeout(2) do + Process.waitpid(@test_pid) + end + end + rescue Timeout::Error + Process.kill(:KILL, @test_pid) if @test_pid + ensure + @test_out&.close + end + end + + def test_ignore_jzero + @test_out, o = IO.pipe + @test_pid = spawn(*@options[:ruby], TESTS+"/runner.rb", + "--ruby", @options[:ruby].join(" "), + "-j","0", out: File::NULL, err: o) + o.close + Timeout.timeout(TIMEOUT) { + assert_match(/Error: parameter of -j option should be greater than 0/,@test_out.read) + } + end + + def test_should_run_all_without_any_leaks + spawn_runner + buf = Timeout.timeout(TIMEOUT) {@test_out.read} + assert_match(/^9 tests/,buf) + end + + def test_should_retry_failed_on_workers + spawn_runner + buf = Timeout.timeout(TIMEOUT) {@test_out.read} + assert_match(/^Retrying\.+$/,buf) + end + + def test_no_retry_option + spawn_runner "--no-retry" + buf = Timeout.timeout(TIMEOUT) {@test_out.read} + refute_match(/^Retrying\.+$/,buf) + assert_match(/^ +\d+\) Failure:\nTestD#test_fail_at_worker/,buf) + end + + def test_jobs_status + spawn_runner "--jobs-status" + buf = Timeout.timeout(TIMEOUT) {@test_out.read} + assert_match(/\d+=ptest_(first|second|third|forth) */,buf) + end + + def test_separate + # this test depends to --jobs-status + spawn_runner "--jobs-status", "--separate" + buf = Timeout.timeout(TIMEOUT) {@test_out.read} + assert(buf.scan(/^\[\s*\d+\/\d+\]\s*(\d+?)=/).flatten.uniq.size > 1, + message("retried tests should run in different processes") {buf}) + end + + def test_hungup + spawn_runner "--worker-timeout=1", "test4test_hungup.rb" + buf = Timeout.timeout(TIMEOUT) {@test_out.read} + assert_match(/^Retrying hung up testcases\.+$/, buf) + assert_match(/^2 tests,.* 0 failures,/, buf) + end + end +end diff --git a/tool/test/testunit/test_redefinition.rb b/tool/test/testunit/test_redefinition.rb new file mode 100644 index 0000000000..b4f5cabd4f --- /dev/null +++ b/tool/test/testunit/test_redefinition.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: false +require 'test/unit' + +class TestRedefinition < Test::Unit::TestCase + def test_redefinition + message = %r[test/unit: method TestForTestRedefinition#test_redefinition is redefined$] + assert_raise_with_message(Test::Unit::AssertionFailedError, message) do + require_relative("test4test_redefinition.rb") + end + end +end diff --git a/tool/test/testunit/test_sorting.rb b/tool/test/testunit/test_sorting.rb new file mode 100644 index 0000000000..7678249ec2 --- /dev/null +++ b/tool/test/testunit/test_sorting.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: false +require 'test/unit' + +class TestTestUnitSorting < Test::Unit::TestCase + def test_sorting + result = sorting("--show-skip") + assert_match(/^ 1\) Skipped:/, result) + assert_match(/^ 2\) Failure:/, result) + assert_match(/^ 3\) Error:/, result) + end + + def sorting(*args) + IO.popen([*@options[:ruby], "#{File.dirname(__FILE__)}/test4test_sorting.rb", + "--verbose", *args], err: [:child, :out]) {|f| + f.read + } + end + + Item = Struct.new(:name) + SEED = 0x50975eed + + def make_test_list + (1..16).map {"test_%.3x" % rand(0x1000)}.freeze + end + + def test_sort_alpha + sorter = Test::Unit::Order::Types[:alpha].new(SEED) + assert_kind_of(Test::Unit::Order::Types[:sorted], sorter) + + list = make_test_list + sorted = list.sort + 16.times do + assert_equal(sorted, sorter.sort_by_string(list)) + end + + list = list.map {|s| Item.new(s)}.freeze + sorted = list.sort_by(&:name) + 16.times do + assert_equal(sorted, sorter.sort_by_name(list)) + end + end + + def test_sort_nosort + sorter = Test::Unit::Order::Types[:nosort].new(SEED) + + list = make_test_list + 16.times do + assert_equal(list, sorter.sort_by_string(list)) + end + + list = list.map {|s| Item.new(s)}.freeze + 16.times do + assert_equal(list, sorter.sort_by_name(list)) + end + end + + def test_sort_random + type = Test::Unit::Order::Types[:random] + sorter = type.new(SEED) + + list = make_test_list + sorted = type.new(SEED).sort_by_string(list).freeze + 16.times do + assert_equal(sorted, sorter.sort_by_string(list)) + end + assert_not_equal(sorted, type.new(SEED+1).sort_by_string(list)) + + list = list.map {|s| Item.new(s)}.freeze + sorted = sorted.map {|s| Item.new(s)}.freeze + 16.times do + assert_equal(sorted, sorter.sort_by_name(list)) + end + assert_not_equal(sorted, type.new(SEED+1).sort_by_name(list)) + end +end diff --git a/tool/test/testunit/tests_for_parallel/ptest_first.rb b/tool/test/testunit/tests_for_parallel/ptest_first.rb new file mode 100644 index 0000000000..f8687335b5 --- /dev/null +++ b/tool/test/testunit/tests_for_parallel/ptest_first.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: false +require 'test/unit' + +class TestA < Test::Unit::TestCase + def test_nothing_test + end +end + diff --git a/tool/test/testunit/tests_for_parallel/ptest_forth.rb b/tool/test/testunit/tests_for_parallel/ptest_forth.rb new file mode 100644 index 0000000000..8831676e19 --- /dev/null +++ b/tool/test/testunit/tests_for_parallel/ptest_forth.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: false +require 'test/unit' + +class TestE < Test::Unit::TestCase + class UnknownError < RuntimeError; end + + def test_not_fail + assert_equal(1,1) + end + + def test_always_skip + skip "always" + end + + def test_always_fail + assert_equal(0,1) + end + + def test_skip_after_unknown_error + begin + raise UnknownError, "unknown error" + rescue + skip "after raise" + end + end + + def test_unknown_error + raise UnknownError, "unknown error" + end +end diff --git a/tool/test/testunit/tests_for_parallel/ptest_second.rb b/tool/test/testunit/tests_for_parallel/ptest_second.rb new file mode 100644 index 0000000000..a793c17eb3 --- /dev/null +++ b/tool/test/testunit/tests_for_parallel/ptest_second.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: false +require 'test/unit' + +class TestB < Test::Unit::TestCase + def test_nothing + end +end + +class TestC < Test::Unit::TestCase + def test_nothing + end +end diff --git a/tool/test/testunit/tests_for_parallel/ptest_third.rb b/tool/test/testunit/tests_for_parallel/ptest_third.rb new file mode 100644 index 0000000000..3f448ecfc1 --- /dev/null +++ b/tool/test/testunit/tests_for_parallel/ptest_third.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: false +require 'test/unit' + +class TestD < Test::Unit::TestCase + def test_fail_at_worker + #if /test\/unit\/parallel\.rb/ =~ $0 + if on_parallel_worker? + assert_equal(0,1) + end + end +end diff --git a/tool/test/testunit/tests_for_parallel/runner.rb b/tool/test/testunit/tests_for_parallel/runner.rb new file mode 100644 index 0000000000..02699e271e --- /dev/null +++ b/tool/test/testunit/tests_for_parallel/runner.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: false +require 'rbconfig' + +$LOAD_PATH.unshift "#{File.dirname(__FILE__)}/../../../lib" + +require 'test/unit' + +src_testdir = File.dirname(File.expand_path(__FILE__)) + +class Test::Unit::Runner + @@testfile_prefix = "ptest" +end + +exit Test::Unit::AutoRunner.run(true, src_testdir) diff --git a/tool/test/testunit/tests_for_parallel/test4test_hungup.rb b/tool/test/testunit/tests_for_parallel/test4test_hungup.rb new file mode 100644 index 0000000000..65a75f7c4d --- /dev/null +++ b/tool/test/testunit/tests_for_parallel/test4test_hungup.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true +require_relative '../../../lib/test/unit' + +class TestHung < Test::Unit::TestCase + def test_success_at_worker + assert true + end + + def test_hungup_at_worker + if on_parallel_worker? + sleep 10 + end + assert true + end +end diff --git a/tool/test/webrick/.htaccess b/tool/test/webrick/.htaccess new file mode 100644 index 0000000000..69d4659b9f --- /dev/null +++ b/tool/test/webrick/.htaccess @@ -0,0 +1 @@ +this file should not be published. diff --git a/tool/test/webrick/test_cgi.rb b/tool/test/webrick/test_cgi.rb new file mode 100644 index 0000000000..7a75cf565e --- /dev/null +++ b/tool/test/webrick/test_cgi.rb @@ -0,0 +1,170 @@ +# coding: US-ASCII +# frozen_string_literal: false +require_relative "utils" +require "webrick" +require "test/unit" + +class TestWEBrickCGI < Test::Unit::TestCase + CRLF = "\r\n" + + def teardown + WEBrick::Utils::TimeoutHandler.terminate + super + end + + def start_cgi_server(log_tester=TestWEBrick::DefaultLogTester, &block) + config = { + :CGIInterpreter => TestWEBrick::RubyBin, + :DocumentRoot => File.dirname(__FILE__), + :DirectoryIndex => ["webrick.cgi"], + :RequestCallback => Proc.new{|req, res| + def req.meta_vars + meta = super + meta["RUBYLIB"] = $:.join(File::PATH_SEPARATOR) + meta[RbConfig::CONFIG['LIBPATHENV']] = ENV[RbConfig::CONFIG['LIBPATHENV']] if RbConfig::CONFIG['LIBPATHENV'] + return meta + end + }, + } + if RUBY_PLATFORM =~ /mswin|mingw|cygwin|bccwin32/ + config[:CGIPathEnv] = ENV['PATH'] # runtime dll may not be in system dir. + end + TestWEBrick.start_httpserver(config, log_tester){|server, addr, port, log| + block.call(server, addr, port, log) + } + end + + def test_cgi + start_cgi_server{|server, addr, port, log| + http = Net::HTTP.new(addr, port) + req = Net::HTTP::Get.new("/webrick.cgi") + http.request(req){|res| assert_equal("/webrick.cgi", res.body, log.call)} + req = Net::HTTP::Get.new("/webrick.cgi/path/info") + http.request(req){|res| assert_equal("/path/info", res.body, log.call)} + req = Net::HTTP::Get.new("/webrick.cgi/%3F%3F%3F?foo=bar") + http.request(req){|res| assert_equal("/???", res.body, log.call)} + unless RUBY_PLATFORM =~ /mswin|mingw|cygwin|bccwin32|java/ + # Path info of res.body is passed via ENV. + # ENV[] returns different value on Windows depending on locale. + req = Net::HTTP::Get.new("/webrick.cgi/%A4%DB%A4%B2/%A4%DB%A4%B2") + http.request(req){|res| + assert_equal("/\xA4\xDB\xA4\xB2/\xA4\xDB\xA4\xB2", res.body, log.call)} + end + req = Net::HTTP::Get.new("/webrick.cgi?a=1;a=2;b=x") + http.request(req){|res| assert_equal("a=1, a=2, b=x", res.body, log.call)} + req = Net::HTTP::Get.new("/webrick.cgi?a=1&a=2&b=x") + http.request(req){|res| assert_equal("a=1, a=2, b=x", res.body, log.call)} + + req = Net::HTTP::Post.new("/webrick.cgi?a=x;a=y;b=1") + req["Content-Type"] = "application/x-www-form-urlencoded" + http.request(req, "a=1;a=2;b=x"){|res| + assert_equal("a=1, a=2, b=x", res.body, log.call)} + req = Net::HTTP::Post.new("/webrick.cgi?a=x&a=y&b=1") + req["Content-Type"] = "application/x-www-form-urlencoded" + http.request(req, "a=1&a=2&b=x"){|res| + assert_equal("a=1, a=2, b=x", res.body, log.call)} + req = Net::HTTP::Get.new("/") + http.request(req){|res| + ary = res.body.lines.to_a + assert_match(%r{/$}, ary[0], log.call) + assert_match(%r{/webrick.cgi$}, ary[1], log.call) + } + + req = Net::HTTP::Get.new("/webrick.cgi") + req["Cookie"] = "CUSTOMER=WILE_E_COYOTE; PART_NUMBER=ROCKET_LAUNCHER_0001" + http.request(req){|res| + assert_equal( + "CUSTOMER=WILE_E_COYOTE\nPART_NUMBER=ROCKET_LAUNCHER_0001\n", + res.body, log.call) + } + + req = Net::HTTP::Get.new("/webrick.cgi") + cookie = %{$Version="1"; } + cookie << %{Customer="WILE_E_COYOTE"; $Path="/acme"; } + cookie << %{Part_Number="Rocket_Launcher_0001"; $Path="/acme"; } + cookie << %{Shipping="FedEx"; $Path="/acme"} + req["Cookie"] = cookie + http.request(req){|res| + assert_equal("Customer=WILE_E_COYOTE, Shipping=FedEx", + res["Set-Cookie"], log.call) + assert_equal("Customer=WILE_E_COYOTE\n" + + "Part_Number=Rocket_Launcher_0001\n" + + "Shipping=FedEx\n", res.body, log.call) + } + } + end + + def test_bad_request + log_tester = lambda {|log, access_log| + assert_match(/BadRequest/, log.join) + } + start_cgi_server(log_tester) {|server, addr, port, log| + sock = TCPSocket.new(addr, port) + begin + sock << "POST /webrick.cgi HTTP/1.0" << CRLF + sock << "Content-Type: application/x-www-form-urlencoded" << CRLF + sock << "Content-Length: 1024" << CRLF + sock << CRLF + sock << "a=1&a=2&b=x" + sock.close_write + assert_match(%r{\AHTTP/\d.\d 400 Bad Request}, sock.read, log.call) + ensure + sock.close + end + } + end + + def test_cgi_env + start_cgi_server do |server, addr, port, log| + http = Net::HTTP.new(addr, port) + req = Net::HTTP::Get.new("/webrick.cgi/dumpenv") + req['proxy'] = 'http://example.com/' + req['hello'] = 'world' + http.request(req) do |res| + env = Marshal.load(res.body) + assert_equal 'world', env['HTTP_HELLO'] + assert_not_operator env, :include?, 'HTTP_PROXY' + end + end + end + + CtrlSeq = [0x7f, *(1..31)].pack("C*").gsub(/\s+/, '') + CtrlPat = /#{Regexp.quote(CtrlSeq)}/o + DumpPat = /#{Regexp.quote(CtrlSeq.dump[1...-1])}/o + + def test_bad_uri + log_tester = lambda {|log, access_log| + assert_equal(1, log.length) + assert_match(/ERROR bad URI/, log[0]) + } + start_cgi_server(log_tester) {|server, addr, port, log| + res = TCPSocket.open(addr, port) {|sock| + sock << "GET /#{CtrlSeq}#{CRLF}#{CRLF}" + sock.close_write + sock.read + } + assert_match(%r{\AHTTP/\d.\d 400 Bad Request}, res) + s = log.call.each_line.grep(/ERROR bad URI/)[0] + assert_match(DumpPat, s) + assert_not_match(CtrlPat, s) + } + end + + def test_bad_header + log_tester = lambda {|log, access_log| + assert_equal(1, log.length) + assert_match(/ERROR bad header/, log[0]) + } + start_cgi_server(log_tester) {|server, addr, port, log| + res = TCPSocket.open(addr, port) {|sock| + sock << "GET / HTTP/1.0#{CRLF}#{CtrlSeq}#{CRLF}#{CRLF}" + sock.close_write + sock.read + } + assert_match(%r{\AHTTP/\d.\d 400 Bad Request}, res) + s = log.call.each_line.grep(/ERROR bad header/)[0] + assert_match(DumpPat, s) + assert_not_match(CtrlPat, s) + } + end +end diff --git a/tool/test/webrick/test_config.rb b/tool/test/webrick/test_config.rb new file mode 100644 index 0000000000..a54a667452 --- /dev/null +++ b/tool/test/webrick/test_config.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: false +require "test/unit" +require "webrick/config" + +class TestWEBrickConfig < Test::Unit::TestCase + def test_server_name_default + config = WEBrick::Config::General.dup + assert_equal(false, config.key?(:ServerName)) + assert_equal(WEBrick::Utils.getservername, config[:ServerName]) + assert_equal(true, config.key?(:ServerName)) + end + + def test_server_name_set_nil + config = WEBrick::Config::General.dup.update(ServerName: nil) + assert_equal(nil, config[:ServerName]) + end +end diff --git a/tool/test/webrick/test_cookie.rb b/tool/test/webrick/test_cookie.rb new file mode 100644 index 0000000000..e46185f127 --- /dev/null +++ b/tool/test/webrick/test_cookie.rb @@ -0,0 +1,141 @@ +# frozen_string_literal: false +require "test/unit" +require "webrick/cookie" + +class TestWEBrickCookie < Test::Unit::TestCase + def test_new + cookie = WEBrick::Cookie.new("foo","bar") + assert_equal("foo", cookie.name) + assert_equal("bar", cookie.value) + assert_equal("foo=bar", cookie.to_s) + end + + def test_time + cookie = WEBrick::Cookie.new("foo","bar") + t = 1000000000 + cookie.max_age = t + assert_match(t.to_s, cookie.to_s) + + cookie = WEBrick::Cookie.new("foo","bar") + t = Time.at(1000000000) + cookie.expires = t + assert_equal(Time, cookie.expires.class) + assert_equal(t, cookie.expires) + ts = t.httpdate + cookie.expires = ts + assert_equal(Time, cookie.expires.class) + assert_equal(t, cookie.expires) + assert_match(ts, cookie.to_s) + end + + def test_parse + data = "" + data << '$Version="1"; ' + data << 'Customer="WILE_E_COYOTE"; $Path="/acme"; ' + data << 'Part_Number="Rocket_Launcher_0001"; $Path="/acme"; ' + data << 'Shipping="FedEx"; $Path="/acme"' + cookies = WEBrick::Cookie.parse(data) + assert_equal(3, cookies.size) + assert_equal(1, cookies[0].version) + assert_equal("Customer", cookies[0].name) + assert_equal("WILE_E_COYOTE", cookies[0].value) + assert_equal("/acme", cookies[0].path) + assert_equal(1, cookies[1].version) + assert_equal("Part_Number", cookies[1].name) + assert_equal("Rocket_Launcher_0001", cookies[1].value) + assert_equal(1, cookies[2].version) + assert_equal("Shipping", cookies[2].name) + assert_equal("FedEx", cookies[2].value) + + data = "hoge=moge; __div__session=9865ecfd514be7f7" + cookies = WEBrick::Cookie.parse(data) + assert_equal(2, cookies.size) + assert_equal(0, cookies[0].version) + assert_equal("hoge", cookies[0].name) + assert_equal("moge", cookies[0].value) + assert_equal("__div__session", cookies[1].name) + assert_equal("9865ecfd514be7f7", cookies[1].value) + + # don't allow ,-separator + data = "hoge=moge, __div__session=9865ecfd514be7f7" + cookies = WEBrick::Cookie.parse(data) + assert_equal(1, cookies.size) + assert_equal(0, cookies[0].version) + assert_equal("hoge", cookies[0].name) + assert_equal("moge, __div__session=9865ecfd514be7f7", cookies[0].value) + end + + def test_parse_no_whitespace + data = [ + '$Version="1"; ', + 'Customer="WILE_E_COYOTE";$Path="/acme";', # no SP between cookie-string + 'Part_Number="Rocket_Launcher_0001";$Path="/acme";', # no SP between cookie-string + 'Shipping="FedEx";$Path="/acme"' + ].join + cookies = WEBrick::Cookie.parse(data) + assert_equal(1, cookies.size) + end + + def test_parse_too_much_whitespaces + # According to RFC6265, + # cookie-string = cookie-pair *( ";" SP cookie-pair ) + # So single 0x20 is needed after ';'. We allow multiple spaces here for + # compatibility with older WEBrick versions. + data = [ + '$Version="1"; ', + 'Customer="WILE_E_COYOTE";$Path="/acme"; ', # no SP between cookie-string + 'Part_Number="Rocket_Launcher_0001";$Path="/acme"; ', # no SP between cookie-string + 'Shipping="FedEx";$Path="/acme"' + ].join + cookies = WEBrick::Cookie.parse(data) + assert_equal(3, cookies.size) + end + + def test_parse_set_cookie + data = %(Customer="WILE_E_COYOTE"; Version="1"; Path="/acme") + cookie = WEBrick::Cookie.parse_set_cookie(data) + assert_equal("Customer", cookie.name) + assert_equal("WILE_E_COYOTE", cookie.value) + assert_equal(1, cookie.version) + assert_equal("/acme", cookie.path) + + data = %(Shipping="FedEx"; Version="1"; Path="/acme"; Secure) + cookie = WEBrick::Cookie.parse_set_cookie(data) + assert_equal("Shipping", cookie.name) + assert_equal("FedEx", cookie.value) + assert_equal(1, cookie.version) + assert_equal("/acme", cookie.path) + assert_equal(true, cookie.secure) + end + + def test_parse_set_cookies + data = %(Shipping="FedEx"; Version="1"; Path="/acme"; Secure) + data << %(, CUSTOMER=WILE_E_COYOTE; path=/; expires=Wednesday, 09-Nov-99 23:12:40 GMT; path=/; Secure) + data << %(, name="Aaron"; Version="1"; path="/acme") + cookies = WEBrick::Cookie.parse_set_cookies(data) + assert_equal(3, cookies.length) + + fed_ex = cookies.find { |c| c.name == 'Shipping' } + assert_not_nil(fed_ex) + assert_equal("Shipping", fed_ex.name) + assert_equal("FedEx", fed_ex.value) + assert_equal(1, fed_ex.version) + assert_equal("/acme", fed_ex.path) + assert_equal(true, fed_ex.secure) + + name = cookies.find { |c| c.name == 'name' } + assert_not_nil(name) + assert_equal("name", name.name) + assert_equal("Aaron", name.value) + assert_equal(1, name.version) + assert_equal("/acme", name.path) + + customer = cookies.find { |c| c.name == 'CUSTOMER' } + assert_not_nil(customer) + assert_equal("CUSTOMER", customer.name) + assert_equal("WILE_E_COYOTE", customer.value) + assert_equal(0, customer.version) + assert_equal("/", customer.path) + assert_equal(Time.utc(1999, 11, 9, 23, 12, 40), customer.expires) + end +end diff --git a/tool/test/webrick/test_do_not_reverse_lookup.rb b/tool/test/webrick/test_do_not_reverse_lookup.rb new file mode 100644 index 0000000000..efcb5a9299 --- /dev/null +++ b/tool/test/webrick/test_do_not_reverse_lookup.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: false +require "test/unit" +require "webrick" +require_relative "utils" + +class TestDoNotReverseLookup < Test::Unit::TestCase + class DNRL < WEBrick::GenericServer + def run(sock) + sock << sock.do_not_reverse_lookup.to_s + end + end + + @@original_do_not_reverse_lookup_value = Socket.do_not_reverse_lookup + + def teardown + Socket.do_not_reverse_lookup = @@original_do_not_reverse_lookup_value + end + + def do_not_reverse_lookup?(config) + result = nil + TestWEBrick.start_server(DNRL, config) do |server, addr, port, log| + TCPSocket.open(addr, port) do |sock| + result = {'true' => true, 'false' => false}[sock.gets] + end + end + result + end + + # +--------------------------------------------------------------------------+ + # | Expected interaction between Socket.do_not_reverse_lookup | + # | and WEBrick::Config::General[:DoNotReverseLookup] | + # +----------------------------+---------------------------------------------+ + # | |WEBrick::Config::General[:DoNotReverseLookup]| + # +----------------------------+--------------+---------------+--------------+ + # |Socket.do_not_reverse_lookup| TRUE | FALSE | NIL | + # +----------------------------+--------------+---------------+--------------+ + # | TRUE | true | false | true | + # +----------------------------+--------------+---------------+--------------+ + # | FALSE | true | false | false | + # +----------------------------+--------------+---------------+--------------+ + + def test_socket_dnrl_true_server_dnrl_true + Socket.do_not_reverse_lookup = true + assert_equal(true, do_not_reverse_lookup?(:DoNotReverseLookup => true)) + end + + def test_socket_dnrl_true_server_dnrl_false + Socket.do_not_reverse_lookup = true + assert_equal(false, do_not_reverse_lookup?(:DoNotReverseLookup => false)) + end + + def test_socket_dnrl_true_server_dnrl_nil + Socket.do_not_reverse_lookup = true + assert_equal(true, do_not_reverse_lookup?(:DoNotReverseLookup => nil)) + end + + def test_socket_dnrl_false_server_dnrl_true + Socket.do_not_reverse_lookup = false + assert_equal(true, do_not_reverse_lookup?(:DoNotReverseLookup => true)) + end + + def test_socket_dnrl_false_server_dnrl_false + Socket.do_not_reverse_lookup = false + assert_equal(false, do_not_reverse_lookup?(:DoNotReverseLookup => false)) + end + + def test_socket_dnrl_false_server_dnrl_nil + Socket.do_not_reverse_lookup = false + assert_equal(false, do_not_reverse_lookup?(:DoNotReverseLookup => nil)) + end +end diff --git a/tool/test/webrick/test_filehandler.rb b/tool/test/webrick/test_filehandler.rb new file mode 100644 index 0000000000..146d8ce792 --- /dev/null +++ b/tool/test/webrick/test_filehandler.rb @@ -0,0 +1,403 @@ +# frozen_string_literal: false +require "test/unit" +require_relative "utils.rb" +require "webrick" +require "stringio" +require "tmpdir" + +class WEBrick::TestFileHandler < Test::Unit::TestCase + def teardown + WEBrick::Utils::TimeoutHandler.terminate + super + end + + def default_file_handler(filename) + klass = WEBrick::HTTPServlet::DefaultFileHandler + klass.new(WEBrick::Config::HTTP, filename) + end + + def windows? + File.directory?("\\") + end + + def get_res_body(res) + sio = StringIO.new + sio.binmode + res.send_body(sio) + sio.string + end + + def make_range_request(range_spec) + msg = <<-END_OF_REQUEST + GET / HTTP/1.0 + Range: #{range_spec} + + END_OF_REQUEST + return StringIO.new(msg.gsub(/^ {6}/, "")) + end + + def make_range_response(file, range_spec) + req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) + req.parse(make_range_request(range_spec)) + res = WEBrick::HTTPResponse.new(WEBrick::Config::HTTP) + size = File.size(file) + handler = default_file_handler(file) + handler.make_partial_content(req, res, file, size) + return res + end + + def test_make_partial_content + filename = __FILE__ + filesize = File.size(filename) + + res = make_range_response(filename, "bytes=#{filesize-100}-") + assert_match(%r{^text/plain}, res["content-type"]) + assert_equal(100, get_res_body(res).size) + + res = make_range_response(filename, "bytes=-100") + assert_match(%r{^text/plain}, res["content-type"]) + assert_equal(100, get_res_body(res).size) + + res = make_range_response(filename, "bytes=0-99") + assert_match(%r{^text/plain}, res["content-type"]) + assert_equal(100, get_res_body(res).size) + + res = make_range_response(filename, "bytes=100-199") + assert_match(%r{^text/plain}, res["content-type"]) + assert_equal(100, get_res_body(res).size) + + res = make_range_response(filename, "bytes=0-0") + assert_match(%r{^text/plain}, res["content-type"]) + assert_equal(1, get_res_body(res).size) + + res = make_range_response(filename, "bytes=-1") + assert_match(%r{^text/plain}, res["content-type"]) + assert_equal(1, get_res_body(res).size) + + res = make_range_response(filename, "bytes=0-0, -2") + assert_match(%r{^multipart/byteranges}, res["content-type"]) + body = get_res_body(res) + boundary = /; boundary=(.+)/.match(res['content-type'])[1] + off = filesize - 2 + last = filesize - 1 + + exp = "--#{boundary}\r\n" \ + "Content-Type: text/plain\r\n" \ + "Content-Range: bytes 0-0/#{filesize}\r\n" \ + "\r\n" \ + "#{IO.read(__FILE__, 1)}\r\n" \ + "--#{boundary}\r\n" \ + "Content-Type: text/plain\r\n" \ + "Content-Range: bytes #{off}-#{last}/#{filesize}\r\n" \ + "\r\n" \ + "#{IO.read(__FILE__, 2, off)}\r\n" \ + "--#{boundary}--\r\n" + assert_equal exp, body + end + + def test_filehandler + config = { :DocumentRoot => File.dirname(__FILE__), } + this_file = File.basename(__FILE__) + filesize = File.size(__FILE__) + this_data = File.binread(__FILE__) + range = nil + bug2593 = '[ruby-dev:40030]' + + TestWEBrick.start_httpserver(config) do |server, addr, port, log| + begin + server[:DocumentRootOptions][:NondisclosureName] = [] + http = Net::HTTP.new(addr, port) + req = Net::HTTP::Get.new("/") + http.request(req){|res| + assert_equal("200", res.code, log.call) + assert_equal("text/html", res.content_type, log.call) + assert_match(/HREF="#{this_file}"/, res.body, log.call) + } + req = Net::HTTP::Get.new("/#{this_file}") + http.request(req){|res| + assert_equal("200", res.code, log.call) + assert_equal("text/plain", res.content_type, log.call) + assert_equal(this_data, res.body, log.call) + } + + req = Net::HTTP::Get.new("/#{this_file}", "range"=>"bytes=#{filesize-100}-") + http.request(req){|res| + assert_equal("206", res.code, log.call) + assert_equal("text/plain", res.content_type, log.call) + assert_nothing_raised(bug2593) {range = res.content_range} + assert_equal((filesize-100)..(filesize-1), range, log.call) + assert_equal(this_data[-100..-1], res.body, log.call) + } + + req = Net::HTTP::Get.new("/#{this_file}", "range"=>"bytes=-100") + http.request(req){|res| + assert_equal("206", res.code, log.call) + assert_equal("text/plain", res.content_type, log.call) + assert_nothing_raised(bug2593) {range = res.content_range} + assert_equal((filesize-100)..(filesize-1), range, log.call) + assert_equal(this_data[-100..-1], res.body, log.call) + } + + req = Net::HTTP::Get.new("/#{this_file}", "range"=>"bytes=0-99") + http.request(req){|res| + assert_equal("206", res.code, log.call) + assert_equal("text/plain", res.content_type, log.call) + assert_nothing_raised(bug2593) {range = res.content_range} + assert_equal(0..99, range, log.call) + assert_equal(this_data[0..99], res.body, log.call) + } + + req = Net::HTTP::Get.new("/#{this_file}", "range"=>"bytes=100-199") + http.request(req){|res| + assert_equal("206", res.code, log.call) + assert_equal("text/plain", res.content_type, log.call) + assert_nothing_raised(bug2593) {range = res.content_range} + assert_equal(100..199, range, log.call) + assert_equal(this_data[100..199], res.body, log.call) + } + + req = Net::HTTP::Get.new("/#{this_file}", "range"=>"bytes=0-0") + http.request(req){|res| + assert_equal("206", res.code, log.call) + assert_equal("text/plain", res.content_type, log.call) + assert_nothing_raised(bug2593) {range = res.content_range} + assert_equal(0..0, range, log.call) + assert_equal(this_data[0..0], res.body, log.call) + } + + req = Net::HTTP::Get.new("/#{this_file}", "range"=>"bytes=-1") + http.request(req){|res| + assert_equal("206", res.code, log.call) + assert_equal("text/plain", res.content_type, log.call) + assert_nothing_raised(bug2593) {range = res.content_range} + assert_equal((filesize-1)..(filesize-1), range, log.call) + assert_equal(this_data[-1, 1], res.body, log.call) + } + + req = Net::HTTP::Get.new("/#{this_file}", "range"=>"bytes=0-0, -2") + http.request(req){|res| + assert_equal("206", res.code, log.call) + assert_equal("multipart/byteranges", res.content_type, log.call) + } + ensure + server[:DocumentRootOptions].delete :NondisclosureName + end + end + end + + def test_non_disclosure_name + config = { :DocumentRoot => File.dirname(__FILE__), } + log_tester = lambda {|log, access_log| + log = log.reject {|s| /ERROR `.*\' not found\./ =~ s } + log = log.reject {|s| /WARN the request refers nondisclosure name/ =~ s } + assert_equal([], log) + } + this_file = File.basename(__FILE__) + TestWEBrick.start_httpserver(config, log_tester) do |server, addr, port, log| + http = Net::HTTP.new(addr, port) + doc_root_opts = server[:DocumentRootOptions] + doc_root_opts[:NondisclosureName] = %w(.ht* *~ test_*) + req = Net::HTTP::Get.new("/") + http.request(req){|res| + assert_equal("200", res.code, log.call) + assert_equal("text/html", res.content_type, log.call) + assert_no_match(/HREF="#{File.basename(__FILE__)}"/, res.body) + } + req = Net::HTTP::Get.new("/#{this_file}") + http.request(req){|res| + assert_equal("404", res.code, log.call) + } + doc_root_opts[:NondisclosureName] = %w(.ht* *~ TEST_*) + http.request(req){|res| + assert_equal("404", res.code, log.call) + } + end + end + + def test_directory_traversal + return if File.executable?(__FILE__) # skip on strange file system + + config = { :DocumentRoot => File.dirname(__FILE__), } + log_tester = lambda {|log, access_log| + log = log.reject {|s| /ERROR bad URI/ =~ s } + log = log.reject {|s| /ERROR `.*\' not found\./ =~ s } + assert_equal([], log) + } + TestWEBrick.start_httpserver(config, log_tester) do |server, addr, port, log| + http = Net::HTTP.new(addr, port) + req = Net::HTTP::Get.new("/../../") + http.request(req){|res| assert_equal("400", res.code, log.call) } + req = Net::HTTP::Get.new("/..%5c../#{File.basename(__FILE__)}") + http.request(req){|res| assert_equal(windows? ? "200" : "404", res.code, log.call) } + req = Net::HTTP::Get.new("/..%5c..%5cruby.c") + http.request(req){|res| assert_equal("404", res.code, log.call) } + end + end + + def test_unwise_in_path + if windows? + config = { :DocumentRoot => File.dirname(__FILE__), } + TestWEBrick.start_httpserver(config) do |server, addr, port, log| + http = Net::HTTP.new(addr, port) + req = Net::HTTP::Get.new("/..%5c..") + http.request(req){|res| assert_equal("301", res.code, log.call) } + end + end + end + + def test_short_filename + return if File.executable?(__FILE__) # skip on strange file system + return if /mswin/ =~ RUBY_PLATFORM && ENV.key?('GITHUB_ACTIONS') # not working from the beginning + + config = { + :CGIInterpreter => TestWEBrick::RubyBin, + :DocumentRoot => File.dirname(__FILE__), + :CGIPathEnv => ENV['PATH'], + } + log_tester = lambda {|log, access_log| + log = log.reject {|s| /ERROR `.*\' not found\./ =~ s } + log = log.reject {|s| /WARN the request refers nondisclosure name/ =~ s } + assert_equal([], log) + } + TestWEBrick.start_httpserver(config, log_tester) do |server, addr, port, log| + http = Net::HTTP.new(addr, port) + if windows? + root = config[:DocumentRoot].tr("/", "\\") + fname = IO.popen(%W[dir /x #{root}\\webrick_long_filename.cgi], encoding: "binary", &:read) + fname.sub!(/\A.*$^$.*$^$/m, '') + if fname + fname = fname[/\s(w.+?cgi)\s/i, 1] + fname.downcase! + end + else + fname = "webric~1.cgi" + end + req = Net::HTTP::Get.new("/#{fname}/test") + http.request(req) do |res| + if windows? + assert_equal("200", res.code, log.call) + assert_equal("/test", res.body, log.call) + else + assert_equal("404", res.code, log.call) + end + end + + req = Net::HTTP::Get.new("/.htaccess") + http.request(req) {|res| assert_equal("404", res.code, log.call) } + req = Net::HTTP::Get.new("/htacce~1") + http.request(req) {|res| assert_equal("404", res.code, log.call) } + req = Net::HTTP::Get.new("/HTACCE~1") + http.request(req) {|res| assert_equal("404", res.code, log.call) } + end + end + + def test_multibyte_char_in_path + if Encoding.default_external == Encoding.find('US-ASCII') + reset_encoding = true + verb = $VERBOSE + $VERBOSE = false + Encoding.default_external = Encoding.find('UTF-8') + end + + c = "\u00a7" + begin + c = c.encode('filesystem') + rescue EncodingError + c = c.b + end + Dir.mktmpdir(c) do |dir| + basename = "#{c}.txt" + File.write("#{dir}/#{basename}", "test_multibyte_char_in_path") + Dir.mkdir("#{dir}/#{c}") + File.write("#{dir}/#{c}/#{basename}", "nested") + config = { + :DocumentRoot => dir, + :DirectoryIndex => [basename], + } + TestWEBrick.start_httpserver(config) do |server, addr, port, log| + http = Net::HTTP.new(addr, port) + path = "/#{basename}" + req = Net::HTTP::Get.new(WEBrick::HTTPUtils::escape(path)) + http.request(req){|res| assert_equal("200", res.code, log.call + "\nFilesystem encoding is #{Encoding.find('filesystem')}") } + path = "/#{c}/#{basename}" + req = Net::HTTP::Get.new(WEBrick::HTTPUtils::escape(path)) + http.request(req){|res| assert_equal("200", res.code, log.call) } + req = Net::HTTP::Get.new('/') + http.request(req){|res| + assert_equal("test_multibyte_char_in_path", res.body, log.call) + } + end + end + ensure + if reset_encoding + Encoding.default_external = Encoding.find('US-ASCII') + $VERBOSE = verb + end + end + + def test_script_disclosure + return if File.executable?(__FILE__) # skip on strange file system + + config = { + :CGIInterpreter => TestWEBrick::RubyBinArray, + :DocumentRoot => File.dirname(__FILE__), + :CGIPathEnv => ENV['PATH'], + :RequestCallback => Proc.new{|req, res| + def req.meta_vars + meta = super + meta["RUBYLIB"] = $:.join(File::PATH_SEPARATOR) + meta[RbConfig::CONFIG['LIBPATHENV']] = ENV[RbConfig::CONFIG['LIBPATHENV']] if RbConfig::CONFIG['LIBPATHENV'] + return meta + end + }, + } + log_tester = lambda {|log, access_log| + log = log.reject {|s| /ERROR `.*\' not found\./ =~ s } + assert_equal([], log) + } + TestWEBrick.start_httpserver(config, log_tester) do |server, addr, port, log| + http = Net::HTTP.new(addr, port) + http.read_timeout = EnvUtil.apply_timeout_scale(60) + http.write_timeout = EnvUtil.apply_timeout_scale(60) if http.respond_to?(:write_timeout=) + + req = Net::HTTP::Get.new("/webrick.cgi/test") + http.request(req) do |res| + assert_equal("200", res.code, log.call) + assert_equal("/test", res.body, log.call) + end + + resok = windows? + response_assertion = Proc.new do |res| + if resok + assert_equal("200", res.code, log.call) + assert_equal("/test", res.body, log.call) + else + assert_equal("404", res.code, log.call) + end + end + req = Net::HTTP::Get.new("/webrick.cgi%20/test") + http.request(req, &response_assertion) + req = Net::HTTP::Get.new("/webrick.cgi./test") + http.request(req, &response_assertion) + resok &&= File.exist?(__FILE__+"::$DATA") + req = Net::HTTP::Get.new("/webrick.cgi::$DATA/test") + http.request(req, &response_assertion) + end + end + + def test_erbhandler + config = { :DocumentRoot => File.dirname(__FILE__) } + log_tester = lambda {|log, access_log| + log = log.reject {|s| /ERROR `.*\' not found\./ =~ s } + assert_equal([], log) + } + TestWEBrick.start_httpserver(config, log_tester) do |server, addr, port, log| + http = Net::HTTP.new(addr, port) + req = Net::HTTP::Get.new("/webrick.rhtml") + http.request(req) do |res| + assert_equal("200", res.code, log.call) + assert_match %r!\Areq to http://[^/]+/webrick\.rhtml {}\n!, res.body + end + end + end +end diff --git a/tool/test/webrick/test_htgroup.rb b/tool/test/webrick/test_htgroup.rb new file mode 100644 index 0000000000..8749711df5 --- /dev/null +++ b/tool/test/webrick/test_htgroup.rb @@ -0,0 +1,19 @@ +require "tempfile" +require "test/unit" +require "webrick/httpauth/htgroup" + +class TestHtgroup < Test::Unit::TestCase + def test_htgroup + Tempfile.create('test_htgroup') do |tmpfile| + tmpfile.close + tmp_group = WEBrick::HTTPAuth::Htgroup.new(tmpfile.path) + tmp_group.add 'superheroes', %w[spiderman batman] + tmp_group.add 'supervillains', %w[joker] + tmp_group.flush + + htgroup = WEBrick::HTTPAuth::Htgroup.new(tmpfile.path) + assert_equal(htgroup.members('superheroes'), %w[spiderman batman]) + assert_equal(htgroup.members('supervillains'), %w[joker]) + end + end +end diff --git a/tool/test/webrick/test_htmlutils.rb b/tool/test/webrick/test_htmlutils.rb new file mode 100644 index 0000000000..ae1b8efa95 --- /dev/null +++ b/tool/test/webrick/test_htmlutils.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: false +require "test/unit" +require "webrick/htmlutils" + +class TestWEBrickHTMLUtils < Test::Unit::TestCase + include WEBrick::HTMLUtils + + def test_escape + assert_equal("foo", escape("foo")) + assert_equal("foo bar", escape("foo bar")) + assert_equal("foo&bar", escape("foo&bar")) + assert_equal("foo"bar", escape("foo\"bar")) + assert_equal("foo>bar", escape("foo>bar")) + assert_equal("foo<bar", escape("foo<bar")) + assert_equal("\u{3053 3093 306B 3061 306F}", escape("\u{3053 3093 306B 3061 306F}")) + bug8425 = '[Bug #8425] [ruby-core:55052]' + assert_nothing_raised(ArgumentError, Encoding::CompatibilityError, bug8425) { + assert_equal("\u{3053 3093 306B}\xff<", escape("\u{3053 3093 306B}\xff<")) + } + end +end diff --git a/tool/test/webrick/test_httpauth.rb b/tool/test/webrick/test_httpauth.rb new file mode 100644 index 0000000000..9fe8af8be2 --- /dev/null +++ b/tool/test/webrick/test_httpauth.rb @@ -0,0 +1,366 @@ +# frozen_string_literal: false +require "test/unit" +require "net/http" +require "tempfile" +require "webrick" +require "webrick/httpauth/basicauth" +require "stringio" +require_relative "utils" + +class TestWEBrickHTTPAuth < Test::Unit::TestCase + def teardown + WEBrick::Utils::TimeoutHandler.terminate + super + end + + def test_basic_auth + log_tester = lambda {|log, access_log| + assert_equal(1, log.length) + assert_match(/ERROR WEBrick::HTTPStatus::Unauthorized/, log[0]) + } + TestWEBrick.start_httpserver({}, log_tester) {|server, addr, port, log| + realm = "WEBrick's realm" + path = "/basic_auth" + + server.mount_proc(path){|req, res| + WEBrick::HTTPAuth.basic_auth(req, res, realm){|user, pass| + user == "webrick" && pass == "supersecretpassword" + } + res.body = "hoge" + } + http = Net::HTTP.new(addr, port) + g = Net::HTTP::Get.new(path) + g.basic_auth("webrick", "supersecretpassword") + http.request(g){|res| assert_equal("hoge", res.body, log.call)} + g.basic_auth("webrick", "not super") + http.request(g){|res| assert_not_equal("hoge", res.body, log.call)} + } + end + + def test_basic_auth_sha + Tempfile.create("test_webrick_auth") {|tmpfile| + tmpfile.puts("webrick:{SHA}GJYFRpBbdchp595jlh3Bhfmgp8k=") + tmpfile.flush + assert_raise(NotImplementedError){ + WEBrick::HTTPAuth::Htpasswd.new(tmpfile.path) + } + } + end + + def test_basic_auth_md5 + Tempfile.create("test_webrick_auth") {|tmpfile| + tmpfile.puts("webrick:$apr1$IOVMD/..$rmnOSPXr0.wwrLPZHBQZy0") + tmpfile.flush + assert_raise(NotImplementedError){ + WEBrick::HTTPAuth::Htpasswd.new(tmpfile.path) + } + } + end + + [nil, :crypt, :bcrypt].each do |hash_algo| + # OpenBSD does not support insecure DES-crypt + next if /openbsd/ =~ RUBY_PLATFORM && hash_algo != :bcrypt + + begin + case hash_algo + when :crypt + # require 'string/crypt' + when :bcrypt + require 'bcrypt' + end + rescue LoadError + next + end + + define_method(:"test_basic_auth_htpasswd_#{hash_algo}") do + log_tester = lambda {|log, access_log| + log.reject! {|line| /\A\s*\z/ =~ line } + pats = [ + /ERROR Basic WEBrick's realm: webrick: password unmatch\./, + /ERROR WEBrick::HTTPStatus::Unauthorized/ + ] + pats.each {|pat| + assert(!log.grep(pat).empty?, "webrick log doesn't have expected error: #{pat.inspect}") + log.reject! {|line| pat =~ line } + } + assert_equal([], log) + } + TestWEBrick.start_httpserver({}, log_tester) {|server, addr, port, log| + realm = "WEBrick's realm" + path = "/basic_auth2" + + Tempfile.create("test_webrick_auth") {|tmpfile| + tmpfile.close + tmp_pass = WEBrick::HTTPAuth::Htpasswd.new(tmpfile.path, password_hash: hash_algo) + tmp_pass.set_passwd(realm, "webrick", "supersecretpassword") + tmp_pass.set_passwd(realm, "foo", "supersecretpassword") + tmp_pass.flush + + htpasswd = WEBrick::HTTPAuth::Htpasswd.new(tmpfile.path, password_hash: hash_algo) + users = [] + htpasswd.each{|user, pass| users << user } + assert_equal(2, users.size, log.call) + assert(users.member?("webrick"), log.call) + assert(users.member?("foo"), log.call) + + server.mount_proc(path){|req, res| + auth = WEBrick::HTTPAuth::BasicAuth.new( + :Realm => realm, :UserDB => htpasswd, + :Logger => server.logger + ) + auth.authenticate(req, res) + res.body = "hoge" + } + http = Net::HTTP.new(addr, port) + g = Net::HTTP::Get.new(path) + g.basic_auth("webrick", "supersecretpassword") + http.request(g){|res| assert_equal("hoge", res.body, log.call)} + g.basic_auth("webrick", "not super") + http.request(g){|res| assert_not_equal("hoge", res.body, log.call)} + } + } + end + + define_method(:"test_basic_auth_bad_username_htpasswd_#{hash_algo}") do + log_tester = lambda {|log, access_log| + assert_equal(2, log.length) + assert_match(/ERROR Basic WEBrick's realm: foo\\ebar: the user is not allowed\./, log[0]) + assert_match(/ERROR WEBrick::HTTPStatus::Unauthorized/, log[1]) + } + TestWEBrick.start_httpserver({}, log_tester) {|server, addr, port, log| + realm = "WEBrick's realm" + path = "/basic_auth" + + Tempfile.create("test_webrick_auth") {|tmpfile| + tmpfile.close + tmp_pass = WEBrick::HTTPAuth::Htpasswd.new(tmpfile.path, password_hash: hash_algo) + tmp_pass.set_passwd(realm, "webrick", "supersecretpassword") + tmp_pass.set_passwd(realm, "foo", "supersecretpassword") + tmp_pass.flush + + htpasswd = WEBrick::HTTPAuth::Htpasswd.new(tmpfile.path, password_hash: hash_algo) + users = [] + htpasswd.each{|user, pass| users << user } + server.mount_proc(path){|req, res| + auth = WEBrick::HTTPAuth::BasicAuth.new( + :Realm => realm, :UserDB => htpasswd, + :Logger => server.logger + ) + auth.authenticate(req, res) + res.body = "hoge" + } + http = Net::HTTP.new(addr, port) + g = Net::HTTP::Get.new(path) + g.basic_auth("foo\ebar", "passwd") + http.request(g){|res| assert_not_equal("hoge", res.body, log.call) } + } + } + end + end + + DIGESTRES_ = / + ([a-zA-Z\-]+) + [ \t]*(?:\r\n[ \t]*)* + = + [ \t]*(?:\r\n[ \t]*)* + (?: + "((?:[^"]+|\\[\x00-\x7F])*)" | + ([!\#$%&'*+\-.0-9A-Z^_`a-z|~]+) + )/x + + def test_digest_auth + log_tester = lambda {|log, access_log| + log.reject! {|line| /\A\s*\z/ =~ line } + pats = [ + /ERROR Digest WEBrick's realm: no credentials in the request\./, + /ERROR WEBrick::HTTPStatus::Unauthorized/, + /ERROR Digest WEBrick's realm: webrick: digest unmatch\./ + ] + pats.each {|pat| + assert(!log.grep(pat).empty?, "webrick log doesn't have expected error: #{pat.inspect}") + log.reject! {|line| pat =~ line } + } + assert_equal([], log) + } + TestWEBrick.start_httpserver({}, log_tester) {|server, addr, port, log| + realm = "WEBrick's realm" + path = "/digest_auth" + + Tempfile.create("test_webrick_auth") {|tmpfile| + tmpfile.close + tmp_pass = WEBrick::HTTPAuth::Htdigest.new(tmpfile.path) + tmp_pass.set_passwd(realm, "webrick", "supersecretpassword") + tmp_pass.set_passwd(realm, "foo", "supersecretpassword") + tmp_pass.flush + + htdigest = WEBrick::HTTPAuth::Htdigest.new(tmpfile.path) + users = [] + htdigest.each{|user, pass| users << user } + assert_equal(2, users.size, log.call) + assert(users.member?("webrick"), log.call) + assert(users.member?("foo"), log.call) + + auth = WEBrick::HTTPAuth::DigestAuth.new( + :Realm => realm, :UserDB => htdigest, + :Algorithm => 'MD5', + :Logger => server.logger + ) + server.mount_proc(path){|req, res| + auth.authenticate(req, res) + res.body = "hoge" + } + + Net::HTTP.start(addr, port) do |http| + g = Net::HTTP::Get.new(path) + params = {} + http.request(g) do |res| + assert_equal('401', res.code, log.call) + res["www-authenticate"].scan(DIGESTRES_) do |key, quoted, token| + params[key.downcase] = token || quoted.delete('\\') + end + params['uri'] = "http://#{addr}:#{port}#{path}" + end + + g['Authorization'] = credentials_for_request('webrick', "supersecretpassword", params) + http.request(g){|res| assert_equal("hoge", res.body, log.call)} + + params['algorithm'].downcase! #4936 + g['Authorization'] = credentials_for_request('webrick', "supersecretpassword", params) + http.request(g){|res| assert_equal("hoge", res.body, log.call)} + + g['Authorization'] = credentials_for_request('webrick', "not super", params) + http.request(g){|res| assert_not_equal("hoge", res.body, log.call)} + end + } + } + end + + def test_digest_auth_int + log_tester = lambda {|log, access_log| + log.reject! {|line| /\A\s*\z/ =~ line } + pats = [ + /ERROR Digest wb auth-int realm: no credentials in the request\./, + /ERROR WEBrick::HTTPStatus::Unauthorized/, + /ERROR Digest wb auth-int realm: foo: digest unmatch\./ + ] + pats.each {|pat| + assert(!log.grep(pat).empty?, "webrick log doesn't have expected error: #{pat.inspect}") + log.reject! {|line| pat =~ line } + } + assert_equal([], log) + } + TestWEBrick.start_httpserver({}, log_tester) {|server, addr, port, log| + realm = "wb auth-int realm" + path = "/digest_auth_int" + + Tempfile.create("test_webrick_auth_int") {|tmpfile| + tmpfile.close + tmp_pass = WEBrick::HTTPAuth::Htdigest.new(tmpfile.path) + tmp_pass.set_passwd(realm, "foo", "Hunter2") + tmp_pass.flush + + htdigest = WEBrick::HTTPAuth::Htdigest.new(tmpfile.path) + users = [] + htdigest.each{|user, pass| users << user } + assert_equal %w(foo), users + + auth = WEBrick::HTTPAuth::DigestAuth.new( + :Realm => realm, :UserDB => htdigest, + :Algorithm => 'MD5', + :Logger => server.logger, + :Qop => %w(auth-int), + ) + server.mount_proc(path){|req, res| + auth.authenticate(req, res) + res.body = "bbb" + } + Net::HTTP.start(addr, port) do |http| + post = Net::HTTP::Post.new(path) + params = {} + data = 'hello=world' + body = StringIO.new(data) + post.content_length = data.bytesize + post['Content-Type'] = 'application/x-www-form-urlencoded' + post.body_stream = body + + http.request(post) do |res| + assert_equal('401', res.code, log.call) + res["www-authenticate"].scan(DIGESTRES_) do |key, quoted, token| + params[key.downcase] = token || quoted.delete('\\') + end + params['uri'] = "http://#{addr}:#{port}#{path}" + end + + body.rewind + cred = credentials_for_request('foo', 'Hunter3', params, body) + post['Authorization'] = cred + post.body_stream = body + http.request(post){|res| + assert_equal('401', res.code, log.call) + assert_not_equal("bbb", res.body, log.call) + } + + body.rewind + cred = credentials_for_request('foo', 'Hunter2', params, body) + post['Authorization'] = cred + post.body_stream = body + http.request(post){|res| assert_equal("bbb", res.body, log.call)} + end + } + } + end + + def test_digest_auth_invalid + digest_auth = WEBrick::HTTPAuth::DigestAuth.new(Realm: 'realm', UserDB: '') + + def digest_auth.error(fmt, *) + end + + def digest_auth.try_bad_request(len) + request = {"Authorization" => %[Digest a="#{'\b'*len}]} + authenticate request, nil + end + + bad_request = WEBrick::HTTPStatus::BadRequest + t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC) + assert_raise(bad_request) {digest_auth.try_bad_request(10)} + limit = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - t0) + [20, 50, 100, 200].each do |len| + assert_raise(bad_request) do + Timeout.timeout(len*limit) {digest_auth.try_bad_request(len)} + end + end + end + + private + def credentials_for_request(user, password, params, body = nil) + cnonce = "hoge" + nonce_count = 1 + ha1 = "#{user}:#{params['realm']}:#{password}" + if body + dig = Digest::MD5.new + while buf = body.read(16384) + dig.update(buf) + end + body.rewind + ha2 = "POST:#{params['uri']}:#{dig.hexdigest}" + else + ha2 = "GET:#{params['uri']}" + end + + request_digest = + "#{Digest::MD5.hexdigest(ha1)}:" \ + "#{params['nonce']}:#{'%08x' % nonce_count}:#{cnonce}:#{params['qop']}:" \ + "#{Digest::MD5.hexdigest(ha2)}" + "Digest username=\"#{user}\"" \ + ", realm=\"#{params['realm']}\"" \ + ", nonce=\"#{params['nonce']}\"" \ + ", uri=\"#{params['uri']}\"" \ + ", qop=#{params['qop']}" \ + ", nc=#{'%08x' % nonce_count}" \ + ", cnonce=\"#{cnonce}\"" \ + ", response=\"#{Digest::MD5.hexdigest(request_digest)}\"" \ + ", opaque=\"#{params['opaque']}\"" \ + ", algorithm=#{params['algorithm']}" + end +end diff --git a/tool/test/webrick/test_httpproxy.rb b/tool/test/webrick/test_httpproxy.rb new file mode 100644 index 0000000000..66dae6f6f6 --- /dev/null +++ b/tool/test/webrick/test_httpproxy.rb @@ -0,0 +1,467 @@ +# frozen_string_literal: false +require "test/unit" +require "net/http" +require "webrick" +require "webrick/httpproxy" +begin + require "webrick/ssl" + require "net/https" +rescue LoadError + # test_connect will be skipped +end +require File.expand_path("utils.rb", File.dirname(__FILE__)) + +class TestWEBrickHTTPProxy < Test::Unit::TestCase + def teardown + WEBrick::Utils::TimeoutHandler.terminate + super + end + + def test_fake_proxy + assert_nil(WEBrick::FakeProxyURI.scheme) + assert_nil(WEBrick::FakeProxyURI.host) + assert_nil(WEBrick::FakeProxyURI.port) + assert_nil(WEBrick::FakeProxyURI.path) + assert_nil(WEBrick::FakeProxyURI.userinfo) + assert_raise(NoMethodError){ WEBrick::FakeProxyURI.foo } + end + + def test_proxy + # Testing GET or POST to the proxy server + # Note that the proxy server works as the origin server. + # +------+ + # V | + # client -------> proxy ---+ + # GET / POST GET / POST + # + proxy_handler_called = request_handler_called = 0 + config = { + :ServerName => "localhost.localdomain", + :ProxyContentHandler => Proc.new{|req, res| proxy_handler_called += 1 }, + :RequestCallback => Proc.new{|req, res| request_handler_called += 1 } + } + TestWEBrick.start_httpproxy(config){|server, addr, port, log| + server.mount_proc("/"){|req, res| + res.body = "#{req.request_method} #{req.path} #{req.body}" + } + http = Net::HTTP.new(addr, port, addr, port) + + req = Net::HTTP::Get.new("/") + http.request(req){|res| + assert_equal("1.1 localhost.localdomain:#{port}", res["via"], log.call) + assert_equal("GET / ", res.body, log.call) + } + assert_equal(1, proxy_handler_called, log.call) + assert_equal(2, request_handler_called, log.call) + + req = Net::HTTP::Head.new("/") + http.request(req){|res| + assert_equal("1.1 localhost.localdomain:#{port}", res["via"], log.call) + assert_nil(res.body, log.call) + } + assert_equal(2, proxy_handler_called, log.call) + assert_equal(4, request_handler_called, log.call) + + req = Net::HTTP::Post.new("/") + req.body = "post-data" + req.content_type = "application/x-www-form-urlencoded" + http.request(req){|res| + assert_equal("1.1 localhost.localdomain:#{port}", res["via"], log.call) + assert_equal("POST / post-data", res.body, log.call) + } + assert_equal(3, proxy_handler_called, log.call) + assert_equal(6, request_handler_called, log.call) + } + end + + def test_no_proxy + # Testing GET or POST to the proxy server without proxy request. + # + # client -------> proxy + # GET / POST + # + proxy_handler_called = request_handler_called = 0 + config = { + :ServerName => "localhost.localdomain", + :ProxyContentHandler => Proc.new{|req, res| proxy_handler_called += 1 }, + :RequestCallback => Proc.new{|req, res| request_handler_called += 1 } + } + TestWEBrick.start_httpproxy(config){|server, addr, port, log| + server.mount_proc("/"){|req, res| + res.body = "#{req.request_method} #{req.path} #{req.body}" + } + http = Net::HTTP.new(addr, port) + + req = Net::HTTP::Get.new("/") + http.request(req){|res| + assert_nil(res["via"], log.call) + assert_equal("GET / ", res.body, log.call) + } + assert_equal(0, proxy_handler_called, log.call) + assert_equal(1, request_handler_called, log.call) + + req = Net::HTTP::Head.new("/") + http.request(req){|res| + assert_nil(res["via"], log.call) + assert_nil(res.body, log.call) + } + assert_equal(0, proxy_handler_called, log.call) + assert_equal(2, request_handler_called, log.call) + + req = Net::HTTP::Post.new("/") + req.content_type = "application/x-www-form-urlencoded" + req.body = "post-data" + http.request(req){|res| + assert_nil(res["via"], log.call) + assert_equal("POST / post-data", res.body, log.call) + } + assert_equal(0, proxy_handler_called, log.call) + assert_equal(3, request_handler_called, log.call) + } + end + + def test_big_bodies + require 'digest/md5' + rand_str = File.read(__FILE__) + rand_str.freeze + nr = 1024 ** 2 / rand_str.size # bigger works, too + exp = Digest::MD5.new + nr.times { exp.update(rand_str) } + exp = exp.hexdigest + TestWEBrick.start_httpserver do |o_server, o_addr, o_port, o_log| + o_server.mount_proc('/') do |req, res| + case req.request_method + when 'GET' + res['content-type'] = 'application/octet-stream' + if req.path == '/length' + res['content-length'] = (nr * rand_str.size).to_s + else + res.chunked = true + end + res.body = ->(socket) { nr.times { socket.write(rand_str) } } + when 'POST' + dig = Digest::MD5.new + req.body { |buf| dig.update(buf); buf.clear } + res['content-type'] = 'text/plain' + res['content-length'] = '32' + res.body = dig.hexdigest + end + end + + http = Net::HTTP.new(o_addr, o_port) + IO.pipe do |rd, wr| + headers = { + 'Content-Type' => 'application/octet-stream', + 'Transfer-Encoding' => 'chunked', + } + post = Net::HTTP::Post.new('/', headers) + th = Thread.new { nr.times { wr.write(rand_str) }; wr.close } + post.body_stream = rd + http.request(post) do |res| + assert_equal 'text/plain', res['content-type'] + assert_equal 32, res.content_length + assert_equal exp, res.body + end + assert_nil th.value + end + + TestWEBrick.start_httpproxy do |p_server, p_addr, p_port, p_log| + http = Net::HTTP.new(o_addr, o_port, p_addr, p_port) + http.request_get('/length') do |res| + assert_equal(nr * rand_str.size, res.content_length) + dig = Digest::MD5.new + res.read_body { |buf| dig.update(buf); buf.clear } + assert_equal exp, dig.hexdigest + end + http.request_get('/') do |res| + assert_predicate res, :chunked? + dig = Digest::MD5.new + res.read_body { |buf| dig.update(buf); buf.clear } + assert_equal exp, dig.hexdigest + end + + IO.pipe do |rd, wr| + headers = { + 'Content-Type' => 'application/octet-stream', + 'Content-Length' => (nr * rand_str.size).to_s, + } + post = Net::HTTP::Post.new('/', headers) + th = Thread.new { nr.times { wr.write(rand_str) }; wr.close } + post.body_stream = rd + http.request(post) do |res| + assert_equal 'text/plain', res['content-type'] + assert_equal 32, res.content_length + assert_equal exp, res.body + end + assert_nil th.value + end + + IO.pipe do |rd, wr| + headers = { + 'Content-Type' => 'application/octet-stream', + 'Transfer-Encoding' => 'chunked', + } + post = Net::HTTP::Post.new('/', headers) + th = Thread.new { nr.times { wr.write(rand_str) }; wr.close } + post.body_stream = rd + http.request(post) do |res| + assert_equal 'text/plain', res['content-type'] + assert_equal 32, res.content_length + assert_equal exp, res.body + end + assert_nil th.value + end + end + end + end if RUBY_VERSION >= '2.5' + + def test_http10_proxy_chunked + # Testing HTTP/1.0 client request and HTTP/1.1 chunked response + # from origin server. + # +------+ + # V | + # client -------> proxy ---+ + # GET GET + # HTTP/1.0 HTTP/1.1 + # non-chunked chunked + # + proxy_handler_called = request_handler_called = 0 + config = { + :ServerName => "localhost.localdomain", + :ProxyContentHandler => Proc.new{|req, res| proxy_handler_called += 1 }, + :RequestCallback => Proc.new{|req, res| request_handler_called += 1 } + } + log_tester = lambda {|log, access_log| + log.reject! {|str| + %r{WARN chunked is set for an HTTP/1\.0 request\. \(ignored\)} =~ str + } + assert_equal([], log) + } + TestWEBrick.start_httpproxy(config, log_tester){|server, addr, port, log| + body = nil + server.mount_proc("/"){|req, res| + body = "#{req.request_method} #{req.path} #{req.body}" + res.chunked = true + res.body = -> (socket) { body.each_char {|c| socket.write c } } + } + + # Don't use Net::HTTP because it uses HTTP/1.1. + TCPSocket.open(addr, port) {|s| + s.write "GET / HTTP/1.0\r\nHost: localhost.localdomain\r\n\r\n" + response = s.read + assert_equal(body, response[/.*\z/]) + } + } + end + + def make_certificate(key, cn) + subject = OpenSSL::X509::Name.parse("/DC=org/DC=ruby-lang/CN=#{cn}") + exts = [ + ["keyUsage", "keyEncipherment,digitalSignature", true], + ] + cert = OpenSSL::X509::Certificate.new + cert.version = 2 + cert.serial = 1 + cert.subject = subject + cert.issuer = subject + cert.public_key = key + cert.not_before = Time.now - 3600 + cert.not_after = Time.now + 3600 + ef = OpenSSL::X509::ExtensionFactory.new(cert, cert) + exts.each {|args| cert.add_extension(ef.create_extension(*args)) } + cert.sign(key, "sha256") + return cert + end if defined?(OpenSSL::SSL) + + def test_connect + # Testing CONNECT to proxy server + # + # client -----------> proxy -----------> https + # 1. CONNECT establish TCP + # 2. ---- establish SSL session ---> + # 3. ------- GET or POST ----------> + # + key = TEST_KEY_RSA2048 + cert = make_certificate(key, "127.0.0.1") + s_config = { + :SSLEnable =>true, + :ServerName => "localhost", + :SSLCertificate => cert, + :SSLPrivateKey => key, + } + config = { + :ServerName => "localhost.localdomain", + :RequestCallback => Proc.new{|req, res| + assert_equal("CONNECT", req.request_method) + }, + } + TestWEBrick.start_httpserver(s_config){|s_server, s_addr, s_port, s_log| + s_server.mount_proc("/"){|req, res| + res.body = "SSL #{req.request_method} #{req.path} #{req.body}" + } + TestWEBrick.start_httpproxy(config){|server, addr, port, log| + http = Net::HTTP.new("127.0.0.1", s_port, addr, port) + http.use_ssl = true + http.verify_callback = Proc.new do |preverify_ok, store_ctx| + store_ctx.current_cert.to_der == cert.to_der + end + + req = Net::HTTP::Get.new("/") + req["Content-Type"] = "application/x-www-form-urlencoded" + http.request(req){|res| + assert_equal("SSL GET / ", res.body, s_log.call + log.call) + } + + req = Net::HTTP::Post.new("/") + req["Content-Type"] = "application/x-www-form-urlencoded" + req.body = "post-data" + http.request(req){|res| + assert_equal("SSL POST / post-data", res.body, s_log.call + log.call) + } + } + } + end if defined?(OpenSSL::SSL) + + def test_upstream_proxy + return if /mswin/ =~ RUBY_PLATFORM && ENV.key?('GITHUB_ACTIONS') # not working from the beginning + # Testing GET or POST through the upstream proxy server + # Note that the upstream proxy server works as the origin server. + # +------+ + # V | + # client -------> proxy -------> proxy ---+ + # GET / POST GET / POST GET / POST + # + up_proxy_handler_called = up_request_handler_called = 0 + proxy_handler_called = request_handler_called = 0 + up_config = { + :ServerName => "localhost.localdomain", + :ProxyContentHandler => Proc.new{|req, res| up_proxy_handler_called += 1}, + :RequestCallback => Proc.new{|req, res| up_request_handler_called += 1} + } + TestWEBrick.start_httpproxy(up_config){|up_server, up_addr, up_port, up_log| + up_server.mount_proc("/"){|req, res| + res.body = "#{req.request_method} #{req.path} #{req.body}" + } + config = { + :ServerName => "localhost.localdomain", + :ProxyURI => URI.parse("http://localhost:#{up_port}"), + :ProxyContentHandler => Proc.new{|req, res| proxy_handler_called += 1}, + :RequestCallback => Proc.new{|req, res| request_handler_called += 1}, + } + TestWEBrick.start_httpproxy(config){|server, addr, port, log| + http = Net::HTTP.new(up_addr, up_port, addr, port) + + req = Net::HTTP::Get.new("/") + http.request(req){|res| + skip res.message unless res.code == '200' + via = res["via"].split(/,\s+/) + assert(via.include?("1.1 localhost.localdomain:#{up_port}"), up_log.call + log.call) + assert(via.include?("1.1 localhost.localdomain:#{port}"), up_log.call + log.call) + assert_equal("GET / ", res.body) + } + assert_equal(1, up_proxy_handler_called, up_log.call + log.call) + assert_equal(2, up_request_handler_called, up_log.call + log.call) + assert_equal(1, proxy_handler_called, up_log.call + log.call) + assert_equal(1, request_handler_called, up_log.call + log.call) + + req = Net::HTTP::Head.new("/") + http.request(req){|res| + via = res["via"].split(/,\s+/) + assert(via.include?("1.1 localhost.localdomain:#{up_port}"), up_log.call + log.call) + assert(via.include?("1.1 localhost.localdomain:#{port}"), up_log.call + log.call) + assert_nil(res.body, up_log.call + log.call) + } + assert_equal(2, up_proxy_handler_called, up_log.call + log.call) + assert_equal(4, up_request_handler_called, up_log.call + log.call) + assert_equal(2, proxy_handler_called, up_log.call + log.call) + assert_equal(2, request_handler_called, up_log.call + log.call) + + req = Net::HTTP::Post.new("/") + req.body = "post-data" + req.content_type = "application/x-www-form-urlencoded" + http.request(req){|res| + via = res["via"].split(/,\s+/) + assert(via.include?("1.1 localhost.localdomain:#{up_port}"), up_log.call + log.call) + assert(via.include?("1.1 localhost.localdomain:#{port}"), up_log.call + log.call) + assert_equal("POST / post-data", res.body, up_log.call + log.call) + } + assert_equal(3, up_proxy_handler_called, up_log.call + log.call) + assert_equal(6, up_request_handler_called, up_log.call + log.call) + assert_equal(3, proxy_handler_called, up_log.call + log.call) + assert_equal(3, request_handler_called, up_log.call + log.call) + + if defined?(OpenSSL::SSL) + # Testing CONNECT to the upstream proxy server + # + # client -------> proxy -------> proxy -------> https + # 1. CONNECT CONNECT establish TCP + # 2. -------- establish SSL session ------> + # 3. ---------- GET or POST --------------> + # + key = TEST_KEY_RSA2048 + cert = make_certificate(key, "127.0.0.1") + s_config = { + :SSLEnable =>true, + :ServerName => "localhost", + :SSLCertificate => cert, + :SSLPrivateKey => key, + } + TestWEBrick.start_httpserver(s_config){|s_server, s_addr, s_port, s_log| + s_server.mount_proc("/"){|req2, res| + res.body = "SSL #{req2.request_method} #{req2.path} #{req2.body}" + } + http = Net::HTTP.new("127.0.0.1", s_port, addr, port, up_log.call + log.call + s_log.call) + http.use_ssl = true + http.verify_callback = Proc.new do |preverify_ok, store_ctx| + store_ctx.current_cert.to_der == cert.to_der + end + + req2 = Net::HTTP::Get.new("/") + http.request(req2){|res| + assert_equal("SSL GET / ", res.body, up_log.call + log.call + s_log.call) + } + + req2 = Net::HTTP::Post.new("/") + req2.body = "post-data" + req2.content_type = "application/x-www-form-urlencoded" + http.request(req2){|res| + assert_equal("SSL POST / post-data", res.body, up_log.call + log.call + s_log.call) + } + } + end + } + } + end + + if defined?(OpenSSL::SSL) + TEST_KEY_RSA2048 = OpenSSL::PKey.read <<-_end_of_pem_ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAuV9ht9J7k4NBs38jOXvvTKY9gW8nLICSno5EETR1cuF7i4pN +s9I1QJGAFAX0BEO4KbzXmuOvfCpD3CU+Slp1enenfzq/t/e/1IRW0wkJUJUFQign +4CtrkJL+P07yx18UjyPlBXb81ApEmAB5mrJVSrWmqbjs07JbuS4QQGGXLc+Su96D +kYKmSNVjBiLxVVSpyZfAY3hD37d60uG+X8xdW5v68JkRFIhdGlb6JL8fllf/A/bl +NwdJOhVr9mESHhwGjwfSeTDPfd8ZLE027E5lyAVX9KZYcU00mOX+fdxOSnGqS/8J +DRh0EPHDL15RcJjV2J6vZjPb0rOYGDoMcH+94wIDAQABAoIBAAzsamqfYQAqwXTb +I0CJtGg6msUgU7HVkOM+9d3hM2L791oGHV6xBAdpXW2H8LgvZHJ8eOeSghR8+dgq +PIqAffo4x1Oma+FOg3A0fb0evyiACyrOk+EcBdbBeLo/LcvahBtqnDfiUMQTpy6V +seSoFCwuN91TSCeGIsDpRjbG1vxZgtx+uI+oH5+ytqJOmfCksRDCkMglGkzyfcl0 +Xc5CUhIJ0my53xijEUQl19rtWdMnNnnkdbG8PT3LZlOta5Do86BElzUYka0C6dUc +VsBDQ0Nup0P6rEQgy7tephHoRlUGTYamsajGJaAo1F3IQVIrRSuagi7+YpSpCqsW +wORqorkCgYEA7RdX6MDVrbw7LePnhyuaqTiMK+055/R1TqhB1JvvxJ1CXk2rDL6G +0TLHQ7oGofd5LYiemg4ZVtWdJe43BPZlVgT6lvL/iGo8JnrncB9Da6L7nrq/+Rvj +XGjf1qODCK+LmreZWEsaLPURIoR/Ewwxb9J2zd0CaMjeTwafJo1CZvcCgYEAyCgb +aqoWvUecX8VvARfuA593Lsi50t4MEArnOXXcd1RnXoZWhbx5rgO8/ATKfXr0BK/n +h2GF9PfKzHFm/4V6e82OL7gu/kLy2u9bXN74vOvWFL5NOrOKPM7Kg+9I131kNYOw +Ivnr/VtHE5s0dY7JChYWE1F3vArrOw3T00a4CXUCgYEA0SqY+dS2LvIzW4cHCe9k +IQqsT0yYm5TFsUEr4sA3xcPfe4cV8sZb9k/QEGYb1+SWWZ+AHPV3UW5fl8kTbSNb +v4ng8i8rVVQ0ANbJO9e5CUrepein2MPL0AkOATR8M7t7dGGpvYV0cFk8ZrFx0oId +U0PgYDotF/iueBWlbsOM430CgYEAqYI95dFyPI5/AiSkY5queeb8+mQH62sdcCCr +vd/w/CZA/K5sbAo4SoTj8dLk4evU6HtIa0DOP63y071eaxvRpTNqLUOgmLh+D6gS +Cc7TfLuFrD+WDBatBd5jZ+SoHccVrLR/4L8jeodo5FPW05A+9gnKXEXsTxY4LOUC +9bS4e1kCgYAqVXZh63JsMwoaxCYmQ66eJojKa47VNrOeIZDZvd2BPVf30glBOT41 +gBoDG3WMPZoQj9pb7uMcrnvs4APj2FIhMU8U15LcPAj59cD6S6rWnAxO8NFK7HQG +4Jxg3JNNf8ErQoCHb1B3oVdXJkmbJkARoDpBKmTCgKtP8ADYLmVPQw== +-----END RSA PRIVATE KEY----- + _end_of_pem_ + end +end diff --git a/tool/test/webrick/test_httprequest.rb b/tool/test/webrick/test_httprequest.rb new file mode 100644 index 0000000000..759ccbdada --- /dev/null +++ b/tool/test/webrick/test_httprequest.rb @@ -0,0 +1,488 @@ +# frozen_string_literal: false +require "webrick" +require "stringio" +require "test/unit" + +class TestWEBrickHTTPRequest < Test::Unit::TestCase + def teardown + WEBrick::Utils::TimeoutHandler.terminate + super + end + + def test_simple_request + msg = <<-_end_of_message_ +GET / + _end_of_message_ + req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) + req.parse(StringIO.new(msg)) + assert(req.meta_vars) # fails if @header was not initialized and iteration is attempted on the nil reference + end + + def test_parse_09 + msg = <<-_end_of_message_ + GET / + foobar # HTTP/0.9 request don't have header nor entity body. + _end_of_message_ + req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) + req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) + assert_equal("GET", req.request_method) + assert_equal("/", req.unparsed_uri) + assert_equal(WEBrick::HTTPVersion.new("0.9"), req.http_version) + assert_equal(WEBrick::Config::HTTP[:ServerName], req.host) + assert_equal(80, req.port) + assert_equal(false, req.keep_alive?) + assert_equal(nil, req.body) + assert(req.query.empty?) + end + + def test_parse_10 + msg = <<-_end_of_message_ + GET / HTTP/1.0 + + _end_of_message_ + req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) + req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) + assert_equal("GET", req.request_method) + assert_equal("/", req.unparsed_uri) + assert_equal(WEBrick::HTTPVersion.new("1.0"), req.http_version) + assert_equal(WEBrick::Config::HTTP[:ServerName], req.host) + assert_equal(80, req.port) + assert_equal(false, req.keep_alive?) + assert_equal(nil, req.body) + assert(req.query.empty?) + end + + def test_parse_11 + msg = <<-_end_of_message_ + GET /path HTTP/1.1 + + _end_of_message_ + req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) + req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) + assert_equal("GET", req.request_method) + assert_equal("/path", req.unparsed_uri) + assert_equal("", req.script_name) + assert_equal("/path", req.path_info) + assert_equal(WEBrick::HTTPVersion.new("1.1"), req.http_version) + assert_equal(WEBrick::Config::HTTP[:ServerName], req.host) + assert_equal(80, req.port) + assert_equal(true, req.keep_alive?) + assert_equal(nil, req.body) + assert(req.query.empty?) + end + + def test_request_uri_too_large + msg = <<-_end_of_message_ + GET /#{"a"*2084} HTTP/1.1 + _end_of_message_ + req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) + assert_raise(WEBrick::HTTPStatus::RequestURITooLarge){ + req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) + } + end + + def test_parse_headers + msg = <<-_end_of_message_ + GET /path HTTP/1.1 + Host: test.ruby-lang.org:8080 + Connection: close + Accept: text/*;q=0.3, text/html;q=0.7, text/html;level=1, + text/html;level=2;q=0.4, */*;q=0.5 + Accept-Encoding: compress;q=0.5 + Accept-Encoding: gzip;q=1.0, identity; q=0.4, *;q=0 + Accept-Language: en;q=0.5, *; q=0 + Accept-Language: ja + Content-Type: text/plain + Content-Length: 7 + X-Empty-Header: + + foobar + _end_of_message_ + req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) + req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) + assert_equal( + URI.parse("http://test.ruby-lang.org:8080/path"), req.request_uri) + assert_equal("test.ruby-lang.org", req.host) + assert_equal(8080, req.port) + assert_equal(false, req.keep_alive?) + assert_equal( + %w(text/html;level=1 text/html */* text/html;level=2 text/*), + req.accept) + assert_equal(%w(gzip compress identity *), req.accept_encoding) + assert_equal(%w(ja en *), req.accept_language) + assert_equal(7, req.content_length) + assert_equal("text/plain", req.content_type) + assert_equal("foobar\n", req.body) + assert_equal("", req["x-empty-header"]) + assert_equal(nil, req["x-no-header"]) + assert(req.query.empty?) + end + + def test_parse_header2() + msg = <<-_end_of_message_ + POST /foo/bar/../baz?q=a HTTP/1.0 + Content-Length: 9 + User-Agent: + FOO BAR + BAZ + + hogehoge + _end_of_message_ + req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) + req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) + assert_equal("POST", req.request_method) + assert_equal("/foo/baz", req.path) + assert_equal("", req.script_name) + assert_equal("/foo/baz", req.path_info) + assert_equal("9", req['content-length']) + assert_equal("FOO BAR BAZ", req['user-agent']) + assert_equal("hogehoge\n", req.body) + end + + def test_parse_headers3 + msg = <<-_end_of_message_ + GET /path HTTP/1.1 + Host: test.ruby-lang.org + + _end_of_message_ + req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) + req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) + assert_equal(URI.parse("http://test.ruby-lang.org/path"), req.request_uri) + assert_equal("test.ruby-lang.org", req.host) + assert_equal(80, req.port) + + msg = <<-_end_of_message_ + GET /path HTTP/1.1 + Host: 192.168.1.1 + + _end_of_message_ + req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) + req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) + assert_equal(URI.parse("http://192.168.1.1/path"), req.request_uri) + assert_equal("192.168.1.1", req.host) + assert_equal(80, req.port) + + msg = <<-_end_of_message_ + GET /path HTTP/1.1 + Host: [fe80::208:dff:feef:98c7] + + _end_of_message_ + req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) + req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) + assert_equal(URI.parse("http://[fe80::208:dff:feef:98c7]/path"), + req.request_uri) + assert_equal("[fe80::208:dff:feef:98c7]", req.host) + assert_equal(80, req.port) + + msg = <<-_end_of_message_ + GET /path HTTP/1.1 + Host: 192.168.1.1:8080 + + _end_of_message_ + req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) + req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) + assert_equal(URI.parse("http://192.168.1.1:8080/path"), req.request_uri) + assert_equal("192.168.1.1", req.host) + assert_equal(8080, req.port) + + msg = <<-_end_of_message_ + GET /path HTTP/1.1 + Host: [fe80::208:dff:feef:98c7]:8080 + + _end_of_message_ + req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) + req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) + assert_equal(URI.parse("http://[fe80::208:dff:feef:98c7]:8080/path"), + req.request_uri) + assert_equal("[fe80::208:dff:feef:98c7]", req.host) + assert_equal(8080, req.port) + end + + def test_parse_get_params + param = "foo=1;foo=2;foo=3;bar=x" + msg = <<-_end_of_message_ + GET /path?#{param} HTTP/1.1 + Host: test.ruby-lang.org:8080 + + _end_of_message_ + req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) + req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) + query = req.query + assert_equal("1", query["foo"]) + assert_equal(["1", "2", "3"], query["foo"].to_ary) + assert_equal(["1", "2", "3"], query["foo"].list) + assert_equal("x", query["bar"]) + assert_equal(["x"], query["bar"].list) + end + + def test_parse_post_params + param = "foo=1;foo=2;foo=3;bar=x" + msg = <<-_end_of_message_ + POST /path?foo=x;foo=y;foo=z;bar=1 HTTP/1.1 + Host: test.ruby-lang.org:8080 + Content-Length: #{param.size} + Content-Type: application/x-www-form-urlencoded + + #{param} + _end_of_message_ + req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) + req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) + query = req.query + assert_equal("1", query["foo"]) + assert_equal(["1", "2", "3"], query["foo"].to_ary) + assert_equal(["1", "2", "3"], query["foo"].list) + assert_equal("x", query["bar"]) + assert_equal(["x"], query["bar"].list) + end + + def test_chunked + crlf = "\x0d\x0a" + expect = File.binread(__FILE__).freeze + msg = <<-_end_of_message_ + POST /path HTTP/1.1 + Host: test.ruby-lang.org:8080 + Transfer-Encoding: chunked + + _end_of_message_ + msg.gsub!(/^ {6}/, "") + open(__FILE__){|io| + while chunk = io.read(100) + msg << chunk.size.to_s(16) << crlf + msg << chunk << crlf + end + } + msg << "0" << crlf + req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) + req.parse(StringIO.new(msg)) + assert_equal(expect, req.body) + + # chunked req.body_reader + req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) + req.parse(StringIO.new(msg)) + dst = StringIO.new + IO.copy_stream(req.body_reader, dst) + assert_equal(expect, dst.string) + end + + def test_forwarded + msg = <<-_end_of_message_ + GET /foo HTTP/1.1 + Host: localhost:10080 + User-Agent: w3m/0.5.2 + X-Forwarded-For: 123.123.123.123 + X-Forwarded-Host: forward.example.com + X-Forwarded-Server: server.example.com + Connection: Keep-Alive + + _end_of_message_ + msg.gsub!(/^ {6}/, "") + req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) + req.parse(StringIO.new(msg)) + assert_equal("server.example.com", req.server_name) + assert_equal("http://forward.example.com/foo", req.request_uri.to_s) + assert_equal("forward.example.com", req.host) + assert_equal(80, req.port) + assert_equal("123.123.123.123", req.remote_ip) + assert(!req.ssl?) + + msg = <<-_end_of_message_ + GET /foo HTTP/1.1 + Host: localhost:10080 + User-Agent: w3m/0.5.2 + X-Forwarded-For: 192.168.1.10, 172.16.1.1, 123.123.123.123 + X-Forwarded-Host: forward.example.com:8080 + X-Forwarded-Server: server.example.com + Connection: Keep-Alive + + _end_of_message_ + msg.gsub!(/^ {6}/, "") + req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) + req.parse(StringIO.new(msg)) + assert_equal("server.example.com", req.server_name) + assert_equal("http://forward.example.com:8080/foo", req.request_uri.to_s) + assert_equal("forward.example.com", req.host) + assert_equal(8080, req.port) + assert_equal("123.123.123.123", req.remote_ip) + assert(!req.ssl?) + + msg = <<-_end_of_message_ + GET /foo HTTP/1.1 + Host: localhost:10080 + Client-IP: 234.234.234.234 + X-Forwarded-Proto: https, http + X-Forwarded-For: 192.168.1.10, 10.0.0.1, 123.123.123.123 + X-Forwarded-Host: forward.example.com + X-Forwarded-Server: server.example.com + X-Requested-With: XMLHttpRequest + Connection: Keep-Alive + + _end_of_message_ + msg.gsub!(/^ {6}/, "") + req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) + req.parse(StringIO.new(msg)) + assert_equal("server.example.com", req.server_name) + assert_equal("https://forward.example.com/foo", req.request_uri.to_s) + assert_equal("forward.example.com", req.host) + assert_equal(443, req.port) + assert_equal("234.234.234.234", req.remote_ip) + assert(req.ssl?) + + msg = <<-_end_of_message_ + GET /foo HTTP/1.1 + Host: localhost:10080 + Client-IP: 234.234.234.234 + X-Forwarded-Proto: https + X-Forwarded-For: 192.168.1.10 + X-Forwarded-Host: forward1.example.com:1234, forward2.example.com:5678 + X-Forwarded-Server: server1.example.com, server2.example.com + X-Requested-With: XMLHttpRequest + Connection: Keep-Alive + + _end_of_message_ + msg.gsub!(/^ {6}/, "") + req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) + req.parse(StringIO.new(msg)) + assert_equal("server1.example.com", req.server_name) + assert_equal("https://forward1.example.com:1234/foo", req.request_uri.to_s) + assert_equal("forward1.example.com", req.host) + assert_equal(1234, req.port) + assert_equal("234.234.234.234", req.remote_ip) + assert(req.ssl?) + + msg = <<-_end_of_message_ + GET /foo HTTP/1.1 + Host: localhost:10080 + Client-IP: 234.234.234.234 + X-Forwarded-Proto: https + X-Forwarded-For: 192.168.1.10 + X-Forwarded-Host: [fd20:8b1e:b255:8154:250:56ff:fea8:4d84], forward2.example.com:5678 + X-Forwarded-Server: server1.example.com, server2.example.com + X-Requested-With: XMLHttpRequest + Connection: Keep-Alive + + _end_of_message_ + msg.gsub!(/^ {6}/, "") + req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) + req.parse(StringIO.new(msg)) + assert_equal("server1.example.com", req.server_name) + assert_equal("https://[fd20:8b1e:b255:8154:250:56ff:fea8:4d84]/foo", req.request_uri.to_s) + assert_equal("[fd20:8b1e:b255:8154:250:56ff:fea8:4d84]", req.host) + assert_equal(443, req.port) + assert_equal("234.234.234.234", req.remote_ip) + assert(req.ssl?) + + msg = <<-_end_of_message_ + GET /foo HTTP/1.1 + Host: localhost:10080 + Client-IP: 234.234.234.234 + X-Forwarded-Proto: https + X-Forwarded-For: 192.168.1.10 + X-Forwarded-Host: [fd20:8b1e:b255:8154:250:56ff:fea8:4d84]:1234, forward2.example.com:5678 + X-Forwarded-Server: server1.example.com, server2.example.com + X-Requested-With: XMLHttpRequest + Connection: Keep-Alive + + _end_of_message_ + msg.gsub!(/^ {6}/, "") + req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) + req.parse(StringIO.new(msg)) + assert_equal("server1.example.com", req.server_name) + assert_equal("https://[fd20:8b1e:b255:8154:250:56ff:fea8:4d84]:1234/foo", req.request_uri.to_s) + assert_equal("[fd20:8b1e:b255:8154:250:56ff:fea8:4d84]", req.host) + assert_equal(1234, req.port) + assert_equal("234.234.234.234", req.remote_ip) + assert(req.ssl?) + end + + def test_continue_sent + msg = <<-_end_of_message_ + POST /path HTTP/1.1 + Expect: 100-continue + + _end_of_message_ + msg.gsub!(/^ {6}/, "") + req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) + req.parse(StringIO.new(msg)) + assert req['expect'] + l = msg.size + req.continue + assert_not_equal l, msg.size + assert_match(/HTTP\/1.1 100 continue\r\n\r\n\z/, msg) + assert !req['expect'] + end + + def test_continue_not_sent + msg = <<-_end_of_message_ + POST /path HTTP/1.1 + + _end_of_message_ + msg.gsub!(/^ {6}/, "") + req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) + req.parse(StringIO.new(msg)) + assert !req['expect'] + l = msg.size + req.continue + assert_equal l, msg.size + end + + def test_empty_post + msg = <<-_end_of_message_ + POST /path?foo=x;foo=y;foo=z;bar=1 HTTP/1.1 + Host: test.ruby-lang.org:8080 + Content-Type: application/x-www-form-urlencoded + + _end_of_message_ + req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) + req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) + req.body + end + + def test_bad_messages + param = "foo=1;foo=2;foo=3;bar=x" + msg = <<-_end_of_message_ + POST /path?foo=x;foo=y;foo=z;bar=1 HTTP/1.1 + Host: test.ruby-lang.org:8080 + Content-Type: application/x-www-form-urlencoded + + #{param} + _end_of_message_ + assert_raise(WEBrick::HTTPStatus::LengthRequired){ + req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) + req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) + req.body + } + + msg = <<-_end_of_message_ + POST /path?foo=x;foo=y;foo=z;bar=1 HTTP/1.1 + Host: test.ruby-lang.org:8080 + Content-Length: 100000 + + body is too short. + _end_of_message_ + assert_raise(WEBrick::HTTPStatus::BadRequest){ + req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) + req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) + req.body + } + + msg = <<-_end_of_message_ + POST /path?foo=x;foo=y;foo=z;bar=1 HTTP/1.1 + Host: test.ruby-lang.org:8080 + Transfer-Encoding: foobar + + body is too short. + _end_of_message_ + assert_raise(WEBrick::HTTPStatus::NotImplemented){ + req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) + req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) + req.body + } + end + + def test_eof_raised_when_line_is_nil + assert_raise(WEBrick::HTTPStatus::EOFError) { + req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) + req.parse(StringIO.new("")) + } + end +end diff --git a/tool/test/webrick/test_httpresponse.rb b/tool/test/webrick/test_httpresponse.rb new file mode 100644 index 0000000000..4410f63e89 --- /dev/null +++ b/tool/test/webrick/test_httpresponse.rb @@ -0,0 +1,282 @@ +# frozen_string_literal: false +require "webrick" +require "test/unit" +require "stringio" +require "net/http" + +module WEBrick + class TestHTTPResponse < Test::Unit::TestCase + class FakeLogger + attr_reader :messages + + def initialize + @messages = [] + end + + def warn msg + @messages << msg + end + end + + attr_reader :config, :logger, :res + + def setup + super + @logger = FakeLogger.new + @config = Config::HTTP + @config[:Logger] = logger + @res = HTTPResponse.new config + @res.keep_alive = true + end + + def test_prevent_response_splitting_headers_crlf + res['X-header'] = "malicious\r\nCookie: cracked_indicator_for_test" + io = StringIO.new + res.send_response io + io.rewind + res = Net::HTTPResponse.read_new(Net::BufferedIO.new(io)) + assert_equal '500', res.code + refute_match 'cracked_indicator_for_test', io.string + end + + def test_prevent_response_splitting_cookie_headers_crlf + user_input = "malicious\r\nCookie: cracked_indicator_for_test" + res.cookies << WEBrick::Cookie.new('author', user_input) + io = StringIO.new + res.send_response io + io.rewind + res = Net::HTTPResponse.read_new(Net::BufferedIO.new(io)) + assert_equal '500', res.code + refute_match 'cracked_indicator_for_test', io.string + end + + def test_prevent_response_splitting_headers_cr + res['X-header'] = "malicious\rCookie: cracked_indicator_for_test" + io = StringIO.new + res.send_response io + io.rewind + res = Net::HTTPResponse.read_new(Net::BufferedIO.new(io)) + assert_equal '500', res.code + refute_match 'cracked_indicator_for_test', io.string + end + + def test_prevent_response_splitting_cookie_headers_cr + user_input = "malicious\rCookie: cracked_indicator_for_test" + res.cookies << WEBrick::Cookie.new('author', user_input) + io = StringIO.new + res.send_response io + io.rewind + res = Net::HTTPResponse.read_new(Net::BufferedIO.new(io)) + assert_equal '500', res.code + refute_match 'cracked_indicator_for_test', io.string + end + + def test_prevent_response_splitting_headers_lf + res['X-header'] = "malicious\nCookie: cracked_indicator_for_test" + io = StringIO.new + res.send_response io + io.rewind + res = Net::HTTPResponse.read_new(Net::BufferedIO.new(io)) + assert_equal '500', res.code + refute_match 'cracked_indicator_for_test', io.string + end + + def test_prevent_response_splitting_cookie_headers_lf + user_input = "malicious\nCookie: cracked_indicator_for_test" + res.cookies << WEBrick::Cookie.new('author', user_input) + io = StringIO.new + res.send_response io + io.rewind + res = Net::HTTPResponse.read_new(Net::BufferedIO.new(io)) + assert_equal '500', res.code + refute_match 'cracked_indicator_for_test', io.string + end + + def test_set_redirect_response_splitting + url = "malicious\r\nCookie: cracked_indicator_for_test" + assert_raise(URI::InvalidURIError) do + res.set_redirect(WEBrick::HTTPStatus::MultipleChoices, url) + end + end + + def test_set_redirect_html_injection + url = 'http://example.com////?a</a><head></head><body><img src=1></body>' + assert_raise(WEBrick::HTTPStatus::MultipleChoices) do + res.set_redirect(WEBrick::HTTPStatus::MultipleChoices, url) + end + res.status = 300 + io = StringIO.new + res.send_response(io) + io.rewind + res = Net::HTTPResponse.read_new(Net::BufferedIO.new(io)) + assert_equal '300', res.code + refute_match(/<img/, io.string) + end + + def test_304_does_not_log_warning + res.status = 304 + res.setup_header + assert_equal 0, logger.messages.length + end + + def test_204_does_not_log_warning + res.status = 204 + res.setup_header + + assert_equal 0, logger.messages.length + end + + def test_1xx_does_not_log_warnings + res.status = 105 + res.setup_header + + assert_equal 0, logger.messages.length + end + + def test_200_chunked_does_not_set_content_length + res.chunked = false + res["Transfer-Encoding"] = 'chunked' + res.setup_header + assert_nil res.header.fetch('content-length', nil) + end + + def test_send_body_io + IO.pipe {|body_r, body_w| + body_w.write 'hello' + body_w.close + + @res.body = body_r + + IO.pipe {|r, w| + + @res.send_body w + + w.close + + assert_equal 'hello', r.read + } + } + assert_equal 0, logger.messages.length + end + + def test_send_body_string + @res.body = 'hello' + + IO.pipe {|r, w| + @res.send_body w + + w.close + + assert_equal 'hello', r.read + } + assert_equal 0, logger.messages.length + end + + def test_send_body_string_io + @res.body = StringIO.new 'hello' + + IO.pipe {|r, w| + @res.send_body w + + w.close + + assert_equal 'hello', r.read + } + assert_equal 0, logger.messages.length + end + + def test_send_body_io_chunked + @res.chunked = true + + IO.pipe {|body_r, body_w| + + body_w.write 'hello' + body_w.close + + @res.body = body_r + + IO.pipe {|r, w| + @res.send_body w + + w.close + + r.binmode + assert_equal "5\r\nhello\r\n0\r\n\r\n", r.read + } + } + assert_equal 0, logger.messages.length + end + + def test_send_body_string_chunked + @res.chunked = true + + @res.body = 'hello' + + IO.pipe {|r, w| + @res.send_body w + + w.close + + r.binmode + assert_equal "5\r\nhello\r\n0\r\n\r\n", r.read + } + assert_equal 0, logger.messages.length + end + + def test_send_body_string_io_chunked + @res.chunked = true + + @res.body = StringIO.new 'hello' + + IO.pipe {|r, w| + @res.send_body w + + w.close + + r.binmode + assert_equal "5\r\nhello\r\n0\r\n\r\n", r.read + } + assert_equal 0, logger.messages.length + end + + def test_send_body_proc + @res.body = Proc.new { |out| out.write('hello') } + IO.pipe do |r, w| + @res.send_body(w) + w.close + r.binmode + assert_equal 'hello', r.read + end + assert_equal 0, logger.messages.length + end + + def test_send_body_proc_chunked + @res.body = Proc.new { |out| out.write('hello') } + @res.chunked = true + IO.pipe do |r, w| + @res.send_body(w) + w.close + r.binmode + assert_equal "5\r\nhello\r\n0\r\n\r\n", r.read + end + assert_equal 0, logger.messages.length + end + + def test_set_error + status = 400 + message = 'missing attribute' + @res.status = status + error = WEBrick::HTTPStatus[status].new(message) + body = @res.set_error(error) + assert_match(/#{@res.reason_phrase}/, body) + assert_match(/#{message}/, body) + end + + def test_no_extraneous_space + [200, 300, 400, 500].each do |status| + @res.status = status + assert_match(/\S\r\n/, @res.status_line) + end + end + end +end diff --git a/tool/test/webrick/test_https.rb b/tool/test/webrick/test_https.rb new file mode 100644 index 0000000000..ec0aac354a --- /dev/null +++ b/tool/test/webrick/test_https.rb @@ -0,0 +1,112 @@ +# frozen_string_literal: false +require "test/unit" +require "net/http" +require "webrick" +require "webrick/https" +require "webrick/utils" +require_relative "utils" + +class TestWEBrickHTTPS < Test::Unit::TestCase + empty_log = Object.new + def empty_log.<<(str) + assert_equal('', str) + self + end + NoLog = WEBrick::Log.new(empty_log, WEBrick::BasicLog::WARN) + + class HTTPSNITest < ::Net::HTTP + attr_accessor :sni_hostname + + def ssl_socket_connect(s, timeout) + s.hostname = sni_hostname + super + end + end + + def teardown + WEBrick::Utils::TimeoutHandler.terminate + super + end + + def https_get(addr, port, hostname, path, verifyname = nil) + subject = nil + http = HTTPSNITest.new(addr, port) + http.use_ssl = true + http.verify_mode = OpenSSL::SSL::VERIFY_NONE + http.verify_callback = proc { |x, store| subject = store.chain[0].subject.to_s; x } + http.sni_hostname = hostname + req = Net::HTTP::Get.new(path) + req["Host"] = "#{hostname}:#{port}" + response = http.start { http.request(req).body } + assert_equal("/CN=#{verifyname || hostname}", subject) + response + end + + def test_sni + config = { + :ServerName => "localhost", + :SSLEnable => true, + :SSLCertName => "/CN=localhost", + } + TestWEBrick.start_httpserver(config){|server, addr, port, log| + server.mount_proc("/") {|req, res| res.body = "master" } + + # catch stderr in create_self_signed_cert + stderr_buffer = StringIO.new + old_stderr, $stderr = $stderr, stderr_buffer + + begin + vhost_config1 = { + :ServerName => "vhost1", + :Port => port, + :DoNotListen => true, + :Logger => NoLog, + :AccessLog => [], + :SSLEnable => true, + :SSLCertName => "/CN=vhost1", + } + vhost1 = WEBrick::HTTPServer.new(vhost_config1) + vhost1.mount_proc("/") {|req, res| res.body = "vhost1" } + server.virtual_host(vhost1) + + vhost_config2 = { + :ServerName => "vhost2", + :ServerAlias => ["vhost2alias"], + :Port => port, + :DoNotListen => true, + :Logger => NoLog, + :AccessLog => [], + :SSLEnable => true, + :SSLCertName => "/CN=vhost2", + } + vhost2 = WEBrick::HTTPServer.new(vhost_config2) + vhost2.mount_proc("/") {|req, res| res.body = "vhost2" } + server.virtual_host(vhost2) + ensure + # restore stderr + $stderr = old_stderr + end + + assert_match(/\A([.+*]+\n)+\z/, stderr_buffer.string) + assert_equal("master", https_get(addr, port, "localhost", "/localhost")) + assert_equal("master", https_get(addr, port, "unknown", "/unknown", "localhost")) + assert_equal("vhost1", https_get(addr, port, "vhost1", "/vhost1")) + assert_equal("vhost2", https_get(addr, port, "vhost2", "/vhost2")) + assert_equal("vhost2", https_get(addr, port, "vhost2alias", "/vhost2alias", "vhost2")) + } + end + + def test_check_ssl_virtual + config = { + :ServerName => "localhost", + :SSLEnable => true, + :SSLCertName => "/CN=localhost", + } + TestWEBrick.start_httpserver(config){|server, addr, port, log| + assert_raise ArgumentError do + vhost = WEBrick::HTTPServer.new({:DoNotListen => true, :Logger => NoLog}) + server.virtual_host(vhost) + end + } + end +end diff --git a/tool/test/webrick/test_httpserver.rb b/tool/test/webrick/test_httpserver.rb new file mode 100644 index 0000000000..4133be85ad --- /dev/null +++ b/tool/test/webrick/test_httpserver.rb @@ -0,0 +1,543 @@ +# frozen_string_literal: false +require "test/unit" +require "net/http" +require "webrick" +require_relative "utils" + +class TestWEBrickHTTPServer < Test::Unit::TestCase + empty_log = Object.new + def empty_log.<<(str) + assert_equal('', str) + self + end + NoLog = WEBrick::Log.new(empty_log, WEBrick::BasicLog::WARN) + + def teardown + WEBrick::Utils::TimeoutHandler.terminate + super + end + + def test_mount + httpd = WEBrick::HTTPServer.new( + :Logger => NoLog, + :DoNotListen=>true + ) + httpd.mount("/", :Root) + httpd.mount("/foo", :Foo) + httpd.mount("/foo/bar", :Bar, :bar1) + httpd.mount("/foo/bar/baz", :Baz, :baz1, :baz2) + + serv, opts, script_name, path_info = httpd.search_servlet("/") + assert_equal(:Root, serv) + assert_equal([], opts) + assert_equal("", script_name) + assert_equal("/", path_info) + + serv, opts, script_name, path_info = httpd.search_servlet("/sub") + assert_equal(:Root, serv) + assert_equal([], opts) + assert_equal("", script_name) + assert_equal("/sub", path_info) + + serv, opts, script_name, path_info = httpd.search_servlet("/sub/") + assert_equal(:Root, serv) + assert_equal([], opts) + assert_equal("", script_name) + assert_equal("/sub/", path_info) + + serv, opts, script_name, path_info = httpd.search_servlet("/foo") + assert_equal(:Foo, serv) + assert_equal([], opts) + assert_equal("/foo", script_name) + assert_equal("", path_info) + + serv, opts, script_name, path_info = httpd.search_servlet("/foo/") + assert_equal(:Foo, serv) + assert_equal([], opts) + assert_equal("/foo", script_name) + assert_equal("/", path_info) + + serv, opts, script_name, path_info = httpd.search_servlet("/foo/sub") + assert_equal(:Foo, serv) + assert_equal([], opts) + assert_equal("/foo", script_name) + assert_equal("/sub", path_info) + + serv, opts, script_name, path_info = httpd.search_servlet("/foo/bar") + assert_equal(:Bar, serv) + assert_equal([:bar1], opts) + assert_equal("/foo/bar", script_name) + assert_equal("", path_info) + + serv, opts, script_name, path_info = httpd.search_servlet("/foo/bar/baz") + assert_equal(:Baz, serv) + assert_equal([:baz1, :baz2], opts) + assert_equal("/foo/bar/baz", script_name) + assert_equal("", path_info) + end + + class Req + attr_reader :port, :host + def initialize(addr, port, host) + @addr, @port, @host = addr, port, host + end + def addr + [0,0,0,@addr] + end + end + + def httpd(addr, port, host, ali) + config ={ + :Logger => NoLog, + :DoNotListen => true, + :BindAddress => addr, + :Port => port, + :ServerName => host, + :ServerAlias => ali, + } + return WEBrick::HTTPServer.new(config) + end + + def assert_eql?(v1, v2) + assert_equal(v1.object_id, v2.object_id) + end + + def test_lookup_server + addr1 = "192.168.100.1" + addr2 = "192.168.100.2" + addrz = "192.168.100.254" + local = "127.0.0.1" + port1 = 80 + port2 = 8080 + port3 = 10080 + portz = 32767 + name1 = "www.example.com" + name2 = "www2.example.com" + name3 = "www3.example.com" + namea = "www.example.co.jp" + nameb = "www.example.jp" + namec = "www2.example.co.jp" + named = "www2.example.jp" + namez = "foobar.example.com" + alias1 = [namea, nameb] + alias2 = [namec, named] + + host1 = httpd(nil, port1, name1, nil) + hosts = [ + host2 = httpd(addr1, port1, name1, nil), + host3 = httpd(addr1, port1, name2, alias1), + host4 = httpd(addr1, port2, name1, nil), + host5 = httpd(addr1, port2, name2, alias1), + httpd(addr1, port2, name3, alias2), + host7 = httpd(addr2, nil, name1, nil), + host8 = httpd(addr2, nil, name2, alias1), + httpd(addr2, nil, name3, alias2), + host10 = httpd(local, nil, nil, nil), + host11 = httpd(nil, port3, nil, nil), + ].sort_by{ rand } + hosts.each{|h| host1.virtual_host(h) } + + # connect to addr1 + assert_eql?(host2, host1.lookup_server(Req.new(addr1, port1, name1))) + assert_eql?(host3, host1.lookup_server(Req.new(addr1, port1, name2))) + assert_eql?(host3, host1.lookup_server(Req.new(addr1, port1, namea))) + assert_eql?(host3, host1.lookup_server(Req.new(addr1, port1, nameb))) + assert_eql?(nil, host1.lookup_server(Req.new(addr1, port1, namez))) + assert_eql?(host4, host1.lookup_server(Req.new(addr1, port2, name1))) + assert_eql?(host5, host1.lookup_server(Req.new(addr1, port2, name2))) + assert_eql?(host5, host1.lookup_server(Req.new(addr1, port2, namea))) + assert_eql?(host5, host1.lookup_server(Req.new(addr1, port2, nameb))) + assert_eql?(nil, host1.lookup_server(Req.new(addr1, port2, namez))) + assert_eql?(host11, host1.lookup_server(Req.new(addr1, port3, name1))) + assert_eql?(host11, host1.lookup_server(Req.new(addr1, port3, name2))) + assert_eql?(host11, host1.lookup_server(Req.new(addr1, port3, namea))) + assert_eql?(host11, host1.lookup_server(Req.new(addr1, port3, nameb))) + assert_eql?(host11, host1.lookup_server(Req.new(addr1, port3, namez))) + assert_eql?(nil, host1.lookup_server(Req.new(addr1, portz, name1))) + assert_eql?(nil, host1.lookup_server(Req.new(addr1, portz, name2))) + assert_eql?(nil, host1.lookup_server(Req.new(addr1, portz, namea))) + assert_eql?(nil, host1.lookup_server(Req.new(addr1, portz, nameb))) + assert_eql?(nil, host1.lookup_server(Req.new(addr1, portz, namez))) + + # connect to addr2 + assert_eql?(host7, host1.lookup_server(Req.new(addr2, port1, name1))) + assert_eql?(host8, host1.lookup_server(Req.new(addr2, port1, name2))) + assert_eql?(host8, host1.lookup_server(Req.new(addr2, port1, namea))) + assert_eql?(host8, host1.lookup_server(Req.new(addr2, port1, nameb))) + assert_eql?(nil, host1.lookup_server(Req.new(addr2, port1, namez))) + assert_eql?(host7, host1.lookup_server(Req.new(addr2, port2, name1))) + assert_eql?(host8, host1.lookup_server(Req.new(addr2, port2, name2))) + assert_eql?(host8, host1.lookup_server(Req.new(addr2, port2, namea))) + assert_eql?(host8, host1.lookup_server(Req.new(addr2, port2, nameb))) + assert_eql?(nil, host1.lookup_server(Req.new(addr2, port2, namez))) + assert_eql?(host7, host1.lookup_server(Req.new(addr2, port3, name1))) + assert_eql?(host8, host1.lookup_server(Req.new(addr2, port3, name2))) + assert_eql?(host8, host1.lookup_server(Req.new(addr2, port3, namea))) + assert_eql?(host8, host1.lookup_server(Req.new(addr2, port3, nameb))) + assert_eql?(host11, host1.lookup_server(Req.new(addr2, port3, namez))) + assert_eql?(host7, host1.lookup_server(Req.new(addr2, portz, name1))) + assert_eql?(host8, host1.lookup_server(Req.new(addr2, portz, name2))) + assert_eql?(host8, host1.lookup_server(Req.new(addr2, portz, namea))) + assert_eql?(host8, host1.lookup_server(Req.new(addr2, portz, nameb))) + assert_eql?(nil, host1.lookup_server(Req.new(addr2, portz, namez))) + + # connect to addrz + assert_eql?(nil, host1.lookup_server(Req.new(addrz, port1, name1))) + assert_eql?(nil, host1.lookup_server(Req.new(addrz, port1, name2))) + assert_eql?(nil, host1.lookup_server(Req.new(addrz, port1, namea))) + assert_eql?(nil, host1.lookup_server(Req.new(addrz, port1, nameb))) + assert_eql?(nil, host1.lookup_server(Req.new(addrz, port1, namez))) + assert_eql?(nil, host1.lookup_server(Req.new(addrz, port2, name1))) + assert_eql?(nil, host1.lookup_server(Req.new(addrz, port2, name2))) + assert_eql?(nil, host1.lookup_server(Req.new(addrz, port2, namea))) + assert_eql?(nil, host1.lookup_server(Req.new(addrz, port2, nameb))) + assert_eql?(nil, host1.lookup_server(Req.new(addrz, port2, namez))) + assert_eql?(host11, host1.lookup_server(Req.new(addrz, port3, name1))) + assert_eql?(host11, host1.lookup_server(Req.new(addrz, port3, name2))) + assert_eql?(host11, host1.lookup_server(Req.new(addrz, port3, namea))) + assert_eql?(host11, host1.lookup_server(Req.new(addrz, port3, nameb))) + assert_eql?(host11, host1.lookup_server(Req.new(addrz, port3, namez))) + assert_eql?(nil, host1.lookup_server(Req.new(addrz, portz, name1))) + assert_eql?(nil, host1.lookup_server(Req.new(addrz, portz, name2))) + assert_eql?(nil, host1.lookup_server(Req.new(addrz, portz, namea))) + assert_eql?(nil, host1.lookup_server(Req.new(addrz, portz, nameb))) + assert_eql?(nil, host1.lookup_server(Req.new(addrz, portz, namez))) + + # connect to localhost + assert_eql?(host10, host1.lookup_server(Req.new(local, port1, name1))) + assert_eql?(host10, host1.lookup_server(Req.new(local, port1, name2))) + assert_eql?(host10, host1.lookup_server(Req.new(local, port1, namea))) + assert_eql?(host10, host1.lookup_server(Req.new(local, port1, nameb))) + assert_eql?(host10, host1.lookup_server(Req.new(local, port1, namez))) + assert_eql?(host10, host1.lookup_server(Req.new(local, port2, name1))) + assert_eql?(host10, host1.lookup_server(Req.new(local, port2, name2))) + assert_eql?(host10, host1.lookup_server(Req.new(local, port2, namea))) + assert_eql?(host10, host1.lookup_server(Req.new(local, port2, nameb))) + assert_eql?(host10, host1.lookup_server(Req.new(local, port2, namez))) + assert_eql?(host10, host1.lookup_server(Req.new(local, port3, name1))) + assert_eql?(host10, host1.lookup_server(Req.new(local, port3, name2))) + assert_eql?(host10, host1.lookup_server(Req.new(local, port3, namea))) + assert_eql?(host10, host1.lookup_server(Req.new(local, port3, nameb))) + assert_eql?(host10, host1.lookup_server(Req.new(local, port3, namez))) + assert_eql?(host10, host1.lookup_server(Req.new(local, portz, name1))) + assert_eql?(host10, host1.lookup_server(Req.new(local, portz, name2))) + assert_eql?(host10, host1.lookup_server(Req.new(local, portz, namea))) + assert_eql?(host10, host1.lookup_server(Req.new(local, portz, nameb))) + assert_eql?(host10, host1.lookup_server(Req.new(local, portz, namez))) + end + + def test_callbacks + accepted = started = stopped = 0 + requested0 = requested1 = 0 + config = { + :ServerName => "localhost", + :AcceptCallback => Proc.new{ accepted += 1 }, + :StartCallback => Proc.new{ started += 1 }, + :StopCallback => Proc.new{ stopped += 1 }, + :RequestCallback => Proc.new{|req, res| requested0 += 1 }, + } + log_tester = lambda {|log, access_log| + assert(log.find {|s| %r{ERROR `/' not found\.} =~ s }) + assert_equal([], log.reject {|s| %r{ERROR `/' not found\.} =~ s }) + } + TestWEBrick.start_httpserver(config, log_tester){|server, addr, port, log| + vhost_config = { + :ServerName => "myhostname", + :BindAddress => addr, + :Port => port, + :DoNotListen => true, + :Logger => NoLog, + :AccessLog => [], + :RequestCallback => Proc.new{|req, res| requested1 += 1 }, + } + server.virtual_host(WEBrick::HTTPServer.new(vhost_config)) + + Thread.pass while server.status != :Running + sleep 1 if defined?(RubyVM::MJIT) && RubyVM::MJIT.enabled? # server.status behaves unexpectedly with --jit-wait + assert_equal(1, started, log.call) + assert_equal(0, stopped, log.call) + assert_equal(0, accepted, log.call) + + http = Net::HTTP.new(addr, port) + req = Net::HTTP::Get.new("/") + req["Host"] = "myhostname:#{port}" + http.request(req){|res| assert_equal("404", res.code, log.call)} + http.request(req){|res| assert_equal("404", res.code, log.call)} + http.request(req){|res| assert_equal("404", res.code, log.call)} + req["Host"] = "localhost:#{port}" + http.request(req){|res| assert_equal("404", res.code, log.call)} + http.request(req){|res| assert_equal("404", res.code, log.call)} + http.request(req){|res| assert_equal("404", res.code, log.call)} + assert_equal(6, accepted, log.call) + assert_equal(3, requested0, log.call) + assert_equal(3, requested1, log.call) + } + assert_equal(started, 1) + assert_equal(stopped, 1) + end + + class CustomRequest < ::WEBrick::HTTPRequest; end + class CustomResponse < ::WEBrick::HTTPResponse; end + class CustomServer < ::WEBrick::HTTPServer + def create_request(config) + CustomRequest.new(config) + end + + def create_response(config) + CustomResponse.new(config) + end + end + + def test_custom_server_request_and_response + config = { :ServerName => "localhost" } + TestWEBrick.start_server(CustomServer, config){|server, addr, port, log| + server.mount_proc("/", lambda {|req, res| + assert_kind_of(CustomRequest, req) + assert_kind_of(CustomResponse, res) + res.body = "via custom response" + }) + Thread.pass while server.status != :Running + + Net::HTTP.start(addr, port) do |http| + req = Net::HTTP::Get.new("/") + http.request(req){|res| + assert_equal("via custom response", res.body) + } + server.shutdown + end + } + end + + # This class is needed by test_response_io_with_chunked_set method + class EventManagerForChunkedResponseTest + def initialize + @listeners = [] + end + def add_listener( &block ) + @listeners << block + end + def raise_str_event( str ) + @listeners.each{ |e| e.call( :str, str ) } + end + def raise_close_event() + @listeners.each{ |e| e.call( :cls ) } + end + end + def test_response_io_with_chunked_set + evt_man = EventManagerForChunkedResponseTest.new + t = Thread.new do + begin + config = { + :ServerName => "localhost" + } + TestWEBrick.start_httpserver(config) do |server, addr, port, log| + body_strs = [ 'aaaaaa', 'bb', 'cccc' ] + server.mount_proc( "/", ->( req, res ){ + # Test for setting chunked... + res.chunked = true + r,w = IO.pipe + evt_man.add_listener do |type,str| + type == :cls ? ( w.close ) : ( w << str ) + end + res.body = r + } ) + Thread.pass while server.status != :Running + http = Net::HTTP.new(addr, port) + req = Net::HTTP::Get.new("/") + http.request(req) do |res| + i = 0 + evt_man.raise_str_event( body_strs[i] ) + res.read_body do |s| + assert_equal( body_strs[i], s ) + i += 1 + if i < body_strs.length + evt_man.raise_str_event( body_strs[i] ) + else + evt_man.raise_close_event() + end + end + assert_equal( body_strs.length, i ) + end + end + rescue => err + flunk( 'exception raised in thread: ' + err.to_s ) + end + end + if t.join( 3 ).nil? + evt_man.raise_close_event() + flunk( 'timeout' ) + if t.join( 1 ).nil? + Thread.kill t + end + end + end + + def test_response_io_without_chunked_set + config = { + :ServerName => "localhost" + } + log_tester = lambda {|log, access_log| + assert_equal(1, log.length) + assert_match(/WARN Could not determine content-length of response body./, log[0]) + } + TestWEBrick.start_httpserver(config, log_tester){|server, addr, port, log| + server.mount_proc("/", lambda { |req, res| + r,w = IO.pipe + # Test for not setting chunked... + # res.chunked = true + res.body = r + w << "foo" + w.close + }) + Thread.pass while server.status != :Running + http = Net::HTTP.new(addr, port) + req = Net::HTTP::Get.new("/") + req['Connection'] = 'Keep-Alive' + begin + Timeout.timeout(2) do + http.request(req){|res| assert_equal("foo", res.body) } + end + rescue Timeout::Error + flunk('corrupted response') + end + } + end + + def test_request_handler_callback_is_deprecated + requested = 0 + config = { + :ServerName => "localhost", + :RequestHandler => Proc.new{|req, res| requested += 1 }, + } + log_tester = lambda {|log, access_log| + assert_equal(2, log.length) + assert_match(/WARN :RequestHandler is deprecated, please use :RequestCallback/, log[0]) + assert_match(%r{ERROR `/' not found\.}, log[1]) + } + TestWEBrick.start_httpserver(config, log_tester){|server, addr, port, log| + Thread.pass while server.status != :Running + + http = Net::HTTP.new(addr, port) + req = Net::HTTP::Get.new("/") + req["Host"] = "localhost:#{port}" + http.request(req){|res| assert_equal("404", res.code, log.call)} + assert_match(%r{:RequestHandler is deprecated, please use :RequestCallback$}, log.call, log.call) + } + assert_equal(1, requested) + end + + def test_shutdown_with_busy_keepalive_connection + requested = 0 + config = { + :ServerName => "localhost", + } + TestWEBrick.start_httpserver(config){|server, addr, port, log| + server.mount_proc("/", lambda {|req, res| res.body = "heffalump" }) + Thread.pass while server.status != :Running + + Net::HTTP.start(addr, port) do |http| + req = Net::HTTP::Get.new("/") + http.request(req){|res| assert_equal('Keep-Alive', res['Connection'], log.call) } + server.shutdown + begin + 10.times {|n| http.request(req); requested += 1 } + rescue + # Errno::ECONNREFUSED or similar + end + end + } + assert_equal(0, requested, "Server responded to #{requested} requests after shutdown") + end + + def test_cntrl_in_path + log_ary = [] + access_log_ary = [] + config = { + :Port => 0, + :BindAddress => '127.0.0.1', + :Logger => WEBrick::Log.new(log_ary, WEBrick::BasicLog::WARN), + :AccessLog => [[access_log_ary, '']], + } + s = WEBrick::HTTPServer.new(config) + s.mount('/foo', WEBrick::HTTPServlet::FileHandler, __FILE__) + th = Thread.new { s.start } + addr = s.listeners[0].addr + + http = Net::HTTP.new(addr[3], addr[1]) + req = Net::HTTP::Get.new('/notexist%0a/foo') + http.request(req) { |res| assert_equal('404', res.code) } + exp = %Q(ERROR `/notexist\\n/foo' not found.\n) + assert_equal 1, log_ary.size + assert_include log_ary[0], exp + ensure + s&.shutdown + th&.join + end + + def test_gigantic_request_header + log_tester = lambda {|log, access_log| + assert_equal 1, log.size + assert_include log[0], 'ERROR headers too large' + } + TestWEBrick.start_httpserver({}, log_tester){|server, addr, port, log| + server.mount('/', WEBrick::HTTPServlet::FileHandler, __FILE__) + TCPSocket.open(addr, port) do |c| + c.write("GET / HTTP/1.0\r\n") + junk = -"X-Junk: #{' ' * 1024}\r\n" + assert_raise(Errno::ECONNRESET, Errno::EPIPE, Errno::EPROTOTYPE) do + loop { c.write(junk) } + end + end + } + end + + def test_eof_in_chunk + log_tester = lambda do |log, access_log| + assert_equal 1, log.size + assert_include log[0], 'ERROR bad chunk data size' + end + TestWEBrick.start_httpserver({}, log_tester){|server, addr, port, log| + server.mount_proc('/', ->(req, res) { res.body = req.body }) + TCPSocket.open(addr, port) do |c| + c.write("POST / HTTP/1.1\r\nHost: example.com\r\n" \ + "Transfer-Encoding: chunked\r\n\r\n5\r\na") + c.shutdown(Socket::SHUT_WR) # trigger EOF in server + res = c.read + assert_match %r{\AHTTP/1\.1 400 }, res + end + } + end + + def test_big_chunks + nr_out = 3 + buf = 'big' # 3 bytes is bigger than 2! + config = { :InputBufferSize => 2 }.freeze + total = 0 + all = '' + TestWEBrick.start_httpserver(config){|server, addr, port, log| + server.mount_proc('/', ->(req, res) { + err = [] + ret = req.body do |chunk| + n = chunk.bytesize + n > config[:InputBufferSize] and err << "#{n} > :InputBufferSize" + total += n + all << chunk + end + ret.nil? or err << 'req.body should return nil' + (buf * nr_out) == all or err << 'input body does not match expected' + res.header['connection'] = 'close' + res.body = err.join("\n") + }) + TCPSocket.open(addr, port) do |c| + c.write("POST / HTTP/1.1\r\nHost: example.com\r\n" \ + "Transfer-Encoding: chunked\r\n\r\n") + chunk = "#{buf.bytesize.to_s(16)}\r\n#{buf}\r\n" + nr_out.times { c.write(chunk) } + c.write("0\r\n\r\n") + head, body = c.read.split("\r\n\r\n") + assert_match %r{\AHTTP/1\.1 200 OK}, head + assert_nil body + end + } + end +end diff --git a/tool/test/webrick/test_httpstatus.rb b/tool/test/webrick/test_httpstatus.rb new file mode 100644 index 0000000000..fd0570d5c6 --- /dev/null +++ b/tool/test/webrick/test_httpstatus.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: false +require "test/unit" +require "webrick" + +class TestWEBrickHTTPStatus < Test::Unit::TestCase + def test_info? + assert WEBrick::HTTPStatus.info?(100) + refute WEBrick::HTTPStatus.info?(200) + end + + def test_success? + assert WEBrick::HTTPStatus.success?(200) + refute WEBrick::HTTPStatus.success?(300) + end + + def test_redirect? + assert WEBrick::HTTPStatus.redirect?(300) + refute WEBrick::HTTPStatus.redirect?(400) + end + + def test_error? + assert WEBrick::HTTPStatus.error?(400) + refute WEBrick::HTTPStatus.error?(600) + end + + def test_client_error? + assert WEBrick::HTTPStatus.client_error?(400) + refute WEBrick::HTTPStatus.client_error?(500) + end + + def test_server_error? + assert WEBrick::HTTPStatus.server_error?(500) + refute WEBrick::HTTPStatus.server_error?(600) + end +end diff --git a/tool/test/webrick/test_httputils.rb b/tool/test/webrick/test_httputils.rb new file mode 100644 index 0000000000..00f297bd09 --- /dev/null +++ b/tool/test/webrick/test_httputils.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: false +require "test/unit" +require "webrick/httputils" + +class TestWEBrickHTTPUtils < Test::Unit::TestCase + include WEBrick::HTTPUtils + + def test_normilize_path + assert_equal("/foo", normalize_path("/foo")) + assert_equal("/foo/bar/", normalize_path("/foo/bar/")) + + assert_equal("/", normalize_path("/foo/../")) + assert_equal("/", normalize_path("/foo/..")) + assert_equal("/", normalize_path("/foo/bar/../../")) + assert_equal("/", normalize_path("/foo/bar/../..")) + assert_equal("/", normalize_path("/foo/bar/../..")) + assert_equal("/baz", normalize_path("/foo/bar/../../baz")) + assert_equal("/baz", normalize_path("/foo/../bar/../baz")) + assert_equal("/baz/", normalize_path("/foo/../bar/../baz/")) + assert_equal("/...", normalize_path("/bar/../...")) + assert_equal("/.../", normalize_path("/bar/../.../")) + + assert_equal("/foo/", normalize_path("/foo/./")) + assert_equal("/foo/", normalize_path("/foo/.")) + assert_equal("/foo/", normalize_path("/foo/././")) + assert_equal("/foo/", normalize_path("/foo/./.")) + assert_equal("/foo/bar", normalize_path("/foo/./bar")) + assert_equal("/foo/bar/", normalize_path("/foo/./bar/.")) + assert_equal("/foo/bar/", normalize_path("/./././foo/./bar/.")) + + assert_equal("/foo/bar/", normalize_path("//foo///.//bar/.///.//")) + assert_equal("/", normalize_path("//foo///..///bar/.///..//.//")) + + assert_raise(RuntimeError){ normalize_path("foo/bar") } + assert_raise(RuntimeError){ normalize_path("..") } + assert_raise(RuntimeError){ normalize_path("/..") } + assert_raise(RuntimeError){ normalize_path("/./..") } + assert_raise(RuntimeError){ normalize_path("/./../") } + assert_raise(RuntimeError){ normalize_path("/./../..") } + assert_raise(RuntimeError){ normalize_path("/./../../") } + assert_raise(RuntimeError){ normalize_path("/./../") } + assert_raise(RuntimeError){ normalize_path("/../..") } + assert_raise(RuntimeError){ normalize_path("/../../") } + assert_raise(RuntimeError){ normalize_path("/../../..") } + assert_raise(RuntimeError){ normalize_path("/../../../") } + assert_raise(RuntimeError){ normalize_path("/../foo/../") } + assert_raise(RuntimeError){ normalize_path("/../foo/../../") } + assert_raise(RuntimeError){ normalize_path("/foo/bar/../../../../") } + assert_raise(RuntimeError){ normalize_path("/foo/../bar/../../") } + assert_raise(RuntimeError){ normalize_path("/./../bar/") } + assert_raise(RuntimeError){ normalize_path("/./../") } + end + + def test_split_header_value + assert_equal(['foo', 'bar'], split_header_value('foo, bar')) + assert_equal(['"foo"', 'bar'], split_header_value('"foo", bar')) + assert_equal(['foo', '"bar"'], split_header_value('foo, "bar"')) + assert_equal(['*'], split_header_value('*')) + assert_equal(['W/"xyzzy"', 'W/"r2d2xxxx"', 'W/"c3piozzzz"'], + split_header_value('W/"xyzzy", W/"r2d2xxxx", W/"c3piozzzz"')) + end + + def test_escape + assert_equal("/foo/bar", escape("/foo/bar")) + assert_equal("/~foo/bar", escape("/~foo/bar")) + assert_equal("/~foo%20bar", escape("/~foo bar")) + assert_equal("/~foo%20bar", escape("/~foo bar")) + assert_equal("/~foo%09bar", escape("/~foo\tbar")) + assert_equal("/~foo+bar", escape("/~foo+bar")) + bug8425 = '[Bug #8425] [ruby-core:55052]' + assert_nothing_raised(ArgumentError, Encoding::CompatibilityError, bug8425) { + assert_equal("%E3%83%AB%E3%83%93%E3%83%BC%E3%81%95%E3%82%93", escape("\u{30EB 30D3 30FC 3055 3093}")) + } + end + + def test_escape_form + assert_equal("%2Ffoo%2Fbar", escape_form("/foo/bar")) + assert_equal("%2F~foo%2Fbar", escape_form("/~foo/bar")) + assert_equal("%2F~foo+bar", escape_form("/~foo bar")) + assert_equal("%2F~foo+%2B+bar", escape_form("/~foo + bar")) + end + + def test_unescape + assert_equal("/foo/bar", unescape("%2ffoo%2fbar")) + assert_equal("/~foo/bar", unescape("/%7efoo/bar")) + assert_equal("/~foo/bar", unescape("%2f%7efoo%2fbar")) + assert_equal("/~foo+bar", unescape("/%7efoo+bar")) + end + + def test_unescape_form + assert_equal("//foo/bar", unescape_form("/%2Ffoo/bar")) + assert_equal("//foo/bar baz", unescape_form("/%2Ffoo/bar+baz")) + assert_equal("/~foo/bar baz", unescape_form("/%7Efoo/bar+baz")) + end + + def test_escape_path + assert_equal("/foo/bar", escape_path("/foo/bar")) + assert_equal("/foo/bar/", escape_path("/foo/bar/")) + assert_equal("/%25foo/bar/", escape_path("/%foo/bar/")) + end +end diff --git a/tool/test/webrick/test_httpversion.rb b/tool/test/webrick/test_httpversion.rb new file mode 100644 index 0000000000..e50ee17971 --- /dev/null +++ b/tool/test/webrick/test_httpversion.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: false +require "test/unit" +require "webrick/httpversion" + +class TestWEBrickHTTPVersion < Test::Unit::TestCase + def setup + @v09 = WEBrick::HTTPVersion.new("0.9") + @v10 = WEBrick::HTTPVersion.new("1.0") + @v11 = WEBrick::HTTPVersion.new("1.001") + end + + def test_to_s() + assert_equal("0.9", @v09.to_s) + assert_equal("1.0", @v10.to_s) + assert_equal("1.1", @v11.to_s) + end + + def test_major() + assert_equal(0, @v09.major) + assert_equal(1, @v10.major) + assert_equal(1, @v11.major) + end + + def test_minor() + assert_equal(9, @v09.minor) + assert_equal(0, @v10.minor) + assert_equal(1, @v11.minor) + end + + def test_compar() + assert_equal(0, @v09 <=> "0.9") + assert_equal(0, @v09 <=> "0.09") + + assert_equal(-1, @v09 <=> @v10) + assert_equal(-1, @v09 <=> "1.00") + + assert_equal(1, @v11 <=> @v09) + assert_equal(1, @v11 <=> "1.0") + assert_equal(1, @v11 <=> "0.9") + end +end diff --git a/tool/test/webrick/test_server.rb b/tool/test/webrick/test_server.rb new file mode 100644 index 0000000000..815cc3ce39 --- /dev/null +++ b/tool/test/webrick/test_server.rb @@ -0,0 +1,191 @@ +# frozen_string_literal: false +require "test/unit" +require "tempfile" +require "webrick" +require_relative "utils" + +class TestWEBrickServer < Test::Unit::TestCase + class Echo < WEBrick::GenericServer + def run(sock) + while line = sock.gets + sock << line + end + end + end + + def test_server + TestWEBrick.start_server(Echo){|server, addr, port, log| + TCPSocket.open(addr, port){|sock| + sock.puts("foo"); assert_equal("foo\n", sock.gets, log.call) + sock.puts("bar"); assert_equal("bar\n", sock.gets, log.call) + sock.puts("baz"); assert_equal("baz\n", sock.gets, log.call) + sock.puts("qux"); assert_equal("qux\n", sock.gets, log.call) + } + } + end + + def test_start_exception + stopped = 0 + + log = [] + logger = WEBrick::Log.new(log, WEBrick::BasicLog::WARN) + + assert_raise(SignalException) do + listener = Object.new + def listener.to_io # IO.select invokes #to_io. + raise SignalException, 'SIGTERM' # simulate signal in main thread + end + def listener.shutdown + end + def listener.close + end + + server = WEBrick::HTTPServer.new({ + :BindAddress => "127.0.0.1", :Port => 0, + :StopCallback => Proc.new{ stopped += 1 }, + :Logger => logger, + }) + server.listeners[0].close + server.listeners[0] = listener + + server.start + end + + assert_equal(1, stopped) + assert_equal(1, log.length) + assert_match(/FATAL SignalException: SIGTERM/, log[0]) + end + + def test_callbacks + accepted = started = stopped = 0 + config = { + :AcceptCallback => Proc.new{ accepted += 1 }, + :StartCallback => Proc.new{ started += 1 }, + :StopCallback => Proc.new{ stopped += 1 }, + } + TestWEBrick.start_server(Echo, config){|server, addr, port, log| + true while server.status != :Running + sleep 1 if defined?(RubyVM::MJIT) && RubyVM::MJIT.enabled? # server.status behaves unexpectedly with --jit-wait + assert_equal(1, started, log.call) + assert_equal(0, stopped, log.call) + assert_equal(0, accepted, log.call) + TCPSocket.open(addr, port){|sock| (sock << "foo\n").gets } + TCPSocket.open(addr, port){|sock| (sock << "foo\n").gets } + TCPSocket.open(addr, port){|sock| (sock << "foo\n").gets } + assert_equal(3, accepted, log.call) + } + assert_equal(1, started) + assert_equal(1, stopped) + end + + def test_daemon + begin + r, w = IO.pipe + pid1 = Process.fork{ + r.close + WEBrick::Daemon.start + w.puts(Process.pid) + sleep 10 + } + pid2 = r.gets.to_i + assert(Process.kill(:KILL, pid2)) + assert_not_equal(pid1, pid2) + rescue NotImplementedError + # snip this test + ensure + Process.wait(pid1) if pid1 + r.close + w.close + end + end + + def test_restart_after_shutdown + address = '127.0.0.1' + port = 0 + log = [] + config = { + :BindAddress => address, + :Port => port, + :Logger => WEBrick::Log.new(log, WEBrick::BasicLog::WARN), + } + server = Echo.new(config) + client_proc = lambda {|str| + begin + ret = server.listeners.first.connect_address.connect {|s| + s.write(str) + s.close_write + s.read + } + assert_equal(str, ret) + ensure + server.shutdown + end + } + server_thread = Thread.new { server.start } + client_thread = Thread.new { client_proc.call("a") } + assert_join_threads([client_thread, server_thread]) + server.listen(address, port) + server_thread = Thread.new { server.start } + client_thread = Thread.new { client_proc.call("b") } + assert_join_threads([client_thread, server_thread]) + assert_equal([], log) + end + + def test_restart_after_stop + log = Object.new + class << log + include Test::Unit::Assertions + def <<(msg) + flunk "unexpected log: #{msg.inspect}" + end + end + client_thread = nil + wakeup = -> {client_thread.wakeup} + warn_flunk = WEBrick::Log.new(log, WEBrick::BasicLog::WARN) + server = WEBrick::HTTPServer.new( + :StartCallback => wakeup, + :StopCallback => wakeup, + :BindAddress => '0.0.0.0', + :Port => 0, + :Logger => warn_flunk) + 2.times { + server_thread = Thread.start { + server.start + } + client_thread = Thread.start { + sleep 0.1 until server.status == :Running || !server_thread.status + server.stop + sleep 0.1 until server.status == :Stop || !server_thread.status + } + assert_join_threads([client_thread, server_thread]) + } + end + + def test_port_numbers + config = { + :BindAddress => '0.0.0.0', + :Logger => WEBrick::Log.new([], WEBrick::BasicLog::WARN), + } + + ports = [0, "0"] + + ports.each do |port| + config[:Port]= port + server = WEBrick::GenericServer.new(config) + server_thread = Thread.start { server.start } + client_thread = Thread.start { + sleep 0.1 until server.status == :Running || !server_thread.status + server_port = server.listeners[0].addr[1] + server.stop + assert_equal server.config[:Port], server_port + sleep 0.1 until server.status == :Stop || !server_thread.status + } + assert_join_threads([client_thread, server_thread]) + end + + assert_raise(ArgumentError) do + config[:Port]= "FOO" + WEBrick::GenericServer.new(config) + end + end +end diff --git a/tool/test/webrick/test_ssl_server.rb b/tool/test/webrick/test_ssl_server.rb new file mode 100644 index 0000000000..4e52598bf5 --- /dev/null +++ b/tool/test/webrick/test_ssl_server.rb @@ -0,0 +1,67 @@ +require "test/unit" +require "webrick" +require "webrick/ssl" +require_relative "utils" +require 'timeout' + +class TestWEBrickSSLServer < Test::Unit::TestCase + class Echo < WEBrick::GenericServer + def run(sock) + while line = sock.gets + sock << line + end + end + end + + def test_self_signed_cert_server + assert_self_signed_cert( + :SSLEnable => true, + :SSLCertName => [["C", "JP"], ["O", "www.ruby-lang.org"], ["CN", "Ruby"]], + ) + end + + def test_self_signed_cert_server_with_string + assert_self_signed_cert( + :SSLEnable => true, + :SSLCertName => "/C=JP/O=www.ruby-lang.org/CN=Ruby", + ) + end + + def assert_self_signed_cert(config) + TestWEBrick.start_server(Echo, config){|server, addr, port, log| + io = TCPSocket.new(addr, port) + sock = OpenSSL::SSL::SSLSocket.new(io) + sock.connect + sock.puts(server.ssl_context.cert.subject.to_s) + assert_equal("/C=JP/O=www.ruby-lang.org/CN=Ruby\n", sock.gets, log.call) + sock.close + io.close + } + end + + def test_slow_connect + poke = lambda do |io, msg| + begin + sock = OpenSSL::SSL::SSLSocket.new(io) + sock.connect + sock.puts(msg) + assert_equal "#{msg}\n", sock.gets, msg + ensure + sock&.close + io.close + end + end + config = { + :SSLEnable => true, + :SSLCertName => "/C=JP/O=www.ruby-lang.org/CN=Ruby", + } + EnvUtil.timeout(10) do + TestWEBrick.start_server(Echo, config) do |server, addr, port, log| + outer = TCPSocket.new(addr, port) + inner = TCPSocket.new(addr, port) + poke.call(inner, 'fast TLS negotiation') + poke.call(outer, 'slow TLS negotiation') + end + end + end +end diff --git a/tool/test/webrick/test_utils.rb b/tool/test/webrick/test_utils.rb new file mode 100644 index 0000000000..c2b7a36e8a --- /dev/null +++ b/tool/test/webrick/test_utils.rb @@ -0,0 +1,110 @@ +# frozen_string_literal: false +require "test/unit" +require "webrick/utils" + +class TestWEBrickUtils < Test::Unit::TestCase + def teardown + WEBrick::Utils::TimeoutHandler.terminate + super + end + + def assert_expired(m) + Thread.handle_interrupt(Timeout::Error => :never, EX => :never) do + assert_empty(m::TimeoutHandler.instance.instance_variable_get(:@timeout_info)) + end + end + + def assert_not_expired(m) + Thread.handle_interrupt(Timeout::Error => :never, EX => :never) do + assert_not_empty(m::TimeoutHandler.instance.instance_variable_get(:@timeout_info)) + end + end + + EX = Class.new(StandardError) + + def test_no_timeout + m = WEBrick::Utils + assert_equal(:foo, m.timeout(10){ :foo }) + assert_expired(m) + end + + def test_nested_timeout_outer + m = WEBrick::Utils + i = 0 + assert_raise(Timeout::Error){ + m.timeout(1){ + assert_raise(Timeout::Error){ m.timeout(0.1){ i += 1; sleep(1) } } + assert_not_expired(m) + i += 1 + sleep(2) + } + } + assert_equal(2, i) + assert_expired(m) + end + + def test_timeout_default_exception + m = WEBrick::Utils + assert_raise(Timeout::Error){ m.timeout(0.01){ sleep } } + assert_expired(m) + end + + def test_timeout_custom_exception + m = WEBrick::Utils + ex = EX + assert_raise(ex){ m.timeout(0.01, ex){ sleep } } + assert_expired(m) + end + + def test_nested_timeout_inner_custom_exception + m = WEBrick::Utils + ex = EX + i = 0 + assert_raise(ex){ + m.timeout(10){ + m.timeout(0.01, ex){ i += 1; sleep } + } + sleep + } + assert_equal(1, i) + assert_expired(m) + end + + def test_nested_timeout_outer_custom_exception + m = WEBrick::Utils + ex = EX + i = 0 + assert_raise(Timeout::Error){ + m.timeout(0.01){ + m.timeout(1.0, ex){ i += 1; sleep } + } + sleep + } + assert_equal(1, i) + assert_expired(m) + end + + def test_create_listeners + addr = listener_address(0) + port = addr.slice!(1) + assert_kind_of(Integer, port, "dynamically chosen port number") + assert_equal(["AF_INET", "127.0.0.1", "127.0.0.1"], addr) + + assert_equal(["AF_INET", port, "127.0.0.1", "127.0.0.1"], + listener_address(port), + "specific port number") + + assert_equal(["AF_INET", port, "127.0.0.1", "127.0.0.1"], + listener_address(port.to_s), + "specific port number string") + end + + def listener_address(port) + listeners = WEBrick::Utils.create_listeners("127.0.0.1", port) + srv = listeners.first + assert_kind_of TCPServer, srv + srv.addr + ensure + listeners.each(&:close) if listeners + end +end diff --git a/tool/test/webrick/utils.rb b/tool/test/webrick/utils.rb new file mode 100644 index 0000000000..a8568d0a43 --- /dev/null +++ b/tool/test/webrick/utils.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: false +require "webrick" +begin + require "webrick/https" +rescue LoadError +end +require "webrick/httpproxy" + +module TestWEBrick + NullWriter = Object.new + def NullWriter.<<(msg) + puts msg if $DEBUG + return self + end + + class WEBrick::HTTPServlet::CGIHandler + remove_const :Ruby + require "envutil" unless defined?(EnvUtil) + Ruby = EnvUtil.rubybin + remove_const :CGIRunner + CGIRunner = "\"#{Ruby}\" \"#{WEBrick::Config::LIBDIR}/httpservlet/cgi_runner.rb\"" # :nodoc: + remove_const :CGIRunnerArray + CGIRunnerArray = [Ruby, "#{WEBrick::Config::LIBDIR}/httpservlet/cgi_runner.rb"] # :nodoc: + end + + RubyBin = "\"#{EnvUtil.rubybin}\"" + RubyBin << " --disable-gems" + RubyBin << " \"-I#{File.expand_path("../..", File.dirname(__FILE__))}/lib\"" + RubyBin << " \"-I#{File.dirname(EnvUtil.rubybin)}/.ext/common\"" + RubyBin << " \"-I#{File.dirname(EnvUtil.rubybin)}/.ext/#{RUBY_PLATFORM}\"" + + RubyBinArray = [EnvUtil.rubybin] + RubyBinArray << "--disable-gems" + RubyBinArray << "-I" << "#{File.expand_path("../..", File.dirname(__FILE__))}/lib" + RubyBinArray << "-I" << "#{File.dirname(EnvUtil.rubybin)}/.ext/common" + RubyBinArray << "-I" << "#{File.dirname(EnvUtil.rubybin)}/.ext/#{RUBY_PLATFORM}" + + require "test/unit" unless defined?(Test::Unit) + include Test::Unit::Assertions + extend Test::Unit::Assertions + include Test::Unit::CoreAssertions + extend Test::Unit::CoreAssertions + + module_function + + DefaultLogTester = lambda {|log, access_log| assert_equal([], log) } + + def start_server(klass, config={}, log_tester=DefaultLogTester, &block) + log_ary = [] + access_log_ary = [] + log = proc { "webrick log start:\n" + (log_ary+access_log_ary).join.gsub(/^/, " ").chomp + "\nwebrick log end" } + config = ({ + :BindAddress => "127.0.0.1", :Port => 0, + :ServerType => Thread, + :Logger => WEBrick::Log.new(log_ary, WEBrick::BasicLog::WARN), + :AccessLog => [[access_log_ary, ""]] + }.update(config)) + server = capture_output {break klass.new(config)} + server_thread = server.start + server_thread2 = Thread.new { + server_thread.join + if log_tester + log_tester.call(log_ary, access_log_ary) + end + } + addr = server.listeners[0].addr + client_thread = Thread.new { + begin + block.yield([server, addr[3], addr[1], log]) + ensure + server.shutdown + end + } + assert_join_threads([client_thread, server_thread2]) + end + + def start_httpserver(config={}, log_tester=DefaultLogTester, &block) + start_server(WEBrick::HTTPServer, config, log_tester, &block) + end + + def start_httpproxy(config={}, log_tester=DefaultLogTester, &block) + start_server(WEBrick::HTTPProxyServer, config, log_tester, &block) + end +end diff --git a/tool/test/webrick/webrick.cgi b/tool/test/webrick/webrick.cgi new file mode 100644 index 0000000000..a294fa72f9 --- /dev/null +++ b/tool/test/webrick/webrick.cgi @@ -0,0 +1,38 @@ +#!ruby +require "webrick/cgi" + +class TestApp < WEBrick::CGI + def do_GET(req, res) + res["content-type"] = "text/plain" + if req.path_info == "/dumpenv" + res.body = Marshal.dump(ENV.to_hash) + elsif (p = req.path_info) && p.length > 0 + res.body = p + elsif (q = req.query).size > 0 + res.body = q.keys.sort.collect{|key| + q[key].list.sort.collect{|v| + "#{key}=#{v}" + }.join(", ") + }.join(", ") + elsif %r{/$} =~ req.request_uri.to_s + res.body = "" + res.body << req.request_uri.to_s << "\n" + res.body << req.script_name + elsif !req.cookies.empty? + res.body = req.cookies.inject(""){|result, cookie| + result << "%s=%s\n" % [cookie.name, cookie.value] + } + res.cookies << WEBrick::Cookie.new("Customer", "WILE_E_COYOTE") + res.cookies << WEBrick::Cookie.new("Shipping", "FedEx") + else + res.body = req.script_name + end + end + + def do_POST(req, res) + do_GET(req, res) + end +end + +cgi = TestApp.new +cgi.start diff --git a/tool/test/webrick/webrick.rhtml b/tool/test/webrick/webrick.rhtml new file mode 100644 index 0000000000..a7bbe43fb5 --- /dev/null +++ b/tool/test/webrick/webrick.rhtml @@ -0,0 +1,4 @@ +req to <%= +servlet_request.request_uri +%> <%= +servlet_request.query.inspect %> diff --git a/tool/test/webrick/webrick_long_filename.cgi b/tool/test/webrick/webrick_long_filename.cgi new file mode 100644 index 0000000000..43c1af825c --- /dev/null +++ b/tool/test/webrick/webrick_long_filename.cgi @@ -0,0 +1,36 @@ +#!ruby +require "webrick/cgi" + +class TestApp < WEBrick::CGI + def do_GET(req, res) + res["content-type"] = "text/plain" + if (p = req.path_info) && p.length > 0 + res.body = p + elsif (q = req.query).size > 0 + res.body = q.keys.sort.collect{|key| + q[key].list.sort.collect{|v| + "#{key}=#{v}" + }.join(", ") + }.join(", ") + elsif %r{/$} =~ req.request_uri.to_s + res.body = "" + res.body << req.request_uri.to_s << "\n" + res.body << req.script_name + elsif !req.cookies.empty? + res.body = req.cookies.inject(""){|result, cookie| + result << "%s=%s\n" % [cookie.name, cookie.value] + } + res.cookies << WEBrick::Cookie.new("Customer", "WILE_E_COYOTE") + res.cookies << WEBrick::Cookie.new("Shipping", "FedEx") + else + res.body = req.script_name + end + end + + def do_POST(req, res) + do_GET(req, res) + end +end + +cgi = TestApp.new +cgi.start diff --git a/tool/transcode-tblgen.rb b/tool/transcode-tblgen.rb new file mode 100644 index 0000000000..dba6f33ff9 --- /dev/null +++ b/tool/transcode-tblgen.rb @@ -0,0 +1,1118 @@ +# frozen_string_literal: true + +require 'optparse' +require 'erb' +require 'fileutils' +require 'pp' + +class Array + unless [].respond_to? :product + def product(*args) + if args.empty? + self.map {|e| [e] } + else + result = [] + self.each {|e0| + result.concat args.first.product(*args[1..-1]).map {|es| [e0, *es] } + } + result + end + end + end +end + +class String + unless "".respond_to? :start_with? + def start_with?(*prefixes) + prefixes.each {|prefix| + return true if prefix.length <= self.length && prefix == self[0, prefix.length] + } + false + end + end +end + +NUM_ELEM_BYTELOOKUP = 2 + +C_ESC = { + "\\" => "\\\\", + '"' => '\"', + "\n" => '\n', +} + +0x00.upto(0x1f) {|ch| C_ESC[[ch].pack("C")] ||= "\\%03o" % ch } +0x7f.upto(0xff) {|ch| C_ESC[[ch].pack("C")] = "\\%03o" % ch } +C_ESC_PAT = Regexp.union(*C_ESC.keys) + +def c_esc(str) + '"' + str.gsub(C_ESC_PAT) { C_ESC[$&] } + '"' +end + +HEX2 = /(?:[0-9A-Fa-f]{2})/ + +class ArrayCode + def initialize(type, name) + @type = type + @name = name + @len = 0; + @content = ''.dup + end + + def length + @len + end + + def insert_at_last(num, str) + # newnum = self.length + num + @content << str + @len += num + end + + def to_s + <<"End" +static const #{@type} +#{@name}[#{@len}] = { +#{@content}}; +End + end +end + +class Action + def initialize(value) + @value = value + end + attr_reader :value + + def hash + @value.hash + end + + def eql?(other) + self.class == other.class && + @value == other.value + end + alias == eql? +end + +class Branch + def initialize(byte_min, byte_max, child_tree) + @byte_min = byte_min + @byte_max = byte_max + @child_tree = child_tree + @hash = byte_min.hash ^ byte_max.hash ^ child_tree.hash + end + attr_reader :byte_min, :byte_max, :child_tree, :hash + + def eql?(other) + self.class == other.class && + @hash == other.hash && + @byte_min == other.byte_min && + @byte_max == other.byte_max && + @child_tree == other.child_tree + end + alias == eql? +end + +class ActionMap + def self.parse_to_rects(mapping) + rects = [] + n = 0 + mapping.each {|pat, action| + pat = pat.to_s + if /\A\s*\(empset\)\s*\z/ =~ pat + next + elsif /\A\s*\(empstr\)\s*\z/ =~ pat + rects << ['', '', action] + n += 1 + elsif /\A\s*(#{HEX2}+)\s*\z/o =~ pat + hex = $1.upcase + rects << [hex, hex, action] + elsif /\A\s*((#{HEX2}|\{#{HEX2}(?:-#{HEX2})?(,#{HEX2}(?:-#{HEX2})?)*\})+(\s+|\z))*\z/o =~ pat + pat = pat.upcase + pat.scan(/\S+/) { + pat1 = $& + ranges_list = [] + pat1.scan(/#{HEX2}|\{([^\}]*)\}/o) { + ranges_list << [] + if !$1 + ranges_list.last << [$&,$&] + else + set = {} + $1.scan(/(#{HEX2})(?:-(#{HEX2}))?/o) { + if !$2 + c = $1.to_i(16) + set[c] = true + else + b = $1.to_i(16) + e = $2.to_i(16) + b.upto(e) {|_| set[_] = true } + end + } + i = nil + 0.upto(256) {|j| + if set[j] + if !i + i = j + end + if !set[j+1] + ranges_list.last << ["%02X" % i, "%02X" % j] + i = nil + end + end + } + end + } + first_ranges = ranges_list.shift + first_ranges.product(*ranges_list).each {|range_list| + min = range_list.map {|x, y| x }.join + max = range_list.map {|x, y| y }.join + rects << [min, max, action] + } + } + else + raise ArgumentError, "invalid pattern: #{pat.inspect}" + end + } + rects + end + + def self.unambiguous_action(actions0) + actions = actions0.uniq + if actions.length == 1 + actions[0] + else + actions.delete(:nomap0) + if actions.length == 1 + actions[0] + else + raise ArgumentError, "ambiguous actions: #{actions0.inspect}" + end + end + end + + def self.build_tree(rects) + expand(rects) {|prefix, actions| + unambiguous_action(actions) + } + end + + def self.parse(mapping) + rects = parse_to_rects(mapping) + tree = build_tree(rects) + self.new(tree) + end + + def self.merge_rects(*rects_list) + if rects_list.length < 2 + raise ArgumentError, "not enough arguments" + end + + all_rects = [] + rects_list.each_with_index {|rects, i| + all_rects.concat rects.map {|min, max, action| [min, max, [i, action]] } + } + + tree = expand(all_rects) {|prefix, actions| + args = Array.new(rects_list.length) { [] } + actions.each {|i, action| + args[i] << action + } + yield(prefix, *args) + } + + self.new(tree) + end + + def self.merge(*mappings, &block) + merge_rects(*mappings.map {|m| parse_to_rects(m) }, &block) + end + + def self.merge2(map1, map2, &block) + rects1 = parse_to_rects(map1) + rects2 = parse_to_rects(map2) + + actions = [] + all_rects = [] + + rects1.each {|rect| + _, _, action = rect + rect[2] = actions.length + actions << action + all_rects << rect + } + + boundary = actions.length + + rects2.each {|rect| + _, _, action = rect + rect[2] = actions.length + actions << action + all_rects << rect + } + + tree = expand(all_rects) {|prefix, as0| + as1 = [] + as2 = [] + as0.each {|i| + if i < boundary + as1 << actions[i] + else + as2 << actions[i] + end + } + yield(prefix, as1, as2) + } + + self.new(tree) + end + + def self.expand(rects, &block) + #numsing = numreg = 0 + #rects.each {|min, max, action| if min == max then numsing += 1 else numreg += 1 end } + #puts "#{numsing} singleton mappings and #{numreg} region mappings." + singleton_rects = [] + region_rects = [] + rects.each {|rect| + min, max, = rect + if min == max + singleton_rects << rect + else + region_rects << rect + end + } + @singleton_rects = singleton_rects.sort_by {|min, max, action| min } + @singleton_rects.reverse! + ret = expand_rec("", region_rects, &block) + @singleton_rects = nil + ret + end + + TMPHASH = {} + def self.expand_rec(prefix, region_rects, &block) + return region_rects if region_rects.empty? && !((s_rect = @singleton_rects.last) && s_rect[0].start_with?(prefix)) + if region_rects.empty? ? s_rect[0].length == prefix.length : region_rects[0][0].empty? + h = TMPHASH + while (s_rect = @singleton_rects.last) && s_rect[0].start_with?(prefix) + min, _, action = @singleton_rects.pop + raise ArgumentError, "ambiguous pattern: #{prefix}" if min.length != prefix.length + h[action] = true + end + for min, _, action in region_rects + raise ArgumentError, "ambiguous pattern: #{prefix}" if !min.empty? + h[action] = true + end + tree = Action.new(block.call(prefix, h.keys)) + h.clear + else + tree = [] + each_firstbyte_range(prefix, region_rects) {|byte_min, byte_max, r_rects2| + if byte_min == byte_max + prefix2 = prefix + "%02X" % byte_min + else + prefix2 = prefix + "{%02X-%02X}" % [byte_min, byte_max] + end + child_tree = expand_rec(prefix2, r_rects2, &block) + tree << Branch.new(byte_min, byte_max, child_tree) + } + end + return tree + end + + def self.each_firstbyte_range(prefix, region_rects) + index_from = TMPHASH + + region_ary = [] + region_rects.each {|min, max, action| + raise ArgumentError, "ambiguous pattern: #{prefix}" if min.empty? + min_firstbyte = min[0,2].to_i(16) + min_rest = min[2..-1] + max_firstbyte = max[0,2].to_i(16) + max_rest = max[2..-1] + region_ary << [min_firstbyte, max_firstbyte, [min_rest, max_rest, action]] + index_from[min_firstbyte] = true + index_from[max_firstbyte+1] = true + } + + byte_from = Array.new(index_from.size) + bytes = index_from.keys + bytes.sort! + bytes.reverse! + bytes.each_with_index {|byte, i| + index_from[byte] = i + byte_from[i] = byte + } + + region_rects_ary = Array.new(index_from.size) { [] } + region_ary.each {|min_firstbyte, max_firstbyte, rest_elt| + index_from[min_firstbyte].downto(index_from[max_firstbyte+1]+1) {|i| + region_rects_ary[i] << rest_elt + } + } + + index_from.clear + + r_rects = region_rects_ary.pop + region_byte = byte_from.pop + prev_r_start = region_byte + prev_r_rects = [] + while r_rects && (s_rect = @singleton_rects.last) && (seq = s_rect[0]).start_with?(prefix) + singleton_byte = seq[prefix.length, 2].to_i(16) + min_byte = singleton_byte < region_byte ? singleton_byte : region_byte + if prev_r_start < min_byte && !prev_r_rects.empty? + yield prev_r_start, min_byte-1, prev_r_rects + end + if region_byte < singleton_byte + prev_r_start = region_byte + prev_r_rects = r_rects + r_rects = region_rects_ary.pop + region_byte = byte_from.pop + elsif region_byte > singleton_byte + yield singleton_byte, singleton_byte, prev_r_rects + prev_r_start = singleton_byte+1 + else # region_byte == singleton_byte + prev_r_start = region_byte+1 + prev_r_rects = r_rects + r_rects = region_rects_ary.pop + region_byte = byte_from.pop + yield singleton_byte, singleton_byte, prev_r_rects + end + end + + while r_rects + if prev_r_start < region_byte && !prev_r_rects.empty? + yield prev_r_start, region_byte-1, prev_r_rects + end + prev_r_start = region_byte + prev_r_rects = r_rects + r_rects = region_rects_ary.pop + region_byte = byte_from.pop + end + + while (s_rect = @singleton_rects.last) && (seq = s_rect[0]).start_with?(prefix) + singleton_byte = seq[prefix.length, 2].to_i(16) + yield singleton_byte, singleton_byte, [] + end + end + + def initialize(tree) + @tree = tree + end + + def inspect + "\#<#{self.class}:" + + @tree.inspect + + ">" + end + + def max_input_length_rec(tree) + case tree + when Action + 0 + else + tree.map {|branch| + max_input_length_rec(branch.child_tree) + }.max + 1 + end + end + + def max_input_length + max_input_length_rec(@tree) + end + + def empty_action + if @tree.kind_of? Action + @tree.value + else + nil + end + end + + OffsetsMemo = {} + InfosMemo = {} + + def format_offsets(min, max, offsets) + offsets = offsets[min..max] + code = "%d, %d,\n" % [min, max] + 0.step(offsets.length-1,16) {|i| + code << " " + code << offsets[i,8].map {|off| "%3d," % off.to_s }.join('') + if i+8 < offsets.length + code << " " + code << offsets[i+8,8].map {|off| "%3d," % off.to_s }.join('') + end + code << "\n" + } + code + end + + UsedName = {} + + StrMemo = {} + + def str_name(bytes) + size = @bytes_code.length + rawbytes = [bytes].pack("H*") + + n = nil + if !n && !(suf = rawbytes.gsub(/[^A-Za-z0-9_]/, '')).empty? && !UsedName[nn = "str1_" + suf] then n = nn end + if !n && !UsedName[nn = "str1_" + bytes] then n = nn end + n ||= "str1s_#{size}" + + StrMemo[bytes] = n + UsedName[n] = true + n + end + + def gen_str(bytes) + if n = StrMemo[bytes] + n + else + len = bytes.length/2 + size = @bytes_code.length + n = str_name(bytes) + @bytes_code.insert_at_last(1 + len, + "\#define #{n} makeSTR1(#{size})\n" + + " makeSTR1LEN(#{len})," + bytes.gsub(/../, ' 0x\&,') + "\n\n") + n + end + end + + def generate_info(info) + case info + when :nomap, :nomap0 + # :nomap0 is low priority. it never collides. + "NOMAP" + when :undef + "UNDEF" + when :invalid + "INVALID" + when :func_ii + "FUNii" + when :func_si + "FUNsi" + when :func_io + "FUNio" + when :func_so + "FUNso" + when /\A(#{HEX2})\z/o + "o1(0x#$1)" + when /\A(#{HEX2})(#{HEX2})\z/o + "o2(0x#$1,0x#$2)" + when /\A(#{HEX2})(#{HEX2})(#{HEX2})\z/o + "o3(0x#$1,0x#$2,0x#$3)" + when /funsio\((\d+)\)/ + "funsio(#{$1})" + when /\A(#{HEX2})(3[0-9])(#{HEX2})(3[0-9])\z/o + "g4(0x#$1,0x#$2,0x#$3,0x#$4)" + when /\A(f[0-7])(#{HEX2})(#{HEX2})(#{HEX2})\z/o + "o4(0x#$1,0x#$2,0x#$3,0x#$4)" + when /\A(#{HEX2}){4,259}\z/o + gen_str(info.upcase) + when /\A\/\*BYTE_LOOKUP\*\// # pointer to BYTE_LOOKUP structure + $'.to_s + else + raise "unexpected action: #{info.inspect}" + end + end + + def format_infos(infos) + infos = infos.map {|info| generate_info(info) } + maxlen = infos.map {|info| info.length }.max + columns = maxlen <= 16 ? 4 : 2 + code = "".dup + 0.step(infos.length-1, columns) {|i| + code << " " + is = infos[i,columns] + is.each {|info| + code << sprintf(" %#{maxlen}s,", info) + } + code << "\n" + } + code + end + + def generate_lookup_node(name, table) + bytes_code = @bytes_code + words_code = @words_code + offsets = [] + infos = [] + infomap = {} + min = max = nil + table.each_with_index {|action, byte| + action ||= :invalid + if action != :invalid + min = byte if !min + max = byte + end + unless o = infomap[action] + infomap[action] = o = infos.length + infos[o] = action + end + offsets[byte] = o + } + infomap.clear + if !min + min = max = 0 + end + + offsets_key = [min, max, offsets[min..max]] + if n = OffsetsMemo[offsets_key] + offsets_name = n + else + offsets_name = "#{name}_offsets" + OffsetsMemo[offsets_key] = offsets_name + size = bytes_code.length + bytes_code.insert_at_last(2+max-min+1, + "\#define #{offsets_name} #{size}\n" + + format_offsets(min,max,offsets) + "\n") + end + + if n = InfosMemo[infos] + infos_name = n + else + infos_name = "#{name}_infos" + InfosMemo[infos] = infos_name + + size = words_code.length + words_code.insert_at_last(infos.length, + "\#define #{infos_name} WORDINDEX2INFO(#{size})\n" + + format_infos(infos) + "\n") + end + + size = words_code.length + words_code.insert_at_last(NUM_ELEM_BYTELOOKUP, + "\#define #{name} WORDINDEX2INFO(#{size})\n" + + <<"End" + "\n") + #{offsets_name}, + #{infos_name}, +End + end + + PreMemo = {} + NextName = "a" + + def generate_node(name_hint=nil) + if n = PreMemo[@tree] + return n + end + + table = Array.new(0x100, :invalid) + @tree.each {|branch| + byte_min, byte_max, child_tree = branch.byte_min, branch.byte_max, branch.child_tree + rest = ActionMap.new(child_tree) + if a = rest.empty_action + table.fill(a, byte_min..byte_max) + else + name_hint2 = nil + if name_hint + name_hint2 = "#{name_hint}_#{byte_min == byte_max ? '%02X' % byte_min : '%02Xto%02X' % [byte_min, byte_max]}" + end + v = "/*BYTE_LOOKUP*/" + rest.gennode(@bytes_code, @words_code, name_hint2) + table.fill(v, byte_min..byte_max) + end + } + + if !name_hint + name_hint = "fun_" + NextName + NextName.succ! + end + + PreMemo[@tree] = name_hint + + generate_lookup_node(name_hint, table) + name_hint + end + + def gennode(bytes_code, words_code, name_hint=nil) + @bytes_code = bytes_code + @words_code = words_code + name = generate_node(name_hint) + @bytes_code = nil + @words_code = nil + return name + end +end + +def citrus_mskanji_cstomb(csid, index) + case csid + when 0 + index + when 1 + index + 0x80 + when 2, 3 + row = index >> 8 + raise "invalid byte sequence" if row < 0x21 + if csid == 3 + if row <= 0x2F + offset = (row == 0x22 || row >= 0x26) ? 0xED : 0xF0 + elsif row >= 0x4D && row <= 0x7E + offset = 0xCE + else + raise "invalid byte sequence" + end + else + raise "invalid byte sequence" if row > 0x97 + offset = (row < 0x5F) ? 0x81 : 0xC1 + end + col = index & 0xFF + raise "invalid byte sequence" if (col < 0x21 || col > 0x7E) + + row -= 0x21 + col -= 0x21 + if (row & 1) == 0 + col += 0x40 + col += 1 if (col >= 0x7F) + else + col += 0x9F; + end + row = row / 2 + offset + (row << 8) | col + end.to_s(16) +end + +def citrus_euc_cstomb(csid, index) + case csid + when 0x0000 + index + when 0x8080 + index | 0x8080 + when 0x0080 + index | 0x8E80 + when 0x8000 + index | 0x8F8080 + end.to_s(16) +end + +def citrus_stateless_iso_cstomb(csid, index) + (index | 0x8080 | (csid << 16)).to_s(16) +end + +def citrus_cstomb(ces, csid, index) + case ces + when 'mskanji' + citrus_mskanji_cstomb(csid, index) + when 'euc' + citrus_euc_cstomb(csid, index) + when 'stateless_iso' + citrus_stateless_iso_cstomb(csid, index) + end +end + +SUBDIR = %w/APPLE AST BIG5 CNS CP EBCDIC EMOJI GB GEORGIAN ISO646 ISO-8859 JIS KAZAKH KOI KS MISC TCVN/ + + +def citrus_decode_mapsrc(ces, csid, mapsrcs) + table = [] + mapsrcs.split(',').each do |mapsrc| + path = [$srcdir] + mode = nil + if mapsrc.rindex(/UCS(?:@[A-Z]+)?/, 0) + mode = :from_ucs + from = mapsrc[$&.size+1..-1] + path << SUBDIR.find{|x| from.rindex(x, 0) } + else + mode = :to_ucs + path << SUBDIR.find{|x| mapsrc.rindex(x, 0) } + end + if /\bUCS@(BMP|SMP|SIP|TIP|SSP)\b/ =~ mapsrc + plane = {"BMP"=>0, "SMP"=>1, "SIP"=>2, "TIP"=>3, "SSP"=>14}[$1] + else + plane = 0 + end + plane <<= 16 + path << mapsrc.gsub(':', '@') + path = File.join(*path) + path << ".src" + path[path.rindex('/')] = '%' + STDOUT.puts 'load mapsrc %s' % path if VERBOSE_MODE > 1 + open(path, 'rb') do |f| + f.each_line do |l| + break if /^BEGIN_MAP/ =~ l + end + f.each_line do |l| + next if /^\s*(?:#|$)/ =~ l + break if /^END_MAP/ =~ l + case mode + when :from_ucs + case l + when /0x(\w+)\s*-\s*0x(\w+)\s*=\s*INVALID/ + # Citrus OOB_MODE + when /(0x\w+)\s*=\s*(0x\w+)/ + table.push << [plane | $1.hex, citrus_cstomb(ces, csid, $2.hex)] + else + raise "unknown notation '%s'"% l.chomp + end + when :to_ucs + case l + when /(0x\w+)\s*=\s*(0x\w+)/ + table.push << [citrus_cstomb(ces, csid, $1.hex), plane | $2.hex] + else + raise "unknown notation '%s'"% l.chomp + end + end + end + end + end + return table +end + +def import_ucm(path) + to_ucs = [] + from_ucs = [] + File.foreach(File.join($srcdir, "ucm", path)) do |line| + uc, bs, fb = nil + if /^<U([0-9a-fA-F]+)>\s*([\+0-9a-fA-Fx\\]+)\s*\|(\d)/ =~ line + uc = $1.hex + bs = $2.delete('x\\') + fb = $3.to_i + next if uc < 128 && uc == bs.hex + elsif /^([<U0-9a-fA-F>+]+)\s*([\+0-9a-fA-Fx\\]+)\s*\|(\d)/ =~ line + uc = $1.scan(/[0-9a-fA-F]+>/).map(&:hex).pack("U*").unpack("H*")[0] + bs = $2.delete('x\\') + fb = $3.to_i + end + to_ucs << [bs, uc] if fb == 0 || fb == 3 + from_ucs << [uc, bs] if fb == 0 || fb == 1 + end + [to_ucs, from_ucs] +end + +def encode_utf8(map) + r = [] + map.each {|k, v| + # integer means UTF-8 encoded sequence. + k = [k].pack("U").unpack("H*")[0].upcase if Integer === k + v = [v].pack("U").unpack("H*")[0].upcase if Integer === v + r << [k,v] + } + r +end + +UnspecifiedValidEncoding = Object.new + +def transcode_compile_tree(name, from, map, valid_encoding) + map = encode_utf8(map) + h = {} + map.each {|k, v| + h[k] = v unless h[k] # use first mapping + } + if valid_encoding.equal? UnspecifiedValidEncoding + valid_encoding = ValidEncoding.fetch(from) + end + if valid_encoding + am = ActionMap.merge2(h, {valid_encoding => :undef}) {|prefix, as1, as2| + a1 = as1.empty? ? nil : ActionMap.unambiguous_action(as1) + a2 = as2.empty? ? nil : ActionMap.unambiguous_action(as2) + if !a2 + raise "invalid mapping: #{prefix}" + end + a1 || a2 + } + else + am = ActionMap.parse(h) + end + h.clear + + max_input = am.max_input_length + defined_name = am.gennode(TRANSCODE_GENERATED_BYTES_CODE, TRANSCODE_GENERATED_WORDS_CODE, name) + return defined_name, max_input +end + +TRANSCODERS = [] +TRANSCODE_GENERATED_TRANSCODER_CODE = ''.dup + +def transcode_tbl_only(from, to, map, valid_encoding=UnspecifiedValidEncoding) + if VERBOSE_MODE > 1 + if from.empty? || to.empty? + STDOUT.puts "converter for #{from.empty? ? to : from}" + else + STDOUT.puts "converter from #{from} to #{to}" + end + end + id_from = from.tr('^0-9A-Za-z', '_') + id_to = to.tr('^0-9A-Za-z', '_') + if from == "UTF-8" + tree_name = "to_#{id_to}" + elsif to == "UTF-8" + tree_name = "from_#{id_from}" + else + tree_name = "from_#{id_from}_to_#{id_to}" + end + real_tree_name, max_input = transcode_compile_tree(tree_name, from, map, valid_encoding) + return map, tree_name, real_tree_name, max_input +end + +# +# call-seq: +# transcode_tblgen(from_name, to_name, map [, valid_encoding_check [, ascii_compatibility]]) -> '' +# +# Returns an empty string just in case the result is used somewhere. +# Stores the actual product for later output with transcode_generated_code and +# transcode_register_code. +# +# The first argument is a string that will be used for the source (from) encoding. +# The second argument is a string that will be used for the target (to) encoding. +# +# The third argument is the actual data, a map represented as an array of two-element +# arrays. Each element of the array stands for one character being converted. The +# first element of each subarray is the code of the character in the source encoding, +# the second element of each subarray is the code of the character in the target encoding. +# +# Each code (i.e. byte sequence) is represented as a string of hexadecimal characters +# of even length. Codes can also be represented as integers (usually in the form Ox...), +# in which case they are interpreted as Unicode codepoints encoded in UTF-8. So as +# an example, 0x677E is the same as "E69DBE" (but somewhat easier to produce and check). +# +# In addition, the following symbols can also be used instead of actual codes in the +# second element of a subarray: +# :nomap (no mapping, just copy input to output), :nomap0 (same as :nomap, but low priority), +# :undef (input code undefined in the destination encoding), +# :invalid (input code is an invalid byte sequence in the source encoding), +# :func_ii, :func_si, :func_io, :func_so (conversion by function with specific call +# convention). +# +# The forth argument specifies the overall structure of the encoding. For examples, +# see ValidEncoding below. This is used to cross-check the data in the third argument +# and to automatically add :undef and :invalid mappings where necessary. +# +# The fifth argument gives the ascii-compatibility of the transcoding. See +# rb_transcoder_asciicompat_type_t in transcode_data.h for details. In most +# cases, this argument can be left out. +# +def transcode_tblgen(from, to, map, valid_encoding=UnspecifiedValidEncoding, + ascii_compatibility='asciicompat_converter') + map, tree_name, real_tree_name, max_input = transcode_tbl_only(from, to, map, valid_encoding) + transcoder_name = "rb_#{tree_name}" + TRANSCODERS << transcoder_name + input_unit_length = UnitLength[from] + max_output = map.map {|k,v| String === v ? v.length/2 : 1 }.max + transcoder_code = <<"End" +static const rb_transcoder +#{transcoder_name} = { + #{c_esc from}, #{c_esc to}, #{real_tree_name}, + TRANSCODE_TABLE_INFO, + #{input_unit_length}, /* input_unit_length */ + #{max_input}, /* max_input */ + #{max_output}, /* max_output */ + #{ascii_compatibility}, /* asciicompat_type */ + 0, 0, 0, /* state_size, state_init, state_fini */ + 0, 0, 0, 0, + 0, 0, 0 +}; +End + TRANSCODE_GENERATED_TRANSCODER_CODE << transcoder_code + '' +end + +def transcode_generate_node(am, name_hint=nil) + STDOUT.puts "converter for #{name_hint}" if VERBOSE_MODE > 1 + am.gennode(TRANSCODE_GENERATED_BYTES_CODE, TRANSCODE_GENERATED_WORDS_CODE, name_hint) + '' +end + +def transcode_generated_code + TRANSCODE_GENERATED_BYTES_CODE.to_s + + TRANSCODE_GENERATED_WORDS_CODE.to_s + + "\#define TRANSCODE_TABLE_INFO " + + "#{OUTPUT_PREFIX}byte_array, #{TRANSCODE_GENERATED_BYTES_CODE.length}, " + + "#{OUTPUT_PREFIX}word_array, #{TRANSCODE_GENERATED_WORDS_CODE.length}, " + + "((int)sizeof(unsigned int))\n" + + TRANSCODE_GENERATED_TRANSCODER_CODE +end + +def transcode_register_code + code = ''.dup + TRANSCODERS.each {|transcoder_name| + code << " rb_register_transcoder(&#{transcoder_name});\n" + } + code +end + +UnitLength = { + 'UTF-16BE' => 2, + 'UTF-16LE' => 2, + 'UTF-32BE' => 4, + 'UTF-32LE' => 4, +} +UnitLength.default = 1 + +ValidEncoding = { + '1byte' => '{00-ff}', + '2byte' => '{00-ff}{00-ff}', + '4byte' => '{00-ff}{00-ff}{00-ff}{00-ff}', + 'US-ASCII' => '{00-7f}', + 'UTF-8' => '{00-7f} + {c2-df}{80-bf} + e0{a0-bf}{80-bf} + {e1-ec}{80-bf}{80-bf} + ed{80-9f}{80-bf} + {ee-ef}{80-bf}{80-bf} + f0{90-bf}{80-bf}{80-bf} + {f1-f3}{80-bf}{80-bf}{80-bf} + f4{80-8f}{80-bf}{80-bf}', + 'UTF-16BE' => '{00-d7,e0-ff}{00-ff} + {d8-db}{00-ff}{dc-df}{00-ff}', + 'UTF-16LE' => '{00-ff}{00-d7,e0-ff} + {00-ff}{d8-db}{00-ff}{dc-df}', + 'UTF-32BE' => '0000{00-d7,e0-ff}{00-ff} + 00{01-10}{00-ff}{00-ff}', + 'UTF-32LE' => '{00-ff}{00-d7,e0-ff}0000 + {00-ff}{00-ff}{01-10}00', + 'EUC-JP' => '{00-7f} + {a1-fe}{a1-fe} + 8e{a1-fe} + 8f{a1-fe}{a1-fe}', + 'CP51932' => '{00-7f} + {a1-fe}{a1-fe} + 8e{a1-fe}', + 'EUC-JIS-2004' => '{00-7f} + {a1-fe}{a1-fe} + 8e{a1-fe} + 8f{a1-fe}{a1-fe}', + 'Shift_JIS' => '{00-7f} + {81-9f,e0-fc}{40-7e,80-fc} + {a1-df}', + 'EUC-KR' => '{00-7f} + {a1-fe}{a1-fe}', + 'CP949' => '{00-7f} + {81-fe}{41-5a,61-7a,81-fe}', + 'Big5' => '{00-7f} + {81-fe}{40-7e,a1-fe}', + 'EUC-TW' => '{00-7f} + {a1-fe}{a1-fe} + 8e{a1-b0}{a1-fe}{a1-fe}', + 'GBK' => '{00-80} + {81-fe}{40-7e,80-fe}', + 'GB18030' => '{00-7f} + {81-fe}{40-7e,80-fe} + {81-fe}{30-39}{81-fe}{30-39}', +} + +def ValidEncoding(enc) + ValidEncoding.fetch(enc) +end + +def set_valid_byte_pattern(encoding, pattern_or_label) + pattern = + if ValidEncoding[pattern_or_label] + ValidEncoding[pattern_or_label] + else + pattern_or_label + end + if ValidEncoding[encoding] and ValidEncoding[encoding]!=pattern + raise ArgumentError, "trying to change valid byte pattern for encoding #{encoding} from #{ValidEncoding[encoding]} to #{pattern}" + end + ValidEncoding[encoding] = pattern +end + +# the following may be used in different places, so keep them here for the moment +set_valid_byte_pattern 'ASCII-8BIT', '1byte' +set_valid_byte_pattern 'Windows-31J', 'Shift_JIS' +set_valid_byte_pattern 'eucJP-ms', 'EUC-JP' + +def make_signature(filename, src) + "src=#{filename.dump}, len=#{src.length}, checksum=#{src.sum}" +end + +if __FILE__ == $0 + start_time = Time.now + + output_filename = nil + verbose_mode = 0 + force_mode = false + + op = OptionParser.new + op.def_option("--help", "show help message") { puts op; exit 0 } + op.def_option("--verbose", "verbose mode, twice for more verbose") { verbose_mode += 1 } + op.def_option("--force", "force table generation") { force_mode = true } + op.def_option("--output=FILE", "specify output file") {|arg| output_filename = arg } + op.parse! + + VERBOSE_MODE = verbose_mode + + OUTPUT_FILENAME = output_filename + OUTPUT_PREFIX = output_filename ? File.basename(output_filename)[/\A[A-Za-z0-9_]*/] : "".dup + OUTPUT_PREFIX.sub!(/\A_+/, '') + OUTPUT_PREFIX.sub!(/_*\z/, '_') + + TRANSCODE_GENERATED_BYTES_CODE = ArrayCode.new("unsigned char", "#{OUTPUT_PREFIX}byte_array") + TRANSCODE_GENERATED_WORDS_CODE = ArrayCode.new("unsigned int", "#{OUTPUT_PREFIX}word_array") + + arg = ARGV.shift + $srcdir = File.dirname(arg) + $:.unshift $srcdir unless $:.include? $srcdir + src = File.read(arg) + src.force_encoding("ascii-8bit") if src.respond_to? :force_encoding + this_script = File.read(__FILE__) + this_script.force_encoding("ascii-8bit") if this_script.respond_to? :force_encoding + + base_signature = "/* autogenerated. */\n".dup + base_signature << "/* #{make_signature(File.basename(__FILE__), this_script)} */\n" + base_signature << "/* #{make_signature(File.basename(arg), src)} */\n" + + if !force_mode && output_filename && File.readable?(output_filename) + old_signature = File.open(output_filename) {|f| f.gets("").chomp } + chk_signature = base_signature.dup + old_signature.each_line {|line| + if %r{/\* src="([0-9a-z_.-]+)",} =~ line + name = $1 + next if name == File.basename(arg) || name == File.basename(__FILE__) + path = File.join($srcdir, name) + if File.readable? path + chk_signature << "/* #{make_signature(name, File.read(path))} */\n" + end + end + } + if old_signature == chk_signature + now = Time.now + File.utime(now, now, output_filename) + STDOUT.puts "already up-to-date: #{output_filename}" if VERBOSE_MODE > 0 + exit + end + end + + if VERBOSE_MODE > 0 + if output_filename + STDOUT.puts "generating #{output_filename} ..." + end + end + + libs1 = $".dup + if ERB.instance_method(:initialize).parameters.assoc(:key) # Ruby 2.6+ + erb = ERB.new(src, trim_mode: '%') + else + erb = ERB.new(src, nil, '%') + end + erb.filename = arg + erb_result = erb.result(binding) + libs2 = $".dup + + libs = libs2 - libs1 + lib_sigs = ''.dup + libs.each {|lib| + lib = File.basename(lib) + path = File.join($srcdir, lib) + if File.readable? path + lib_sigs << "/* #{make_signature(lib, File.read(path))} */\n" + end + } + + result = ''.dup + result << base_signature + result << lib_sigs + result << "\n" + result << erb_result + result << "\n" + + if output_filename + new_filename = output_filename + ".new" + FileUtils.mkdir_p(File.dirname(output_filename)) + File.open(new_filename, "wb") {|f| f << result } + File.rename(new_filename, output_filename) + tms = Process.times + elapsed = Time.now - start_time + STDOUT.puts "done. (#{'%.2f' % tms.utime}user #{'%.2f' % tms.stime}system #{'%.2f' % elapsed}elapsed)" if VERBOSE_MODE > 1 + else + print result + end +end diff --git a/tool/transform_mjit_header.rb b/tool/transform_mjit_header.rb new file mode 100644 index 0000000000..2359ceab7c --- /dev/null +++ b/tool/transform_mjit_header.rb @@ -0,0 +1,326 @@ +# Copyright (C) 2017 Vladimir Makarov, <vmakarov@redhat.com> +# This is a script to transform functions to static inline. +# Usage: transform_mjit_header.rb <c-compiler> <header file> <out> + +require 'fileutils' +require 'tempfile' + +PROGRAM = File.basename($0, ".*") + +module MJITHeader + ATTR_VALUE_REGEXP = /[^()]|\([^()]*\)/ + ATTR_REGEXP = /__attribute__\s*\(\(#{ATTR_VALUE_REGEXP}*\)\)/ + # Example: + # VALUE foo(int bar) + # VALUE __attribute__ ((foo)) bar(int baz) + # __attribute__ ((foo)) VALUE bar(int baz) + FUNC_HEADER_REGEXP = /\A[^\[{(]*(\s*#{ATTR_REGEXP})*[^\[{(]*\((#{ATTR_REGEXP}|[^()])*\)(\s*#{ATTR_REGEXP})*\s*/ + TARGET_NAME_REGEXP = /\A(rb|ruby|vm|insn|attr|Init)_/ + + # Predefined macros for compilers which are already supported by MJIT. + # We're going to support cl.exe too (WIP) but `cl.exe -E` can't produce macro. + SUPPORTED_CC_MACROS = [ + '__GNUC__', # gcc + '__clang__', # clang + ] + + # These macros are relied on this script's transformation + PREFIXED_MACROS = [ + 'ALWAYS_INLINE', + 'COLDFUNC', + 'inline', + 'RBIMPL_ATTR_COLD', + ] + + # For MinGW's ras.h. Those macros have its name in its definition and can't be preprocessed multiple times. + RECURSIVE_MACROS = %w[ + RASCTRYINFO + RASIPADDR + ] + + IGNORED_FUNCTIONS = [ + 'rb_vm_search_method_slowpath', # This increases the time to compile when inlined. So we use it as external function. + 'rb_equal_opt', # Not used from VM and not compilable + ] + + ALWAYS_INLINED_FUNCTIONS = [ + 'vm_opt_plus', + 'vm_opt_minus', + 'vm_opt_mult', + 'vm_opt_div', + 'vm_opt_mod', + 'vm_opt_neq', + 'vm_opt_lt', + 'vm_opt_le', + 'vm_opt_gt', + 'vm_opt_ge', + 'vm_opt_ltlt', + 'vm_opt_and', + 'vm_opt_or', + 'vm_opt_aref', + 'vm_opt_aset', + 'vm_opt_aref_with', + 'vm_opt_aset_with', + 'vm_opt_not', + ] + + COLD_FUNCTIONS = %w[ + setup_parameters_complex + vm_call_iseq_setup + vm_call_iseq_setup_2 + vm_call_iseq_setup_tailcall + vm_call_method_each_type + vm_ic_update + ] + + # Return start..stop of last decl in CODE ending STOP + def self.find_decl(code, stop) + level = 0 + i = stop + while i = code.rindex(/[;{}]/, i) + if level == 0 && stop != i && decl_found?($&, i) + return decl_start($&, i)..stop + end + case $& + when '}' + level += 1 + when '{' + level -= 1 + end + i -= 1 + end + nil + end + + def self.decl_found?(code, i) + i == 0 || code == ';' || code == '}' + end + + def self.decl_start(code, i) + if i == 0 && code != ';' && code != '}' + 0 + else + i + 1 + end + end + + # Given DECL return the name of it, nil if failed + def self.decl_name_of(decl) + ident_regex = /\w+/ + decl = decl.gsub(/^#.+$/, '') # remove macros + reduced_decl = decl.gsub(ATTR_REGEXP, '') # remove attributes + su1_regex = /{[^{}]*}/ + su2_regex = /{([^{}]|#{su1_regex})*}/ + su3_regex = /{([^{}]|#{su2_regex})*}/ # 3 nested structs/unions is probably enough + reduced_decl.gsub!(su3_regex, '') # remove structs/unions in the header + id_seq_regex = /\s*(?:#{ident_regex}(?:\s+|\s*[*]+\s*))*/ + # Process function header: + match = /\A#{id_seq_regex}(?<name>#{ident_regex})\s*\(/.match(reduced_decl) + return match[:name] if match + # Process non-function declaration: + reduced_decl.gsub!(/\s*=[^;]+(?=;)/, '') # remove initialization + match = /#{id_seq_regex}(?<name>#{ident_regex})/.match(reduced_decl); + return match[:name] if match + nil + end + + # Return true if CC with CFLAGS compiles successfully the current code. + # Use STAGE in the message in case of a compilation failure + def self.check_code!(code, cc, cflags, stage) + with_code(code) do |path| + cmd = "#{cc} #{cflags} #{path}" + out = IO.popen(cmd, err: [:child, :out], &:read) + unless $?.success? + STDERR.puts "error in #{stage} header file:\n#{out}" + exit false + end + end + end + + # Remove unpreprocessable macros + def self.remove_harmful_macros!(code) + code.gsub!(/^#define #{Regexp.union(RECURSIVE_MACROS)} .*$/, '') + end + + # -dD outputs those macros, and it produces redefinition warnings or errors + # This assumes common.mk passes `-DMJIT_HEADER` first when it creates rb_mjit_header.h. + def self.remove_predefined_macros!(code) + code.sub!(/\A(#define [^\n]+|\n)*(#define MJIT_HEADER 1\n)/, '\2') + end + + # Return [macro, others]. But others include PREFIXED_MACROS to be used in code. + def self.separate_macro_and_code(code) + code.lines.partition do |l| + l.start_with?('#') && PREFIXED_MACROS.all? { |m| !l.start_with?("#define #{m}") } + end.map! { |lines| lines.join('') } + end + + def self.write(code, out) + # create with strict permission, then will install proper + # permission + FileUtils.mkdir_p(File.dirname(out), mode: 0700) + File.binwrite("#{out}.new", code, perm: 0600) + FileUtils.mv("#{out}.new", out) + end + + # Note that this checks runruby. This conservatively covers platform names. + def self.windows? + RUBY_PLATFORM =~ /mswin|mingw|msys/ + end + + def self.cl_exe?(cc) + cc =~ /\Acl(\z| |\.exe)/ + end + + # If code has macro which only supported compilers predefine, return true. + def self.supported_header?(code) + SUPPORTED_CC_MACROS.any? { |macro| code =~ /^#\s*define\s+#{Regexp.escape(macro)}\b/ } + end + + # This checks if syntax check outputs one of the following messages. + # "error: conflicting types for 'restrict'" + # "error: redefinition of parameter 'restrict'" + # If it's true, this script regards platform as AIX or Solaris and adds -std=c99 as workaround. + def self.conflicting_types?(code, cc, cflags) + with_code(code) do |path| + cmd = "#{cc} #{cflags} #{path}" + out = IO.popen(cmd, err: [:child, :out], &:read) + !$?.success? && + (out.match?(/error: conflicting types for '[^']+'/) || + out.match?(/error: redefinition of parameter '[^']+'/)) + end + end + + def self.with_code(code) + # for `system_header` pragma which can't be in the main file. + Tempfile.open(['', '.h'], mode: File::BINARY) do |f| + f.puts code + f.close + Tempfile.open(['', '.c'], mode: File::BINARY) do |c| + c.puts <<SRC +#include "#{f.path}" +SRC + c.close + return yield(c.path) + end + end + end + private_class_method :with_code +end + +if ARGV.size != 3 + abort "Usage: #{$0} <c-compiler> <header file> <out>" +end + +if STDOUT.tty? + require_relative 'lib/colorize' + color = Colorize.new +end +cc = ARGV[0] +code = File.binread(ARGV[1]) # Current version of the header file. +outfile = ARGV[2] +if MJITHeader.cl_exe?(cc) + cflags = '-DMJIT_HEADER -Zs' +else + cflags = '-S -DMJIT_HEADER -fsyntax-only -Werror=implicit-function-declaration -Werror=implicit-int -Wfatal-errors' +end + +if !MJITHeader.cl_exe?(cc) && !MJITHeader.supported_header?(code) + puts "This compiler (#{cc}) looks not supported for MJIT. Giving up to generate MJIT header." + MJITHeader.write("#error MJIT does not support '#{cc}' yet", outfile) + exit +end + +MJITHeader.remove_predefined_macros!(code) + +if MJITHeader.windows? # transformation is broken with Windows headers for now + MJITHeader.remove_harmful_macros!(code) + MJITHeader.check_code!(code, cc, cflags, 'initial') + puts "\nSkipped transforming external functions to static on Windows." + MJITHeader.write(code, outfile) + exit +end + +macro, code = MJITHeader.separate_macro_and_code(code) # note: this does not work on MinGW +code = <<header + code +#ifdef __GNUC__ +# pragma GCC system_header +#endif +header +code_to_check = "#{code}#{macro}" # macro should not affect code again + +if MJITHeader.conflicting_types?(code_to_check, cc, cflags) + cflags = "#{cflags} -std=c99" # For AIX gcc +end + +# Check initial file correctness in the manner of final output. +MJITHeader.check_code!(code_to_check, cc, cflags, 'initial') + +stop_pos = -1 +extern_names = [] +transform_logs = Hash.new { |h, k| h[k] = [] } + +# This loop changes function declarations to static inline. +while (decl_range = MJITHeader.find_decl(code, stop_pos)) + stop_pos = decl_range.begin - 1 + decl = code[decl_range] + decl_name = MJITHeader.decl_name_of(decl) + + if MJITHeader::IGNORED_FUNCTIONS.include?(decl_name) && /#{MJITHeader::FUNC_HEADER_REGEXP}{/.match(decl) + transform_logs[:def_to_decl] << decl_name + code[decl_range] = decl.sub(/{.+}/m, ';') + elsif MJITHeader::COLD_FUNCTIONS.include?(decl_name) && match = /#{MJITHeader::FUNC_HEADER_REGEXP}{/.match(decl) + header = match[0].sub(/{\z/, '').strip + header = "static #{header.sub(/\A((static|inline) )+/, '')}" + decl[match.begin(0)...match.end(0)] = '{' # remove header + code[decl_range] = "\nCOLDFUNC #{header} #{decl}" + elsif MJITHeader::ALWAYS_INLINED_FUNCTIONS.include?(decl_name) && match = /#{MJITHeader::FUNC_HEADER_REGEXP}{/.match(decl) + header = match[0].sub(/{\z/, '').strip + header = "static inline #{header.sub(/\A((static|inline) )+/, '')}" + decl[match.begin(0)...match.end(0)] = '{' # remove header + code[decl_range] = "\nALWAYS_INLINE(#{header});\n#{header} #{decl}" + elsif extern_names.include?(decl_name) && (decl =~ /#{MJITHeader::FUNC_HEADER_REGEXP};/) + decl.sub!(/(extern|static|inline) /, ' ') + unless decl_name =~ /\Aattr_\w+_\w+\z/ # skip too-many false-positive warnings in insns_info.inc. + transform_logs[:static_inline_decl] << decl_name + end + + code[decl_range] = "static inline #{decl}" + elsif (match = /#{MJITHeader::FUNC_HEADER_REGEXP}{/.match(decl)) && (header = match[0]) !~ /static/ + unless decl_name.match(MJITHeader::TARGET_NAME_REGEXP) + transform_logs[:skipped] << decl_name + next + end + + extern_names << decl_name + decl[match.begin(0)...match.end(0)] = '' + + if decl =~ /\bstatic\b/ + abort "#{PROGRAM}: a static decl was found inside external definition #{decl_name.dump}" + end + + header.sub!(/(extern|inline) /, ' ') + unless decl_name =~ /\Aattr_\w+_\w+\z/ # skip too-many false-positive warnings in insns_info.inc. + transform_logs[:static_inline_def] << decl_name + end + code[decl_range] = "static inline #{header}#{decl}" + end +end + +code << macro + +# Check the final file correctness +MJITHeader.check_code!(code, cc, cflags, 'final') + +MJITHeader.write(code, outfile) + +messages = { + def_to_decl: 'changing definition to declaration', + static_inline_def: 'making external definition static inline', + static_inline_decl: 'making declaration static inline', + skipped: 'SKIPPED to transform', +} +transform_logs.each do |key, decl_names| + decl_names = decl_names.map { |s| color.bold(s) } if color + puts("#{PROGRAM}: #{messages.fetch(key)}: #{decl_names.join(', ')}") +end diff --git a/tool/travis_retry.sh b/tool/travis_retry.sh new file mode 100755 index 0000000000..9b79c56550 --- /dev/null +++ b/tool/travis_retry.sh @@ -0,0 +1,13 @@ +#!/bin/sh -eu +# The modified version of `travis_retry` to support custom backoffs, which is used by .travis.yml. +# https://github.com/travis-ci/travis-build/blob/master/lib/travis/build/bash/travis_retry.bash + +for sleep in 0 ${WAITS:- 1 25 100}; do + sleep "$sleep" + + echo "+ $@" + if "$@"; then + exit 0 + fi +done +exit 1 diff --git a/tool/travis_wait.sh b/tool/travis_wait.sh new file mode 100755 index 0000000000..471b765df8 --- /dev/null +++ b/tool/travis_wait.sh @@ -0,0 +1,18 @@ +#!/bin/bash -eu +# The modified version of `travis_wait` to output a log as the command goes. +# https://github.com/travis-ci/travis-ci/issues/4190#issuecomment-353342526 + +# Produce an output log every 9 minutes as the timeout without output is 10 +# minutes. A job finishes with a timeout if it takes longer than 50 minutes. +# https://docs.travis-ci.com/user/customizing-the-build#build-timeouts +while sleep 9m; do + # Print message with bash variable SECONDS. + echo "====[ $SECONDS seconds still running ]====" +done & + +echo "+ $@" +"$@" + +jobs +kill %1 +exit 0 diff --git a/tool/update-bundled_gems.rb b/tool/update-bundled_gems.rb new file mode 100755 index 0000000000..5b9c6b6974 --- /dev/null +++ b/tool/update-bundled_gems.rb @@ -0,0 +1,20 @@ +#!ruby -pla +BEGIN { + require 'rubygems' +} +unless /^[^#]/ !~ (gem = $F[0]) + (gem, src), = Gem::SpecFetcher.fetcher.detect(:latest) {|s| + s.platform == "ruby" && s.name == gem + } + gem = src.fetch_spec(gem) + uri = gem.metadata["source_code_uri"] || gem.homepage + uri = uri.sub(%r[\Ahttps://github\.com/[^/]+/[^/]+\K/tree/.*], "").chomp(".git") + if $F[3] + if $F[3].include?($F[1]) + $F[3][$F[1]] = gem.version.to_s + elsif Gem::Version.new($F[1]) != gem.version and /\A\h+\z/ =~ $F[3] + $F[3..-1] = [] + end + end + $_ = [gem.name, gem.version, uri, *$F[3..-1]].join(" ") +end diff --git a/tool/update-deps b/tool/update-deps new file mode 100755 index 0000000000..2348b36e33 --- /dev/null +++ b/tool/update-deps @@ -0,0 +1,650 @@ +#!/usr/bin/ruby + +# tool/update-deps verify makefile dependencies. + +# Requirements: +# gcc 4.5 (for -save-temps=obj option) +# GNU make (for -p option) +# +# Warning: ccache (and similar tools) must be disabled for +# -save-temps=obj to work properly. +# +# Usage: +# 1. Compile ruby with -save-temps=obj option. +# Ex. ./configure debugflags='-save-temps=obj -g' && make all golf +# 2. Run tool/update-deps to show dependency problems. +# Ex. ./ruby tool/update-deps +# 3. Use --fix to fix makefiles. +# Ex. ./ruby tool/update-deps --fix +# +# Other usages: +# * Fix makefiles using previously detected dependency problems +# Ex. ruby tool/update-deps --actual-fix [file] +# "ruby tool/update-deps --fix" is the same as "ruby tool/update-deps | ruby tool/update-deps --actual-fix". + +require 'optparse' +require 'stringio' +require 'pathname' +require 'open3' +require 'pp' + +# When out-of-place build, files may be built in source directory or +# build directory. +# Some files are always built in the source directory. +# Some files are always built in the build directory. +# Some files are built in the source directory for tarball but build directory for repository (svn). + +=begin +How to build test directories. + +VER=2.2.0 +REV=48577 +tar xf ruby-$VER-r$REV.tar.xz +cp -a ruby-$VER-r$REV tarball_source_dir_original +mv ruby-$VER-r$REV tarball_source_dir_after_build +svn co -q -r$REV https://svn.ruby-lang.org/repos/ruby/trunk ruby +(cd ruby; autoconf) +cp -a ruby repo_source_dir_original +mv ruby repo_source_dir_after_build +mkdir tarball_build_dir repo_build_dir tarball_install_dir repo_install_dir +(cd tarball_build_dir; ../tarball_source_dir_after_build/configure --prefix=$(cd ../tarball_install_dir; pwd) && make all golf install) > tarball.log 2>&1 +(cd repo_build_dir; ../repo_source_dir_after_build/configure --prefix=$(cd ../repo_install_dir; pwd) && make all golf install) > repo.log 2>&1 +ruby -rpp -rfind -e ' +ds = %w[ + repo_source_dir_original + repo_source_dir_after_build + repo_build_dir + tarball_source_dir_original + tarball_source_dir_after_build + tarball_build_dir +] +files = {} +ds.each {|d| + files[d] = {} + Dir.chdir(d) { Find.find(".") {|f| files[d][f] = true if %r{\.(c|h|inc|dmyh)\z} =~ f } } +} +result = {} +files_union = files.values.map {|h| h.keys }.flatten.uniq.sort +files_union.each {|f| + k = files.map {|d,h| h[f] ? d : nil }.compact.sort + next if k == %w[repo_source_dir_after_build repo_source_dir_original tarball_source_dir_after_build tarball_source_dir_original] + next if k == %w[repo_build_dir tarball_build_dir] && File.basename(f) == "extconf.h" + result[k] ||= [] + result[k] << f +} +result.each {|k,v| + k.each {|d| + puts d + } + v.each {|f| + puts " " + f.sub(%r{\A\./}, "") + } + puts +} +' | tee compare.log +=end + +# Files built in the source directory. +# They can be referenced as $(top_srcdir)/filename. +# % ruby -e 'def g(d) Dir.chdir(d) { Dir["**/*.{c,h,inc,dmyh}"] } end; puts((g("repo_source_dir_after_build") - g("repo_source_dir_original")).sort)' +FILES_IN_SOURCE_DIRECTORY = %w[ + revision.h +] + +# Files built in the build directory (except extconf.h). +# They can be referenced as $(topdir)/filename. +# % ruby -e 'def g(d) Dir.chdir(d) { Dir["**/*.{c,h,inc,dmyh}"] } end; puts(g("tarball_build_dir").reject {|f| %r{/extconf.h\z} =~ f }.sort)' +FILES_IN_BUILD_DIRECTORY = %w[ + encdb.h + ext/etc/constdefs.h + ext/socket/constdefs.c + ext/socket/constdefs.h + probes.h + transdb.h + verconf.h +] + +# They are built in the build directory if the source is obtained from the repository. +# However they are pre-built for tarball and they exist in the source directory extracted from the tarball. +# % ruby -e 'def g(d) Dir.chdir(d) { Dir["**/*.{c,h,inc,dmyh}"] } end; puts((g("repo_build_dir") & g("tarball_source_dir_original")).sort)' +FILES_NEED_VPATH = %w[ + ext/rbconfig/sizeof/sizes.c + ext/ripper/eventids1.c + ext/ripper/eventids2table.c + ext/ripper/ripper.c + golf_prelude.c + id.c + id.h + insns.inc + insns_info.inc + known_errors.inc + lex.c + miniprelude.c + mjit_compile.inc + newline.c + node_name.inc + opt_sc.inc + optinsn.inc + optunifs.inc + parse.c + parse.h + probes.dmyh + vm.inc + vmtc.inc + + enc/trans/big5.c + enc/trans/chinese.c + enc/trans/emoji.c + enc/trans/emoji_iso2022_kddi.c + enc/trans/emoji_sjis_docomo.c + enc/trans/emoji_sjis_kddi.c + enc/trans/emoji_sjis_softbank.c + enc/trans/escape.c + enc/trans/gb18030.c + enc/trans/gbk.c + enc/trans/iso2022.c + enc/trans/japanese.c + enc/trans/japanese_euc.c + enc/trans/japanese_sjis.c + enc/trans/korean.c + enc/trans/single_byte.c + enc/trans/utf8_mac.c + enc/trans/utf_16_32.c +] + +# Multiple files with same filename. +# It is not good idea to refer them using VPATH. +# Files in FILES_SAME_NAME_INC is referenced using $(hdrdir). +# Files in FILES_SAME_NAME_TOP is referenced using $(top_srcdir). + +FILES_SAME_NAME_INC = %w[ + include/ruby.h + include/ruby/ruby.h + include/ruby/version.h +] + +FILES_SAME_NAME_TOP = %w[ + version.h +] + +# Other source files exist in the source directory. + +def in_makefile(target, source) + target = target.to_s + source = source.to_s + case target + when %r{\A[^/]*\z}, %r{\Acoroutine/} + target2 = "#{target.sub(/\.o\z/, '.$(OBJEXT)')}" + case source + when *FILES_IN_SOURCE_DIRECTORY then source2 = "$(top_srcdir)/#{source}" + when *FILES_IN_BUILD_DIRECTORY then source2 = "{$(VPATH)}#{source}" # VPATH is not used now but it may changed in future. + when *FILES_NEED_VPATH then source2 = "{$(VPATH)}#{source}" + when *FILES_SAME_NAME_INC then source2 = "$(hdrdir)/#{source.sub(%r{\Ainclude/},'')}" + when *FILES_SAME_NAME_TOP then source2 = "$(top_srcdir)/#{source}" + when 'thread_pthread.c' then source2 = '{$(VPATH)}thread_$(THREAD_MODEL).c' + when 'thread_pthread.h' then source2 = '{$(VPATH)}thread_$(THREAD_MODEL).h' + when %r{\A[^/]*\z} then source2 = "{$(VPATH)}#{File.basename source}" + when %r{\A\.ext/include/[^/]+/ruby/} then source2 = "{$(VPATH)}#{$'}" + when %r{\Ainclude/ruby/} then source2 = "{$(VPATH)}#{$'}" + when %r{\Aenc/} then source2 = "{$(VPATH)}#{$'}" + when %r{\Amissing/} then source2 = "{$(VPATH)}#{$'}" + when %r{\Accan/} then source2 = "$(CCAN_DIR)/#{$'}" + when %r{\Adefs/} then source2 = "{$(VPATH)}#{source}" + when %r{\Acoroutine/} then source2 = "{$(VPATH)}$(COROUTINE_H)" + else source2 = "$(top_srcdir)/#{source}" + end + ["common.mk", target2, source2] + when %r{\Aenc/} + target2 = "#{target.sub(/\.o\z/, '.$(OBJEXT)')}" + case source + when *FILES_IN_SOURCE_DIRECTORY then source2 = "$(top_srcdir)/#{source}" + when *FILES_IN_BUILD_DIRECTORY then source2 = source + when *FILES_NEED_VPATH then source2 = source + when *FILES_SAME_NAME_INC then source2 = "$(hdrdir)/#{source.sub(%r{\Ainclude/},'')}" + when *FILES_SAME_NAME_TOP then source2 = "$(top_srcdir)/#{source}" + when %r{\A\.ext/include/[^/]+/ruby/} then source2 = $' + when %r{\Ainclude/ruby/} then source2 = $' + when %r{\Aenc/unicode/[\d.]+/} then source2 = '$(UNICODE_HDR_DIR)/' + $' + when %r{\Aenc/} then source2 = source + else source2 = "$(top_srcdir)/#{source}" + end + ["enc/depend", target2, source2] + when %r{\Aext/} + targetdir = File.dirname(target) + unless File.exist?("#{targetdir}/extconf.rb") + warn "warning: not found: #{targetdir}/extconf.rb" + end + target2 = File.basename(target) + relpath = Pathname(source).relative_path_from(Pathname(target).dirname).to_s + case source + when *FILES_IN_SOURCE_DIRECTORY then source2 = "$(top_srcdir)/#{source}" + when *FILES_IN_BUILD_DIRECTORY then source2 = relpath + when *FILES_NEED_VPATH then source2 = "{$(VPATH)}#{File.basename source}" + when *FILES_SAME_NAME_INC then source2 = "$(hdrdir)/#{source.sub(%r{\Ainclude/},'')}" + when *FILES_SAME_NAME_TOP then source2 = "$(top_srcdir)/#{source}" + when %r{\A\.ext/include/[^/]+/ruby/} then source2 = "$(arch_hdrdir)/ruby/#{$'}" + when %r{\Ainclude/} then source2 = "$(hdrdir)/#{$'}" + when %r{\A#{Regexp.escape targetdir}/extconf\.h\z} then source2 = "$(RUBY_EXTCONF_H)" + when %r{\A#{Regexp.escape targetdir}/} then source2 = $' + when %r{\A#{Regexp.escape File.dirname(targetdir)}/} then source2 = "$(srcdir)/../#{$'}" + else source2 = "$(top_srcdir)/#{source}" + end + ["#{File.dirname(target)}/depend", target2, source2] + else + raise "unexpected target: #{target}" + end +end + +DEPENDENCIES_SECTION_START_MARK = "\# AUTOGENERATED DEPENDENCIES START\n" +DEPENDENCIES_SECTION_END_MARK = "\# AUTOGENERATED DEPENDENCIES END\n" + +def init_global + ENV['LC_ALL'] = 'C' + if mkflag0 = ENV['GNUMAKEFLAGS'] and (mkflag = mkflag0.sub(/(\A|\s+)-j\d*(?=\s+|\z)/, '')) != mkflag0 + mkflag.strip! + ENV['GNUMAKEFLAGS'] = mkflag + end + + $opt_fix = false + $opt_a = false + $opt_actual_fix = false + $i_not_found = false +end + +def optionparser + op = OptionParser.new + op.banner = 'Usage: ruby tool/update-deps' + op.def_option('-a', 'show valid dependencies') { $opt_a = true } + op.def_option('--fix') { $opt_fix = true } + op.def_option('--actual-fix') { $opt_actual_fix = true } + op +end + +def read_make_deps(cwd) + dependencies = {} + make_p, make_p_stderr, make_p_status = Open3.capture3("make -p all miniruby exe/ruby golf") + File.open('update-deps.make.out.log', 'w') {|f| f.print make_p } + File.open('update-deps.make.err.log', 'w') {|f| f.print make_p_stderr } + if !make_p_status.success? + puts make_p_stderr + raise "make failed" + end + dirstack = [cwd] + curdir = nil + make_p.scan(%r{Entering\ directory\ ['`](.*)'| + ^\#\ (GNU\ Make)\ | + ^CURDIR\ :=\ (.*)| + ^([/0-9a-zA-Z._-]+):(.*)\n((?:\#.*\n)*)| + ^\#\ (Finished\ Make\ data\ base\ on)\ | + Leaving\ directory\ ['`](.*)'}x) { + directory_enter = $1 + data_base_start = $2 + data_base_curdir = $3 + rule_target = $4 + rule_sources = $5 + rule_desc = $6 + data_base_end = $7 + directory_leave = $8 + #p $~ + if directory_enter + enter_dir = Pathname(directory_enter) + #p [:enter, enter_dir] + dirstack.push enter_dir + elsif data_base_start + curdir = nil + elsif data_base_curdir + curdir = Pathname(data_base_curdir) + elsif rule_target && rule_sources && rule_desc && + /Modification time never checked/ !~ rule_desc # This pattern match eliminates rules which VPATH is not expanded. + target = rule_target + deps = rule_sources + deps = deps.scan(%r{[/0-9a-zA-Z._-]+}) + deps.delete_if {|dep| /\.time\z/ =~ dep} # skip timestamp + next if /\.o\z/ !~ target.to_s + next if /\A\./ =~ target.to_s # skip rules such as ".c.o" + #p [curdir, target, deps] + dir = curdir || dirstack.last + dependencies[dir + target] ||= [] + dependencies[dir + target] |= deps.map {|dep| dir + dep } + elsif data_base_end + curdir = nil + elsif directory_leave + leave_dir = Pathname(directory_leave) + #p [:leave, leave_dir] + if leave_dir != dirstack.last + warn "unexpected leave_dir : #{dirstack.last.inspect} != #{leave_dir.inspect}" + end + dirstack.pop + end + } + dependencies +end + +#def guess_compiler_wd(filename, hint0) +# hint = hint0 +# begin +# guess = hint + filename +# if guess.file? +# return hint +# end +# hint = hint.parent +# end while hint.to_s != '.' +# raise ArgumentError, "can not find #{filename} (hint: #{hint0})" +#end + +def read_single_cc_deps(path_i, cwd) + files = {} + compiler_wd = nil + path_i.each_line {|line| + next if /\A\# \d+ "(.*)"/ !~ line + dep = $1 + next if %r{\A<.*>\z} =~ dep # omit <command-line>, etc. + next if /\.erb\z/ =~ dep + compiler_wd ||= dep + files[dep] = true + } + # gcc emits {# 1 "/absolute/directory/of/the/source/file//"} at 2nd line. + if %r{\A/.*//\z} =~ compiler_wd + files.delete compiler_wd + compiler_wd = Pathname(compiler_wd.sub(%r{//\z}, '')) + elsif !(compiler_wd = yield) + raise "compiler working directory not found: #{path_i}" + end + deps = [] + files.each_key {|dep| + dep = Pathname(dep) + if dep.relative? + dep = compiler_wd + dep + end + if !dep.file? + warn "warning: file not found: #{dep}" + next + end + next if !dep.to_s.start_with?(cwd.to_s) # omit system headers. + deps << dep + } + if deps.include?(cwd + "probes.h") + deps << (cwd + "probes.dmyh") + end + deps +end + +def read_cc_deps(cwd) + deps = {} + Pathname.glob('**/*.o').sort.each {|fn_o| + fn_i = fn_o.sub_ext('.i') + if !fn_i.exist? + next if fn_o.sub_ext('.S').exist? + warn "warning: not found: #{fn_i}" + $i_not_found = true + next + end + path_o = cwd + fn_o + path_i = cwd + fn_i + deps[path_o] = read_single_cc_deps(path_i, cwd) do + if fn_o.to_s.start_with?("enc/") + cwd + else + path_i.parent + end + end + } + deps +end + +def concentrate(dependencies, cwd) + deps = {} + dependencies.keys.sort.each {|target| + sources = dependencies[target] + target = target.relative_path_from(cwd) + sources = sources.map {|s| + rel = s.relative_path_from(cwd) + rel + } + if %r{\A\.\.(/|\z)} =~ target.to_s + warn "warning: out of tree target: #{target}" + next + end + sources = sources.reject {|s| + if %r{\A\.\.(/|\z)} =~ s.to_s + warn "warning: out of tree source: #{s}" + true + elsif %r{/\.time\z} =~ s.to_s + true + else + false + end + } + deps[target] = sources + } + deps +end + +def sort_paths(paths) + paths.sort_by {|t| + ary = t.to_s.split(%r{/}) + ary.map.with_index {|e, i| i == ary.length-1 ? [0, e] : [1, e] } # regular file first, directories last. + } +end + +def show_deps(tag, deps) + targets = sort_paths(deps.keys) + targets.each {|t| + sources = sort_paths(deps[t]) + sources.each {|s| + puts "#{tag} #{t}: #{s}" + } + } +end + +def detect_dependencies(out=$stdout) + cwd = Pathname.pwd + make_deps = read_make_deps(cwd) + #pp make_deps + make_deps = concentrate(make_deps, cwd) + #pp make_deps + cc_deps = read_cc_deps(cwd) + #pp cc_deps + cc_deps = concentrate(cc_deps, cwd) + #pp cc_deps + return make_deps, cc_deps +end + +def compare_deps(make_deps, cc_deps, out=$stdout) + targets = make_deps.keys | cc_deps.keys + + makefiles = {} + + make_lines_hash = {} + make_deps.each {|t, sources| + sources.each {|s| + makefile, t2, s2 = in_makefile(t, s) + makefiles[makefile] = true + make_lines_hash[makefile] ||= Hash.new(false) + make_lines_hash[makefile]["#{t2}: #{s2}"] = true + } + } + + cc_lines_hash = {} + cc_deps.each {|t, sources| + sources.each {|s| + makefile, t2, s2 = in_makefile(t, s) + makefiles[makefile] = true + cc_lines_hash[makefile] ||= Hash.new(false) + cc_lines_hash[makefile]["#{t2}: #{s2}"] = true + } + } + + makefiles.keys.sort.each {|makefile| + cc_lines = cc_lines_hash[makefile] || Hash.new(false) + make_lines = make_lines_hash[makefile] || Hash.new(false) + content = begin + File.read(makefile) + rescue Errno::ENOENT + '' + end + if /^#{Regexp.escape DEPENDENCIES_SECTION_START_MARK} + ((?:.*\n)*) + #{Regexp.escape DEPENDENCIES_SECTION_END_MARK}/x =~ content + pre_post_part = [$`, $'] + current_lines = Hash.new(false) + $1.each_line {|line| current_lines[line.chomp] = true } + (cc_lines.keys | current_lines.keys | make_lines.keys).sort.each {|line| + status = [cc_lines[line], current_lines[line], make_lines[line]] + case status + when [true, true, true] + # no problem + when [true, true, false] + out.puts "warning #{makefile} : #{line} (make doesn't detect written dependency)" + when [true, false, true] + out.puts "add_auto #{makefile} : #{line} (harmless)" # This is automatically updatable. + when [true, false, false] + out.puts "add_auto #{makefile} : #{line} (harmful)" # This is automatically updatable. + when [false, true, true] + out.puts "del_cc #{makefile} : #{line}" # Not automatically updatable because build on other OS may need the dependency. + when [false, true, false] + out.puts "del_cc #{makefile} : #{line} (Curious. make doesn't detect this dependency.)" # Not automatically updatable because build on other OS may need the dependency. + when [false, false, true] + out.puts "del_make #{makefile} : #{line}" # Not automatically updatable because the dependency is written manually. + else + raise "unexpected status: #{status.inspect}" + end + } + else + (cc_lines.keys | make_lines.keys).sort.each {|line| + status = [cc_lines[line], make_lines[line]] + case status + when [true, true] + # no problem + when [true, false] + out.puts "add_manual #{makefile} : #{line}" # Not automatically updatable because makefile has no section to update automatically. + when [false, true] + out.puts "del_manual #{makefile} : #{line}" # Not automatically updatable because makefile has no section to update automatically. + else + raise "unexpected status: #{status.inspect}" + end + } + end + } +end + +def prepare_build + unless File.exist?("Makefile") + if File.exist?("autogen.sh") + system("./autogen.sh") + elsif !File.exist?("configure") + system("autoreconf", "-i", "-s") + end + system("./configure", "-q", "--enable-load-relative", "--prefix=/.", + "--disable-install-doc", "debugflags=-save-temps=obj -g") + end +end + +def main_show(out=$stdout) + prepare_build + make_deps, cc_deps = detect_dependencies(out) + compare_deps(make_deps, cc_deps, out) +end + +def extract_deplines(problems) + adds = {} + others = {} + problems.each_line {|line| + case line + when /\Aadd_auto (\S+) : ((\S+): (\S+))/ + (adds[$1] ||= []) << [line, "#{$2}\n"] + when /\A(?:del_cc|del_make|add_manual|del_manual|warning) (\S+) : / + (others[$1] ||= []) << line + else + raise "unexpected line: #{line.inspect}" + end + } + return adds, others +end + +def main_actual_fix(problems) + adds, others = extract_deplines(problems) + (adds.keys | others.keys).sort.each {|makefile| + content = begin + File.read(makefile) + rescue Errno::ENOENT + nil + end + + if content && + /^#{Regexp.escape DEPENDENCIES_SECTION_START_MARK} + ((?:.*\n)*) + #{Regexp.escape DEPENDENCIES_SECTION_END_MARK}/x =~ content + pre_dep_post = [$`, $1, $'] + else + pre_dep_post = nil + end + + if pre_dep_post && adds[makefile] + pre_lines, dep_lines, post_lines = pre_dep_post + dep_lines = dep_lines.lines.to_a + add_lines = adds[makefile].map(&:last) + new_lines = (dep_lines | add_lines).sort.uniq + new_content = [ + pre_lines, + DEPENDENCIES_SECTION_START_MARK, + *new_lines, + DEPENDENCIES_SECTION_END_MARK, + post_lines + ].join + if content != new_content + puts "modified: #{makefile}" + tmp_makefile = "#{makefile}.new.#{$$}" + File.write(tmp_makefile, new_content) + File.rename tmp_makefile, makefile + (add_lines - dep_lines).each {|line| puts " added #{line}" } + else + puts "not modified: #{makefile}" + end + if others[makefile] + others[makefile].each {|line| puts " #{line}" } + end + else + if pre_dep_post + puts "no additional lines: #{makefile}" + elsif content + puts "no dependencies section: #{makefile}" + else + puts "no makefile: #{makefile}" + end + if adds[makefile] + puts " warning: dependencies section was exist at previous phase." + end + if adds[makefile] + adds[makefile].map(&:first).each {|line| puts " #{line}" } + end + if others[makefile] + others[makefile].each {|line| puts " #{line}" } + end + end + } +end + +def main_fix + problems = StringIO.new + main_show(problems) + main_actual_fix(problems.string) +end + +def run + op = optionparser + op.parse!(ARGV) + if $opt_actual_fix + main_actual_fix(ARGF.read) + elsif $opt_fix + main_fix + else + main_show + end +end + +init_global +run +if $i_not_found + warn "warning: missing *.i files, see help in #$0 and ensure ccache is disabled" +end diff --git a/tool/vtlh.rb b/tool/vtlh.rb new file mode 100644 index 0000000000..2e1faf2ce8 --- /dev/null +++ b/tool/vtlh.rb @@ -0,0 +1,17 @@ +# Convert addresses to line numbers for MiniRuby. + +# ARGF = open('ha') +cd = `pwd`.chomp + '/' +ARGF.each{|line| + if /^0x([a-z0-9]+),/ =~ line + stat = line.split(',') + addr = stat[0].hex + 0x00400000 + retired = stat[2].to_i + ticks = stat[3].to_i + + src = `addr2line -e miniruby.exe #{addr.to_s(16)}`.chomp + src.sub!(cd, '') + puts '%-40s 0x%08x %8d %8d' % [src, addr, retired, ticks] + end +} + diff --git a/tool/ytab.sed b/tool/ytab.sed new file mode 100755 index 0000000000..95a9b3e1eb --- /dev/null +++ b/tool/ytab.sed @@ -0,0 +1,80 @@ +#!/bin/sed -f +# This file is used when generating code for the Ruby parser. +/^int yydebug;/{ +i\ +#ifndef yydebug +a\ +#endif +} +/^extern int yydebug;/{ +i\ +#ifndef yydebug +a\ +#endif +} +/^yydestruct.*yymsg/,/{/{ + /^yydestruct/{ + /,$/N + /[, *]p)/!{ + H + s/^/ruby_parser_&/ + s/)$/, p)/ + /\*/s/p)$/struct parser_params *&/ + } + } + /^#endif/{ + x + /yydestruct/{ + i\ +\ struct parser_params *p; + } + x + } + /^{/{ + x + /yydestruct/{ + i\ +#define yydestruct(m, t, v) ruby_parser_yydestruct(m, t, v, p) + } + x + } +} +/^yy_stack_print /,/{/{ + /^yy_stack_print/{ + /[, *]p)/!{ + H + s/^/ruby_parser_&/ + s/)$/, p)/ + /\*/s/p)$/struct parser_params *&/ + } + } + /^#endif/{ + x + /yy_stack_print/{ + i\ +\ struct parser_params *p; + } + x + } + /^{/{ + x + /yy_stack_print/{ + i\ +#define yy_stack_print(b, t) ruby_parser_yy_stack_print(b, t, p) + } + x + } +} +/^yy_reduce_print/,/^}/{ + s/fprintf *(stderr,/YYFPRINTF (p,/g +} +s/^yysyntax_error (/&struct parser_params *p, / +s/ yysyntax_error (/&p, / +s/\( YYFPRINTF *(\)yyoutput,/\1p,/ +s/\( YYFPRINTF *(\)yyo,/\1p,/ +s/\( YYFPRINTF *(\)stderr,/\1p,/ +s/\( YYDPRINTF *((\)stderr,/\1p,/ +s/^\([ ]*\)\(yyerror[ ]*([ ]*parser,\)/\1parser_\2/ +s!^ *extern char \*getenv();!/* & */! +s/^\(#.*\)".*\.tab\.c"/\1"parse.c"/ +/^\(#.*\)".*\.y"/s:\\\\:/:g |