diff options
Diffstat (limited to 'tool/lib')
56 files changed, 13279 insertions, 0 deletions
diff --git a/tool/lib/-test-/integer.rb b/tool/lib/-test-/integer.rb new file mode 100644 index 0000000000..e60abf03a0 --- /dev/null +++ b/tool/lib/-test-/integer.rb @@ -0,0 +1,14 @@ +require 'test/unit' +require '-test-/integer.so' + +module Test::Unit::Assertions + def assert_fixnum(v, msg=nil) + assert_instance_of(Integer, v, msg) + assert_send([Bug::Integer, :fixnum?, v], msg) + end + + def assert_bignum(v, msg=nil) + assert_instance_of(Integer, v, msg) + assert_send([Bug::Integer, :bignum?, v], msg) + end +end diff --git a/tool/lib/bundled_gem.rb b/tool/lib/bundled_gem.rb new file mode 100644 index 0000000000..895aed4510 --- /dev/null +++ b/tool/lib/bundled_gem.rb @@ -0,0 +1,68 @@ +require 'fileutils' +require 'rubygems' +require 'rubygems/package' + +# This library is used by "make extract-gems" to +# unpack bundled gem files. + +module BundledGem + DEFAULT_GEMS_DEPENDENCIES = [ + "net-protocol", # net-ftp + "time", # net-ftp + "singleton", # prime + "ipaddr", # rinda + "forwardable", # prime, rinda + "ruby2_keywords", # drb + "strscan" # rexml + ] + + module_function + + def unpack(file, *rest) + pkg = Gem::Package.new(file) + prepare_test(pkg.spec, *rest) {|dir| pkg.extract_files(dir)} + puts "Unpacked #{file}" + end + + def copy(path, *rest) + spec = Gem::Specification.load(path) + path = File.dirname(path) + prepare_test(spec, *rest) do |dir| + FileUtils.rm_rf(dir) + files = spec.files.reject {|f| f.start_with?(".git")} + dirs = files.map {|f| File.dirname(f) if f.include?("/")}.uniq + FileUtils.mkdir_p(dirs.map {|d| d ? "#{dir}/#{d}" : dir}.sort_by {|d| d.count("/")}) + files.each do |f| + File.copy_stream(File.join(path, f), File.join(dir, f)) + end + end + puts "Copied #{path}" + end + + def prepare_test(spec, dir = ".") + target = spec.full_name + Gem.ensure_gem_subdirectories(dir) + gem_dir = File.join(dir, "gems", target) + yield gem_dir + spec_dir = spec.extensions.empty? ? "specifications" : File.join("gems", target) + if spec.extensions.empty? + spec.dependencies.reject! {|dep| DEFAULT_GEMS_DEPENDENCIES.include?(dep.name)} + end + File.binwrite(File.join(dir, spec_dir, "#{target}.gemspec"), spec.to_ruby) + unless spec.extensions.empty? + spec.dependencies.clear + File.binwrite(File.join(dir, spec_dir, ".bundled.#{target}.gemspec"), spec.to_ruby) + end + if spec.bindir and spec.executables + bindir = File.join(dir, "bin") + Dir.mkdir(bindir) rescue nil + spec.executables.each do |exe| + File.open(File.join(bindir, exe), "wb", 0o777) {|f| + f.print "#!ruby\n", + %[load File.realpath("../gems/#{target}/#{spec.bindir}/#{exe}", __dir__)\n] + } + end + end + FileUtils.rm_rf(Dir.glob("#{gem_dir}/.git*")) + end +end diff --git a/tool/lib/colorize.rb b/tool/lib/colorize.rb new file mode 100644 index 0000000000..11b878d318 --- /dev/null +++ b/tool/lib/colorize.rb @@ -0,0 +1,55 @@ +# frozen-string-literal: true + +class Colorize + # call-seq: + # Colorize.new(colorize = nil) + # Colorize.new(color: color, colors_file: colors_file) + def initialize(color = nil, opts = ((_, color = color, nil)[0] if Hash === color)) + @colors = @reset = nil + @color = (opts[:color] if opts) + if color or (color == nil && STDOUT.tty?) + if (%w[smso so].any? {|attr| /\A\e\[.*m\z/ =~ IO.popen("tput #{attr}", "r", :err => IO::NULL, &:read)} rescue nil) + @beg = "\e[" + colors = (colors = ENV['TEST_COLORS']) ? Hash[colors.scan(/(\w+)=([^:\n]*)/)] : {} + if opts and colors_file = opts[:colors_file] + begin + File.read(colors_file).scan(/(\w+)=([^:\n]*)/) do |n, c| + colors[n] ||= c + end + rescue Errno::ENOENT + end + end + @colors = colors + @reset = "#{@beg}m" + end + end + self + end + + DEFAULTS = { + "pass"=>"32", "fail"=>"31;1", "skip"=>"33;1", + "black"=>"30", "red"=>"31", "green"=>"32", "yellow"=>"33", + "blue"=>"34", "magenta"=>"35", "cyan"=>"36", "white"=>"37", + "bold"=>"1", "underline"=>"4", "reverse"=>"7", + } + + # colorize.decorate(str, name = color_name) + def decorate(str, name = @color) + if @colors and color = (@colors[name] || DEFAULTS[name]) + "#{@beg}#{color}m#{str}#{@reset}" + else + str + end + end + + DEFAULTS.each_key do |name| + define_method(name) {|str| + decorate(str, name) + } + end +end + +if $0 == __FILE__ + colorize = Colorize.new(ARGV.shift) + ARGV.each {|str| puts colorize.decorate(str)} +end diff --git a/tool/lib/core_assertions.rb b/tool/lib/core_assertions.rb new file mode 100644 index 0000000000..acfaf00cef --- /dev/null +++ b/tool/lib/core_assertions.rb @@ -0,0 +1,809 @@ +# frozen_string_literal: true + +module Test + module Unit + module Assertions + def _assertions= n # :nodoc: + @_assertions = n + end + + def _assertions # :nodoc: + @_assertions ||= 0 + end + + ## + # Returns a proc that will output +msg+ along with the default message. + + def message msg = nil, ending = nil, &default + proc { + ending ||= (ending_pattern = /(?<!\.)\z/; ".") + ending_pattern ||= /(?<!#{Regexp.quote(ending)})\z/ + msg = msg.call if Proc === msg + ary = [msg, (default.call if default)].compact.reject(&:empty?) + ary.map! {|str| str.to_s.sub(ending_pattern, ending) } + begin + ary.join("\n") + rescue Encoding::CompatibilityError + ary.map(&:b).join("\n") + end + } + end + end + + module CoreAssertions + require_relative 'envutil' + require 'pp' + nil.pretty_inspect + + def mu_pp(obj) #:nodoc: + obj.pretty_inspect.chomp + end + + def assert_file + AssertFile + end + + FailDesc = proc do |status, message = "", out = ""| + now = Time.now + proc do + EnvUtil.failure_description(status, now, message, out) + end + end + + def assert_in_out_err(args, test_stdin = "", test_stdout = [], test_stderr = [], message = nil, + success: nil, **opt) + args = Array(args).dup + args.insert((Hash === args[0] ? 1 : 0), '--disable=gems') + stdout, stderr, status = EnvUtil.invoke_ruby(args, test_stdin, true, true, **opt) + desc = FailDesc[status, message, stderr] + if block_given? + raise "test_stdout ignored, use block only or without block" if test_stdout != [] + raise "test_stderr ignored, use block only or without block" if test_stderr != [] + yield(stdout.lines.map {|l| l.chomp }, stderr.lines.map {|l| l.chomp }, status) + else + all_assertions(desc) do |a| + [["stdout", test_stdout, stdout], ["stderr", test_stderr, stderr]].each do |key, exp, act| + a.for(key) do + if exp.is_a?(Regexp) + assert_match(exp, act) + elsif exp.all? {|e| String === e} + assert_equal(exp, act.lines.map {|l| l.chomp }) + else + assert_pattern_list(exp, act) + end + end + end + unless success.nil? + a.for("success?") do + if success + assert_predicate(status, :success?) + else + assert_not_predicate(status, :success?) + end + end + end + end + status + end + end + + if defined?(RubyVM::InstructionSequence) + def syntax_check(code, fname, line) + code = code.dup.force_encoding(Encoding::UTF_8) + RubyVM::InstructionSequence.compile(code, fname, fname, line) + :ok + ensure + raise if SyntaxError === $! + end + else + def syntax_check(code, fname, line) + code = code.b + code.sub!(/\A(?:\xef\xbb\xbf)?(\s*\#.*$)*(\n)?/n) { + "#$&#{"\n" if $1 && !$2}BEGIN{throw tag, :ok}\n" + } + code = code.force_encoding(Encoding::UTF_8) + catch {|tag| eval(code, binding, fname, line - 1)} + end + end + + def assert_no_memory_leak(args, prepare, code, message=nil, limit: 2.0, rss: false, **opt) + # TODO: consider choosing some appropriate limit for MJIT and stop skipping this once it does not randomly fail + pend 'assert_no_memory_leak may consider MJIT memory usage as leak' if defined?(RubyVM::MJIT) && RubyVM::MJIT.enabled? + + require_relative 'memory_status' + raise Test::Unit::PendedError, "unsupported platform" unless defined?(Memory::Status) + + token_dump, token_re = new_test_token + envs = args.shift if Array === args and Hash === args.first + args = [ + "--disable=gems", + "-r", File.expand_path("../memory_status", __FILE__), + *args, + "-v", "-", + ] + if defined? Memory::NO_MEMORY_LEAK_ENVS then + envs ||= {} + newenvs = envs.merge(Memory::NO_MEMORY_LEAK_ENVS) { |_, _, _| break } + envs = newenvs if newenvs + end + args.unshift(envs) if envs + cmd = [ + 'END {STDERR.puts '"#{token_dump}"'"FINAL=#{Memory::Status.new}"}', + prepare, + 'STDERR.puts('"#{token_dump}"'"START=#{$initial_status = Memory::Status.new}")', + '$initial_size = $initial_status.size', + code, + 'GC.start', + ].join("\n") + _, err, status = EnvUtil.invoke_ruby(args, cmd, true, true, **opt) + before = err.sub!(/^#{token_re}START=(\{.*\})\n/, '') && Memory::Status.parse($1) + after = err.sub!(/^#{token_re}FINAL=(\{.*\})\n/, '') && Memory::Status.parse($1) + assert(status.success?, FailDesc[status, message, err]) + ([:size, (rss && :rss)] & after.members).each do |n| + b = before[n] + a = after[n] + next unless a > 0 and b > 0 + assert_operator(a.fdiv(b), :<, limit, message(message) {"#{n}: #{b} => #{a}"}) + end + rescue LoadError + pend + end + + # :call-seq: + # assert_nothing_raised( *args, &block ) + # + #If any exceptions are given as arguments, the assertion will + #fail if one of those exceptions are raised. Otherwise, the test fails + #if any exceptions are raised. + # + #The final argument may be a failure message. + # + # assert_nothing_raised RuntimeError do + # raise Exception #Assertion passes, Exception is not a RuntimeError + # end + # + # assert_nothing_raised do + # raise Exception #Assertion fails + # end + def assert_nothing_raised(*args) + self._assertions += 1 + if Module === args.last + msg = nil + else + msg = args.pop + end + begin + yield + rescue Test::Unit::PendedError, *(Test::Unit::AssertionFailedError if args.empty?) + raise + rescue *(args.empty? ? Exception : args) => e + msg = message(msg) { + "Exception raised:\n<#{mu_pp(e)}>\n""Backtrace:\n" << + Test.filter_backtrace(e.backtrace).map{|frame| " #{frame}"}.join("\n") + } + raise Test::Unit::AssertionFailedError, msg.call, e.backtrace + end + end + + def prepare_syntax_check(code, fname = nil, mesg = nil, verbose: nil) + fname ||= caller_locations(2, 1)[0] + mesg ||= fname.to_s + verbose, $VERBOSE = $VERBOSE, verbose + case + when Array === fname + fname, line = *fname + when defined?(fname.path) && defined?(fname.lineno) + fname, line = fname.path, fname.lineno + else + line = 1 + end + yield(code, fname, line, message(mesg) { + if code.end_with?("\n") + "```\n#{code}```\n" + else + "```\n#{code}\n```\n""no-newline" + end + }) + ensure + $VERBOSE = verbose + end + + def assert_valid_syntax(code, *args, **opt) + prepare_syntax_check(code, *args, **opt) do |src, fname, line, mesg| + yield if defined?(yield) + assert_nothing_raised(SyntaxError, mesg) do + assert_equal(:ok, syntax_check(src, fname, line), mesg) + end + end + end + + def assert_normal_exit(testsrc, message = '', child_env: nil, **opt) + assert_valid_syntax(testsrc, caller_locations(1, 1)[0]) + if child_env + child_env = [child_env] + else + child_env = [] + end + out, _, status = EnvUtil.invoke_ruby(child_env + %W'-W0', testsrc, true, :merge_to_stdout, **opt) + assert !status.signaled?, FailDesc[status, message, out] + end + + def assert_ruby_status(args, test_stdin="", message=nil, **opt) + out, _, status = EnvUtil.invoke_ruby(args, test_stdin, true, :merge_to_stdout, **opt) + desc = FailDesc[status, message, out] + assert(!status.signaled?, desc) + message ||= "ruby exit status is not success:" + assert(status.success?, desc) + end + + ABORT_SIGNALS = Signal.list.values_at(*%w"ILL ABRT BUS SEGV TERM") + + def separated_runner(token, out = nil) + include(*Test::Unit::TestCase.ancestors.select {|c| !c.is_a?(Class) }) + out = out ? IO.new(out, 'w') : STDOUT + at_exit { + out.puts "#{token}<error>", [Marshal.dump($!)].pack('m'), "#{token}</error>", "#{token}assertions=#{self._assertions}" + } + Test::Unit::Runner.class_variable_set(:@@stop_auto_run, true) if defined?(Test::Unit::Runner) + end + + def assert_separately(args, file = nil, line = nil, src, ignore_stderr: nil, **opt) + unless file and line + loc, = caller_locations(1,1) + file ||= loc.path + line ||= loc.lineno + end + capture_stdout = true + unless /mswin|mingw/ =~ RUBY_PLATFORM + capture_stdout = false + opt[:out] = Test::Unit::Runner.output if defined?(Test::Unit::Runner) + res_p, res_c = IO.pipe + opt[:ios] = [res_c] + end + token_dump, token_re = new_test_token + src = <<eom +# -*- coding: #{line += __LINE__; src.encoding}; -*- +BEGIN { + require "test/unit";include Test::Unit::Assertions;require #{__FILE__.dump};include Test::Unit::CoreAssertions + separated_runner #{token_dump}, #{res_c&.fileno || 'nil'} +} +#{line -= __LINE__; src} +eom + args = args.dup + args.insert((Hash === args.first ? 1 : 0), "-w", "--disable=gems", *$:.map {|l| "-I#{l}"}) + args << "--debug" if RUBY_ENGINE == 'jruby' # warning: tracing (e.g. set_trace_func) will not capture all events without --debug flag + stdout, stderr, status = EnvUtil.invoke_ruby(args, src, capture_stdout, true, **opt) + ensure + if res_c + res_c.close + res = res_p.read + res_p.close + else + res = stdout + end + raise if $! + abort = status.coredump? || (status.signaled? && ABORT_SIGNALS.include?(status.termsig)) + assert(!abort, FailDesc[status, nil, stderr]) + self._assertions += res[/^#{token_re}assertions=(\d+)/, 1].to_i + begin + res = Marshal.load(res[/^#{token_re}<error>\n\K.*\n(?=#{token_re}<\/error>$)/m].unpack1("m")) + rescue => marshal_error + ignore_stderr = nil + res = nil + end + if res and !(SystemExit === res) + if bt = res.backtrace + bt.each do |l| + l.sub!(/\A-:(\d+)/){"#{file}:#{line + $1.to_i}"} + end + bt.concat(caller) + else + res.set_backtrace(caller) + end + raise res + end + + # really is it succeed? + unless ignore_stderr + # the body of assert_separately must not output anything to detect error + assert(stderr.empty?, FailDesc[status, "assert_separately failed with error message", stderr]) + end + assert(status.success?, FailDesc[status, "assert_separately failed", stderr]) + raise marshal_error if marshal_error + end + + # Run Ractor-related test without influencing the main test suite + def assert_ractor(src, args: [], require: nil, require_relative: nil, file: nil, line: nil, ignore_stderr: nil, **opt) + return unless defined?(Ractor) + + require = "require #{require.inspect}" if require + if require_relative + dir = File.dirname(caller_locations[0,1][0].absolute_path) + full_path = File.expand_path(require_relative, dir) + require = "#{require}; require #{full_path.inspect}" + end + + assert_separately(args, file, line, <<~RUBY, ignore_stderr: ignore_stderr, **opt) + #{require} + previous_verbose = $VERBOSE + $VERBOSE = nil + Ractor.new {} # trigger initial warning + $VERBOSE = previous_verbose + #{src} + RUBY + end + + # :call-seq: + # assert_throw( tag, failure_message = nil, &block ) + # + #Fails unless the given block throws +tag+, returns the caught + #value otherwise. + # + #An optional failure message may be provided as the final argument. + # + # tag = Object.new + # assert_throw(tag, "#{tag} was not thrown!") do + # throw tag + # end + def assert_throw(tag, msg = nil) + ret = catch(tag) do + begin + yield(tag) + rescue UncaughtThrowError => e + thrown = e.tag + end + msg = message(msg) { + "Expected #{mu_pp(tag)} to have been thrown"\ + "#{%Q[, not #{thrown}] if thrown}" + } + assert(false, msg) + end + assert(true) + ret + end + + # :call-seq: + # assert_raise( *args, &block ) + # + #Tests if the given block raises an exception. Acceptable exception + #types may be given as optional arguments. If the last argument is a + #String, it will be used as the error message. + # + # assert_raise do #Fails, no Exceptions are raised + # end + # + # assert_raise NameError do + # puts x #Raises NameError, so assertion succeeds + # end + def assert_raise(*exp, &b) + case exp.last + when String, Proc + msg = exp.pop + end + + begin + yield + rescue Test::Unit::PendedError => e + return e if exp.include? Test::Unit::PendedError + raise e + rescue Exception => e + expected = exp.any? { |ex| + if ex.instance_of? Module then + e.kind_of? ex + else + e.instance_of? ex + end + } + + assert expected, proc { + flunk(message(msg) {"#{mu_pp(exp)} exception expected, not #{mu_pp(e)}"}) + } + + return e + ensure + unless e + exp = exp.first if exp.size == 1 + + flunk(message(msg) {"#{mu_pp(exp)} expected but nothing was raised"}) + end + end + end + + # :call-seq: + # assert_raise_with_message(exception, expected, msg = nil, &block) + # + #Tests if the given block raises an exception with the expected + #message. + # + # assert_raise_with_message(RuntimeError, "foo") do + # nil #Fails, no Exceptions are raised + # end + # + # assert_raise_with_message(RuntimeError, "foo") do + # raise ArgumentError, "foo" #Fails, different Exception is raised + # end + # + # assert_raise_with_message(RuntimeError, "foo") do + # raise "bar" #Fails, RuntimeError is raised but the message differs + # end + # + # assert_raise_with_message(RuntimeError, "foo") do + # raise "foo" #Raises RuntimeError with the message, so assertion succeeds + # end + def assert_raise_with_message(exception, expected, msg = nil, &block) + case expected + when String + assert = :assert_equal + when Regexp + assert = :assert_match + else + raise TypeError, "Expected #{expected.inspect} to be a kind of String or Regexp, not #{expected.class}" + end + + ex = m = nil + EnvUtil.with_default_internal(expected.encoding) do + ex = assert_raise(exception, msg || proc {"Exception(#{exception}) with message matches to #{expected.inspect}"}) do + yield + end + m = ex.message + end + msg = message(msg, "") {"Expected Exception(#{exception}) was raised, but the message doesn't match"} + + if assert == :assert_equal + assert_equal(expected, m, msg) + else + msg = message(msg) { "Expected #{mu_pp expected} to match #{mu_pp m}" } + assert expected =~ m, msg + block.binding.eval("proc{|_|$~=_}").call($~) + end + ex + end + + TEST_DIR = File.join(__dir__, "test/unit") #:nodoc: + + # :call-seq: + # assert(test, [failure_message]) + # + #Tests if +test+ is true. + # + #+msg+ may be a String or a Proc. If +msg+ is a String, it will be used + #as the failure message. Otherwise, the result of calling +msg+ will be + #used as the message if the assertion fails. + # + #If no +msg+ is given, a default message will be used. + # + # assert(false, "This was expected to be true") + def assert(test, *msgs) + case msg = msgs.first + when String, Proc + when nil + msgs.shift + else + bt = caller.reject { |s| s.start_with?(TEST_DIR) } + raise ArgumentError, "assertion message must be String or Proc, but #{msg.class} was given.", bt + end unless msgs.empty? + super + end + + # :call-seq: + # assert_respond_to( object, method, failure_message = nil ) + # + #Tests if the given Object responds to +method+. + # + #An optional failure message may be provided as the final argument. + # + # assert_respond_to("hello", :reverse) #Succeeds + # assert_respond_to("hello", :does_not_exist) #Fails + def assert_respond_to(obj, (meth, *priv), msg = nil) + unless priv.empty? + msg = message(msg) { + "Expected #{mu_pp(obj)} (#{obj.class}) to respond to ##{meth}#{" privately" if priv[0]}" + } + return assert obj.respond_to?(meth, *priv), msg + end + #get rid of overcounting + if caller_locations(1, 1)[0].path.start_with?(TEST_DIR) + return if obj.respond_to?(meth) + end + super(obj, meth, msg) + end + + # :call-seq: + # assert_not_respond_to( object, method, failure_message = nil ) + # + #Tests if the given Object does not respond to +method+. + # + #An optional failure message may be provided as the final argument. + # + # assert_not_respond_to("hello", :reverse) #Fails + # assert_not_respond_to("hello", :does_not_exist) #Succeeds + def assert_not_respond_to(obj, (meth, *priv), msg = nil) + unless priv.empty? + msg = message(msg) { + "Expected #{mu_pp(obj)} (#{obj.class}) to not respond to ##{meth}#{" privately" if priv[0]}" + } + return assert !obj.respond_to?(meth, *priv), msg + end + #get rid of overcounting + if caller_locations(1, 1)[0].path.start_with?(TEST_DIR) + return unless obj.respond_to?(meth) + end + refute_respond_to(obj, meth, msg) + end + + # pattern_list is an array which contains regexp and :*. + # :* means any sequence. + # + # pattern_list is anchored. + # Use [:*, regexp, :*] for non-anchored match. + def assert_pattern_list(pattern_list, actual, message=nil) + rest = actual + anchored = true + pattern_list.each_with_index {|pattern, i| + if pattern == :* + anchored = false + else + if anchored + match = /\A#{pattern}/.match(rest) + else + match = pattern.match(rest) + end + unless match + msg = message(msg) { + expect_msg = "Expected #{mu_pp pattern}\n" + if /\n[^\n]/ =~ rest + actual_mesg = +"to match\n" + rest.scan(/.*\n+/) { + actual_mesg << ' ' << $&.inspect << "+\n" + } + actual_mesg.sub!(/\+\n\z/, '') + else + actual_mesg = "to match " + mu_pp(rest) + end + actual_mesg << "\nafter #{i} patterns with #{actual.length - rest.length} characters" + expect_msg + actual_mesg + } + assert false, msg + end + rest = match.post_match + anchored = true + end + } + if anchored + assert_equal("", rest) + end + end + + def assert_warning(pat, msg = nil) + result = nil + stderr = EnvUtil.with_default_internal(pat.encoding) { + EnvUtil.verbose_warning { + result = yield + } + } + msg = message(msg) {diff pat, stderr} + assert(pat === stderr, msg) + result + end + + def assert_warn(*args) + assert_warning(*args) {$VERBOSE = false; yield} + end + + def assert_deprecated_warning(mesg = /deprecated/) + assert_warning(mesg) do + Warning[:deprecated] = true + yield + end + end + + def assert_deprecated_warn(mesg = /deprecated/) + assert_warn(mesg) do + Warning[:deprecated] = true + yield + end + end + + class << (AssertFile = Struct.new(:failure_message).new) + include Assertions + include CoreAssertions + def assert_file_predicate(predicate, *args) + if /\Anot_/ =~ predicate + predicate = $' + neg = " not" + end + result = File.__send__(predicate, *args) + result = !result if neg + mesg = "Expected file ".dup << args.shift.inspect + mesg << "#{neg} to be #{predicate}" + mesg << mu_pp(args).sub(/\A\[(.*)\]\z/m, '(\1)') unless args.empty? + mesg << " #{failure_message}" if failure_message + assert(result, mesg) + end + alias method_missing assert_file_predicate + + def for(message) + clone.tap {|a| a.failure_message = message} + end + end + + class AllFailures + attr_reader :failures + + def initialize + @count = 0 + @failures = {} + end + + def for(key) + @count += 1 + yield key + rescue Exception => e + @failures[key] = [@count, e] + end + + def foreach(*keys) + keys.each do |key| + @count += 1 + begin + yield key + rescue Exception => e + @failures[key] = [@count, e] + end + end + end + + def message + i = 0 + total = @count.to_s + fmt = "%#{total.size}d" + @failures.map {|k, (n, v)| + v = v.message + "\n#{i+=1}. [#{fmt%n}/#{total}] Assertion for #{k.inspect}\n#{v.b.gsub(/^/, ' | ').force_encoding(v.encoding)}" + }.join("\n") + end + + def pass? + @failures.empty? + end + end + + # threads should respond to shift method. + # Array can be used. + def assert_join_threads(threads, message = nil) + errs = [] + values = [] + while th = threads.shift + begin + values << th.value + rescue Exception + errs << [th, $!] + th = nil + end + end + values + ensure + if th&.alive? + th.raise(Timeout::Error.new) + th.join rescue errs << [th, $!] + end + if !errs.empty? + msg = "exceptions on #{errs.length} threads:\n" + + errs.map {|t, err| + "#{t.inspect}:\n" + + RUBY_VERSION >= "2.5.0" ? err.full_message(highlight: false, order: :top) : err.message + }.join("\n---\n") + if message + msg = "#{message}\n#{msg}" + end + raise Test::Unit::AssertionFailedError, msg + end + end + + def assert_all?(obj, m = nil, &blk) + failed = [] + obj.each do |*a, &b| + unless blk.call(*a, &b) + failed << (a.size > 1 ? a : a[0]) + end + end + assert(failed.empty?, message(m) {failed.pretty_inspect}) + end + + def assert_all_assertions(msg = nil) + all = AllFailures.new + yield all + ensure + assert(all.pass?, message(msg) {all.message.chomp(".")}) + end + alias all_assertions assert_all_assertions + + def assert_all_assertions_foreach(msg = nil, *keys, &block) + all = AllFailures.new + all.foreach(*keys, &block) + ensure + assert(all.pass?, message(msg) {all.message.chomp(".")}) + end + alias all_assertions_foreach assert_all_assertions_foreach + + %w[ + CLOCK_THREAD_CPUTIME_ID CLOCK_PROCESS_CPUTIME_ID + CLOCK_MONOTONIC + ].find do |c| + if Process.const_defined?(c) + [c.to_sym, Process.const_get(c)].find do |clk| + begin + Process.clock_gettime(clk) + rescue + # Constants may be defined but not implemented, e.g., mingw. + else + PERFORMANCE_CLOCK = clk + end + end + end + end + + # Expect +seq+ to respond to +first+ and +each+ methods, e.g., + # Array, Range, Enumerator::ArithmeticSequence and other + # Enumerable-s, and each elements should be size factors. + # + # :yield: each elements of +seq+. + def assert_linear_performance(seq, rehearsal: nil, pre: ->(n) {n}) + pend "No PERFORMANCE_CLOCK found" unless defined?(PERFORMANCE_CLOCK) + + # Timeout testing generally doesn't work when RJIT compilation happens. + rjit_enabled = defined?(RubyVM::RJIT) && RubyVM::RJIT.enabled? + measure = proc do |arg, message| + st = Process.clock_gettime(PERFORMANCE_CLOCK) + yield(*arg) + t = (Process.clock_gettime(PERFORMANCE_CLOCK) - st) + assert_operator 0, :<=, t, message unless rjit_enabled + t + end + + first = seq.first + *arg = pre.call(first) + times = (0..(rehearsal || (2 * first))).map do + measure[arg, "rehearsal"].nonzero? + end + times.compact! + tmin, tmax = times.minmax + + # safe_factor * tmax * rehearsal_time_variance_factor(equals to 1 when variance is small) + tbase = 10 * tmax * [(tmax / tmin) ** 2 / 4, 1].max + info = "(tmin: #{tmin}, tmax: #{tmax}, tbase: #{tbase})" + + seq.each do |i| + next if i == first + t = tbase * i.fdiv(first) + *arg = pre.call(i) + message = "[#{i}]: in #{t}s #{info}" + Timeout.timeout(t, Timeout::Error, message) do + measure[arg, message] + end + end + end + + def diff(exp, act) + require 'pp' + q = PP.new(+"") + q.guard_inspect_key do + q.group(2, "expected: ") do + q.pp exp + end + q.text q.newline + q.group(2, "actual: ") do + q.pp act + end + q.flush + end + q.output + end + + def new_test_token + token = "\e[7;1m#{$$.to_s}:#{Time.now.strftime('%s.%L')}:#{rand(0x10000).to_s(16)}:\e[m" + return token.dump, Regexp.quote(token) + end + end + end +end diff --git a/tool/lib/envutil.rb b/tool/lib/envutil.rb new file mode 100644 index 0000000000..0391b90c1c --- /dev/null +++ b/tool/lib/envutil.rb @@ -0,0 +1,367 @@ +# -*- coding: us-ascii -*- +# frozen_string_literal: true +require "open3" +require "timeout" +require_relative "find_executable" +begin + require 'rbconfig' +rescue LoadError +end +begin + require "rbconfig/sizeof" +rescue LoadError +end + +module EnvUtil + def rubybin + if ruby = ENV["RUBY"] + return ruby + end + ruby = "ruby" + exeext = RbConfig::CONFIG["EXEEXT"] + rubyexe = (ruby + exeext if exeext and !exeext.empty?) + 3.times do + if File.exist? ruby and File.executable? ruby and !File.directory? ruby + return File.expand_path(ruby) + end + if rubyexe and File.exist? rubyexe and File.executable? rubyexe + return File.expand_path(rubyexe) + end + ruby = File.join("..", ruby) + end + if defined?(RbConfig.ruby) + RbConfig.ruby + else + "ruby" + end + end + module_function :rubybin + + LANG_ENVS = %w"LANG LC_ALL LC_CTYPE" + + DEFAULT_SIGNALS = Signal.list + DEFAULT_SIGNALS.delete("TERM") if /mswin|mingw/ =~ RUBY_PLATFORM + + RUBYLIB = ENV["RUBYLIB"] + + class << self + attr_accessor :timeout_scale + attr_reader :original_internal_encoding, :original_external_encoding, + :original_verbose, :original_warning + + def capture_global_values + @original_internal_encoding = Encoding.default_internal + @original_external_encoding = Encoding.default_external + @original_verbose = $VERBOSE + @original_warning = defined?(Warning.[]) ? %i[deprecated experimental].to_h {|i| [i, Warning[i]]} : nil + end + end + + def apply_timeout_scale(t) + if scale = EnvUtil.timeout_scale + t * scale + else + t + end + end + module_function :apply_timeout_scale + + def timeout(sec, klass = nil, message = nil, &blk) + return yield(sec) if sec == nil or sec.zero? + sec = apply_timeout_scale(sec) + Timeout.timeout(sec, klass, message, &blk) + end + module_function :timeout + + def terminate(pid, signal = :TERM, pgroup = nil, reprieve = 1) + reprieve = apply_timeout_scale(reprieve) if reprieve + + signals = Array(signal).select do |sig| + DEFAULT_SIGNALS[sig.to_s] or + DEFAULT_SIGNALS[Signal.signame(sig)] rescue false + end + signals |= [:ABRT, :KILL] + case pgroup + when 0, true + pgroup = -pid + when nil, false + pgroup = pid + end + + lldb = true if /darwin/ =~ RUBY_PLATFORM + + while signal = signals.shift + + if lldb and [:ABRT, :KILL].include?(signal) + lldb = false + # sudo -n: --non-interactive + # lldb -p: attach + # -o: run command + system(*%W[sudo -n lldb -p #{pid} --batch -o bt\ all -o call\ rb_vmdebug_stack_dump_all_threads() -o quit]) + true + end + + begin + Process.kill signal, pgroup + rescue Errno::EINVAL + next + rescue Errno::ESRCH + break + end + if signals.empty? or !reprieve + Process.wait(pid) + else + begin + Timeout.timeout(reprieve) {Process.wait(pid)} + rescue Timeout::Error + else + break + end + end + end + $? + end + module_function :terminate + + def invoke_ruby(args, stdin_data = "", capture_stdout = false, capture_stderr = false, + encoding: nil, timeout: 10, reprieve: 1, timeout_error: Timeout::Error, + stdout_filter: nil, stderr_filter: nil, ios: nil, + signal: :TERM, + rubybin: EnvUtil.rubybin, precommand: nil, + **opt) + timeout = apply_timeout_scale(timeout) + + in_c, in_p = IO.pipe + out_p, out_c = IO.pipe if capture_stdout + err_p, err_c = IO.pipe if capture_stderr && capture_stderr != :merge_to_stdout + opt[:in] = in_c + opt[:out] = out_c if capture_stdout + opt[:err] = capture_stderr == :merge_to_stdout ? out_c : err_c if capture_stderr + if encoding + out_p.set_encoding(encoding) if out_p + err_p.set_encoding(encoding) if err_p + end + ios.each {|i, o = i|opt[i] = o} if ios + + c = "C" + child_env = {} + LANG_ENVS.each {|lc| child_env[lc] = c} + if Array === args and Hash === args.first + child_env.update(args.shift) + end + if RUBYLIB and lib = child_env["RUBYLIB"] + child_env["RUBYLIB"] = [lib, RUBYLIB].join(File::PATH_SEPARATOR) + end + child_env['ASAN_OPTIONS'] = ENV['ASAN_OPTIONS'] if ENV['ASAN_OPTIONS'] + args = [args] if args.kind_of?(String) + pid = spawn(child_env, *precommand, rubybin, *args, opt) + in_c.close + out_c&.close + out_c = nil + err_c&.close + err_c = nil + if block_given? + return yield in_p, out_p, err_p, pid + else + th_stdout = Thread.new { out_p.read } if capture_stdout + th_stderr = Thread.new { err_p.read } if capture_stderr && capture_stderr != :merge_to_stdout + in_p.write stdin_data.to_str unless stdin_data.empty? + in_p.close + if (!th_stdout || th_stdout.join(timeout)) && (!th_stderr || th_stderr.join(timeout)) + timeout_error = nil + else + status = terminate(pid, signal, opt[:pgroup], reprieve) + terminated = Time.now + end + stdout = th_stdout.value if capture_stdout + stderr = th_stderr.value if capture_stderr && capture_stderr != :merge_to_stdout + out_p.close if capture_stdout + err_p.close if capture_stderr && capture_stderr != :merge_to_stdout + status ||= Process.wait2(pid)[1] + stdout = stdout_filter.call(stdout) if stdout_filter + stderr = stderr_filter.call(stderr) if stderr_filter + if timeout_error + bt = caller_locations + msg = "execution of #{bt.shift.label} expired timeout (#{timeout} sec)" + msg = failure_description(status, terminated, msg, [stdout, stderr].join("\n")) + raise timeout_error, msg, bt.map(&:to_s) + end + return stdout, stderr, status + end + ensure + [th_stdout, th_stderr].each do |th| + th.kill if th + end + [in_c, in_p, out_c, out_p, err_c, err_p].each do |io| + io&.close + end + [th_stdout, th_stderr].each do |th| + th.join if th + end + end + module_function :invoke_ruby + + def verbose_warning + class << (stderr = "".dup) + alias write concat + def flush; end + end + stderr, $stderr = $stderr, stderr + $VERBOSE = true + yield stderr + return $stderr + ensure + stderr, $stderr = $stderr, stderr + $VERBOSE = EnvUtil.original_verbose + EnvUtil.original_warning&.each {|i, v| Warning[i] = v} + end + module_function :verbose_warning + + def default_warning + $VERBOSE = false + yield + ensure + $VERBOSE = EnvUtil.original_verbose + end + module_function :default_warning + + def suppress_warning + $VERBOSE = nil + yield + ensure + $VERBOSE = EnvUtil.original_verbose + end + module_function :suppress_warning + + def under_gc_stress(stress = true) + stress, GC.stress = GC.stress, stress + yield + ensure + GC.stress = stress + end + module_function :under_gc_stress + + def with_default_external(enc) + suppress_warning { Encoding.default_external = enc } + yield + ensure + suppress_warning { Encoding.default_external = EnvUtil.original_external_encoding } + end + module_function :with_default_external + + def with_default_internal(enc) + suppress_warning { Encoding.default_internal = enc } + yield + ensure + suppress_warning { Encoding.default_internal = EnvUtil.original_internal_encoding } + end + module_function :with_default_internal + + def labeled_module(name, &block) + Module.new do + singleton_class.class_eval { + define_method(:to_s) {name} + alias inspect to_s + alias name to_s + } + class_eval(&block) if block + end + end + module_function :labeled_module + + def labeled_class(name, superclass = Object, &block) + Class.new(superclass) do + singleton_class.class_eval { + define_method(:to_s) {name} + alias inspect to_s + alias name to_s + } + class_eval(&block) if block + end + end + module_function :labeled_class + + if /darwin/ =~ RUBY_PLATFORM + DIAGNOSTIC_REPORTS_PATH = File.expand_path("~/Library/Logs/DiagnosticReports") + DIAGNOSTIC_REPORTS_TIMEFORMAT = '%Y-%m-%d-%H%M%S' + @ruby_install_name = RbConfig::CONFIG['RUBY_INSTALL_NAME'] + + def self.diagnostic_reports(signame, pid, now) + return unless %w[ABRT QUIT SEGV ILL TRAP].include?(signame) + cmd = File.basename(rubybin) + cmd = @ruby_install_name if "ruby-runner#{RbConfig::CONFIG["EXEEXT"]}" == cmd + path = DIAGNOSTIC_REPORTS_PATH + timeformat = DIAGNOSTIC_REPORTS_TIMEFORMAT + pat = "#{path}/#{cmd}_#{now.strftime(timeformat)}[-_]*.crash" + first = true + 30.times do + first ? (first = false) : sleep(0.1) + Dir.glob(pat) do |name| + log = File.read(name) rescue next + if /\AProcess:\s+#{cmd} \[#{pid}\]$/ =~ log + File.unlink(name) + File.unlink("#{path}/.#{File.basename(name)}.plist") rescue nil + return log + end + end + end + nil + end + else + def self.diagnostic_reports(signame, pid, now) + end + end + + def self.failure_description(status, now, message = "", out = "") + pid = status.pid + if signo = status.termsig + signame = Signal.signame(signo) + sigdesc = "signal #{signo}" + end + log = diagnostic_reports(signame, pid, now) + if signame + sigdesc = "SIG#{signame} (#{sigdesc})" + end + if status.coredump? + sigdesc = "#{sigdesc} (core dumped)" + end + full_message = ''.dup + message = message.call if Proc === message + if message and !message.empty? + full_message << message << "\n" + end + full_message << "pid #{pid}" + full_message << " exit #{status.exitstatus}" if status.exited? + full_message << " killed by #{sigdesc}" if sigdesc + if out and !out.empty? + full_message << "\n" << out.b.gsub(/^/, '| ') + full_message.sub!(/(?<!\n)\z/, "\n") + end + if log + full_message << "Diagnostic reports:\n" << log.b.gsub(/^/, '| ') + end + full_message + end + + def self.gc_stress_to_class? + unless defined?(@gc_stress_to_class) + _, _, status = invoke_ruby(["-e""exit GC.respond_to?(:add_stress_to_class)"]) + @gc_stress_to_class = status.success? + end + @gc_stress_to_class + end +end + +if defined?(RbConfig) + module RbConfig + @ruby = EnvUtil.rubybin + class << self + undef ruby if method_defined?(:ruby) + attr_reader :ruby + end + dir = File.dirname(ruby) + CONFIG['bindir'] = dir + end +end + +EnvUtil.capture_global_values diff --git a/tool/lib/find_executable.rb b/tool/lib/find_executable.rb new file mode 100644 index 0000000000..89c6fb8f3b --- /dev/null +++ b/tool/lib/find_executable.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true +require "rbconfig" + +module EnvUtil + def find_executable(cmd, *args) + exts = RbConfig::CONFIG["EXECUTABLE_EXTS"].split | [RbConfig::CONFIG["EXEEXT"]] + ENV["PATH"].split(File::PATH_SEPARATOR).each do |path| + next if path.empty? + path = File.join(path, cmd) + exts.each do |ext| + cmdline = [path + ext, *args] + begin + return cmdline if yield(IO.popen(cmdline, "r", err: [:child, :out], &:read)) + rescue + next + end + end + end + nil + end + module_function :find_executable +end diff --git a/tool/lib/gc_checker.rb b/tool/lib/gc_checker.rb new file mode 100644 index 0000000000..719da8cac0 --- /dev/null +++ b/tool/lib/gc_checker.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module GCDisabledChecker + def before_setup + if @__gc_disabled__ = GC.enable # return true if GC is disabled + GC.disable + end + + super + end + + def after_teardown + super + + disabled = GC.enable + GC.disable if @__gc_disabled__ + + if @__gc_disabled__ != disabled + label = { + true => 'disabled', + false => 'enabled', + } + raise "GC was #{label[@__gc_disabled__]}, but is #{label[disabled]} after the test." + end + end +end + +module GCCompactChecker + def after_teardown + super + GC.compact + end +end + +Test::Unit::TestCase.include GCDisabledChecker +Test::Unit::TestCase.include GCCompactChecker if ENV['RUBY_TEST_GC_COMPACT'] diff --git a/tool/lib/iseq_loader_checker.rb b/tool/lib/iseq_loader_checker.rb new file mode 100644 index 0000000000..3f07b3a999 --- /dev/null +++ b/tool/lib/iseq_loader_checker.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +begin + require '-test-/iseq_load/iseq_load' +rescue LoadError +end +require 'tempfile' + +class RubyVM::InstructionSequence + def disasm_if_possible + begin + self.disasm + rescue Encoding::CompatibilityError, EncodingError, SecurityError + nil + end + end + + def self.compare_dump_and_load i1, dumper, loader + dump = dumper.call(i1) + return i1 unless dump + i2 = loader.call(dump) + + # compare disassembled result + d1 = i1.disasm_if_possible + d2 = i2.disasm_if_possible + + if d1 != d2 + STDERR.puts "expected:" + STDERR.puts d1 + STDERR.puts "actual:" + STDERR.puts d2 + + t1 = Tempfile.new("expected"); t1.puts d1; t1.close + t2 = Tempfile.new("actual"); t2.puts d2; t2.close + system("diff -u #{t1.path} #{t2.path}") # use diff if available + exit(1) + end + i2 + end + + opt = ENV['RUBY_ISEQ_DUMP_DEBUG'] + + if opt && caller.any?{|e| /test\/runner\.rb/ =~ e} + puts "RUBY_ISEQ_DUMP_DEBUG = #{opt}" if opt + end + + CHECK_TO_A = 'to_a' == opt + CHECK_TO_BINARY = 'to_binary' == opt + + def self.translate i1 + # check to_a/load_iseq + compare_dump_and_load(i1, + proc{|iseq| + ary = iseq.to_a + ary[9] == :top ? ary : nil + }, + proc{|ary| + RubyVM::InstructionSequence.iseq_load(ary) + }) if CHECK_TO_A && defined?(RubyVM::InstructionSequence.iseq_load) + + # check to_binary + i2_bin = compare_dump_and_load(i1, + proc{|iseq| + begin + iseq.to_binary + rescue RuntimeError # not a toplevel + # STDERR.puts [:failed, $!, iseq].inspect + nil + end + }, + proc{|bin| + iseq = RubyVM::InstructionSequence.load_from_binary(bin) + # STDERR.puts iseq.inspect + iseq + }) if CHECK_TO_BINARY + # return value + i2_bin if CHECK_TO_BINARY + end if CHECK_TO_A || CHECK_TO_BINARY +end + +#require_relative 'x'; exit(1) diff --git a/tool/lib/jisx0208.rb b/tool/lib/jisx0208.rb new file mode 100644 index 0000000000..30185fb81b --- /dev/null +++ b/tool/lib/jisx0208.rb @@ -0,0 +1,86 @@ +# Library used by tools/enc-emoji-citrus-gen.rb + +module JISX0208 + class Char + class << self + def from_sjis(sjis) + unless 0x8140 <= sjis && sjis <= 0xFCFC + raise ArgumentError, "out of the range of JIS X 0208: 0x#{sjis.to_s(16)}" + end + sjis_hi, sjis_lo = sjis >> 8, sjis & 0xFF + sjis_hi = (sjis_hi - ((sjis_hi <= 0x9F) ? 0x80 : 0xC0)) << 1 + if sjis_lo <= 0x9E + sjis_hi -= 1 + sjis_lo -= (sjis_lo <= 0x7E) ? 0x3F : 0x40 + else + sjis_lo -= 0x9E + end + return self.new(sjis_hi, sjis_lo) + end + end + + def initialize(row, cell=nil) + if cell + @code = row_cell_to_code(row, cell) + else + @code = row.to_int + end + end + + def ==(other) + if self.class === other + return Integer(self) == Integer(other) + end + return super(other) + end + + def to_int + return @code + end + + def hi + Integer(self) >> 8 + end + + def lo + Integer(self) & 0xFF + end + + def row + self.hi - 0x20 + end + + def cell + self.lo - 0x20 + end + + def succ + succ_hi, succ_lo = self.hi, self.lo + 1 + if succ_lo > 0x7E + succ_lo = 0x21 + succ_hi += 1 + end + return self.class.new(succ_hi << 8 | succ_lo) + end + + def to_sjis + h, l = self.hi, self.lo + h = (h + 1) / 2 + ((0x21..0x5E).include?(h) ? 0x70 : 0xB0) + l += self.hi.odd? ? 0x1F + ((l >= 0x60) ? 1 : 0) : 0x7E + return h << 8 | l + end + + def inspect + "#<JISX0208::Char:#{self.object_id.to_s(16)} sjis=#{self.to_sjis.to_s(16)} jis=#{self.to_int.to_s(16)}>" + end + + private + + def row_cell_to_code(row, cell) + unless 0 < row && (1..94).include?(cell) + raise ArgumentError, "out of row-cell range: #{row}-#{cell}" + end + return (row + 0x20) << 8 | (cell + 0x20) + end + end +end diff --git a/tool/lib/leakchecker.rb b/tool/lib/leakchecker.rb new file mode 100644 index 0000000000..ed50796940 --- /dev/null +++ b/tool/lib/leakchecker.rb @@ -0,0 +1,314 @@ +# frozen_string_literal: true +class LeakChecker + @@try_lsof = nil # not-tried-yet + + def initialize + @fd_info = find_fds + @@skip = false + @tempfile_info = find_tempfiles + @thread_info = find_threads + @env_info = find_env + @encoding_info = find_encodings + @old_verbose = $VERBOSE + @old_warning_flags = find_warning_flags + end + + def check(test_name) + if /i386-solaris/ =~ RUBY_PLATFORM && /TestGem/ =~ test_name + GC.verify_internal_consistency + end + + leaks = [ + check_fd_leak(test_name), + check_thread_leak(test_name), + check_tempfile_leak(test_name), + check_env(test_name), + check_encodings(test_name), + check_verbose(test_name), + check_warning_flags(test_name), + ] + GC.start if leaks.any? + end + + def check_verbose test_name + puts "#{test_name}: $VERBOSE == #{$VERBOSE}" unless @old_verbose == $VERBOSE + end + + def find_fds + if IO.respond_to?(:console) and (m = IO.method(:console)).arity.nonzero? + m[:close] + end + %w"/proc/self/fd /dev/fd".each do |fd_dir| + if File.directory?(fd_dir) + fds = Dir.open(fd_dir) {|d| + a = d.grep(/\A\d+\z/, &:to_i) + if d.respond_to? :fileno + a -= [d.fileno] + end + a + } + return fds.sort + end + end + [] + end + + def check_fd_leak(test_name) + leaked = false + live1 = @fd_info + live2 = find_fds + fd_closed = live1 - live2 + if !fd_closed.empty? + fd_closed.each {|fd| + puts "Closed file descriptor: #{test_name}: #{fd}" + } + end + fd_leaked = live2 - live1 + if !@@skip && !fd_leaked.empty? + leaked = true + h = {} + ObjectSpace.each_object(IO) {|io| + inspect = io.inspect + begin + autoclose = io.autoclose? + fd = io.fileno + rescue IOError # closed IO object + next + end + (h[fd] ||= []) << [io, autoclose, inspect] + } + fd_leaked.select! {|fd| + str = ''.dup + pos = nil + if h[fd] + str << ' :' + h[fd].map {|io, autoclose, inspect| + if ENV["LEAK_CHECKER_TRACE_OBJECT_ALLOCATION"] + pos = "#{ObjectSpace.allocation_sourcefile(io)}:#{ObjectSpace.allocation_sourceline(io)}" + end + s = ' ' + inspect + s << "(not-autoclose)" if !autoclose + s + }.sort.each {|s| + str << s + } + else + begin + io = IO.for_fd(fd, autoclose: false) + s = io.stat + rescue Errno::EBADF + # something un-stat-able + next + else + next if /darwin/ =~ RUBY_PLATFORM and [0, -1].include?(s.dev) + str << ' ' << s.inspect + ensure + io&.close + end + end + puts "Leaked file descriptor: #{test_name}: #{fd}#{str}" + puts " The IO was created at #{pos}" if pos + true + } + unless fd_leaked.empty? + unless @@try_lsof == false + @@try_lsof |= system(*%W[lsof -a -d #{fd_leaked.minmax.uniq.join("-")} -p #$$], out: Test::Unit::Runner.output) + end + end + h.each {|fd, list| + next if list.length <= 1 + if 1 < list.count {|io, autoclose, inspect| autoclose } + str = list.map {|io, autoclose, inspect| " #{inspect}" + (autoclose ? "(autoclose)" : "") }.sort.join + puts "Multiple autoclose IO objects for a file descriptor in: #{test_name}: #{str}" + end + } + end + @fd_info = live2 + @@skip = false + return leaked + end + + def extend_tempfile_counter + return if defined? LeakChecker::TempfileCounter + m = Module.new { + @count = 0 + class << self + attr_accessor :count + end + + def new(data) + LeakChecker::TempfileCounter.count += 1 + super(data) + end + } + LeakChecker.const_set(:TempfileCounter, m) + + class << Tempfile::Remover + prepend LeakChecker::TempfileCounter + end + end + + def find_tempfiles(prev_count=-1) + return [prev_count, []] unless defined? Tempfile + extend_tempfile_counter + count = TempfileCounter.count + if prev_count == count + [prev_count, []] + else + tempfiles = ObjectSpace.each_object(Tempfile).find_all {|t| + t.instance_variable_defined?(:@tmpfile) and t.path + } + [count, tempfiles] + end + end + + def check_tempfile_leak(test_name) + return false unless defined? Tempfile + count1, initial_tempfiles = @tempfile_info + count2, current_tempfiles = find_tempfiles(count1) + leaked = false + tempfiles_leaked = current_tempfiles - initial_tempfiles + if !tempfiles_leaked.empty? + leaked = true + list = tempfiles_leaked.map {|t| t.inspect }.sort + list.each {|str| + puts "Leaked tempfile: #{test_name}: #{str}" + } + tempfiles_leaked.each {|t| t.close! } + end + @tempfile_info = [count2, initial_tempfiles] + return leaked + end + + def find_threads + Thread.list.find_all {|t| + t != Thread.current && t.alive? + } + end + + def check_thread_leak(test_name) + live1 = @thread_info + live2 = find_threads + thread_finished = live1 - live2 + leaked = false + if !thread_finished.empty? + list = thread_finished.map {|t| t.inspect }.sort + list.each {|str| + puts "Finished thread: #{test_name}: #{str}" + } + end + thread_leaked = live2 - live1 + if !thread_leaked.empty? + leaked = true + list = thread_leaked.map {|t| t.inspect }.sort + list.each {|str| + puts "Leaked thread: #{test_name}: #{str}" + } + end + @thread_info = live2 + return leaked + end + + e = ENV["_Ruby_Env_Ignorecase_"], ENV["_RUBY_ENV_IGNORECASE_"] + begin + ENV["_Ruby_Env_Ignorecase_"] = ENV["_RUBY_ENV_IGNORECASE_"] = nil + ENV["_RUBY_ENV_IGNORECASE_"] = "ENV_CASE_TEST" + ENV_IGNORECASE = ENV["_Ruby_Env_Ignorecase_"] == "ENV_CASE_TEST" + ensure + ENV["_Ruby_Env_Ignorecase_"], ENV["_RUBY_ENV_IGNORECASE_"] = e + end + + if ENV_IGNORECASE + def find_env + ENV.to_h {|k, v| [k.upcase, v]} + end + else + def find_env + ENV.to_h + end + end + + def check_env(test_name) + old_env = @env_info + new_env = find_env + return false if old_env == new_env + (old_env.keys | new_env.keys).sort.each {|k| + if old_env.has_key?(k) + if new_env.has_key?(k) + if old_env[k] != new_env[k] + puts "Environment variable changed: #{test_name} : #{k.inspect} changed : #{old_env[k].inspect} -> #{new_env[k].inspect}" + end + else + puts "Environment variable changed: #{test_name} : #{k.inspect} deleted" + end + else + if new_env.has_key?(k) + puts "Environment variable changed: #{test_name} : #{k.inspect} added" + else + flunk "unreachable" + end + end + } + @env_info = new_env + return true + end + + def find_encodings + { + 'Encoding.default_internal' => Encoding.default_internal, + 'Encoding.default_external' => Encoding.default_external, + 'STDIN.internal_encoding' => STDIN.internal_encoding, + 'STDIN.external_encoding' => STDIN.external_encoding, + 'STDOUT.internal_encoding' => STDOUT.internal_encoding, + 'STDOUT.external_encoding' => STDOUT.external_encoding, + 'STDERR.internal_encoding' => STDERR.internal_encoding, + 'STDERR.external_encoding' => STDERR.external_encoding, + } + end + + def check_encodings(test_name) + old_encoding_info = @encoding_info + @encoding_info = find_encodings + leaked = false + @encoding_info.each do |key, new_encoding| + old_encoding = old_encoding_info[key] + if new_encoding != old_encoding + leaked = true + puts "#{key} changed: #{test_name} : #{old_encoding.inspect} to #{new_encoding.inspect}" + end + end + leaked + end + + WARNING_CATEGORIES = (Warning.respond_to?(:[]) ? %i[deprecated experimental] : []).freeze + + def find_warning_flags + WARNING_CATEGORIES.to_h do |category| + [category, Warning[category]] + end + end + + def check_warning_flags(test_name) + new_warning_flags = find_warning_flags + leaked = false + WARNING_CATEGORIES.each do |category| + if new_warning_flags[category] != @old_warning_flags[category] + leaked = true + puts "Warning[#{category.inspect}] changed: #{test_name} : #{@old_warning_flags[category]} to #{new_warning_flags[category]}" + end + end + return leaked + end + + def puts(*a) + output = Test::Unit::Runner.output + if defined?(output.set_encoding) + output.set_encoding(nil, nil) + end + output.puts(*a) + end + + def self.skip + @@skip = true + end +end diff --git a/tool/lib/memory_status.rb b/tool/lib/memory_status.rb new file mode 100644 index 0000000000..5e9e80a68a --- /dev/null +++ b/tool/lib/memory_status.rb @@ -0,0 +1,151 @@ +# frozen_string_literal: true +begin + require '-test-/memory_status.so' +rescue LoadError +end + +module Memory + keys = [] + + case + when File.exist?(procfile = "/proc/self/status") && (pat = /^Vm(\w+):\s+(\d+)/) =~ (data = File.binread(procfile)) + PROC_FILE = procfile + VM_PAT = pat + def self.read_status + IO.foreach(PROC_FILE, encoding: Encoding::ASCII_8BIT) do |l| + yield($1.downcase.intern, $2.to_i * 1024) if VM_PAT =~ l + end + end + + data.scan(pat) {|k, v| keys << k.downcase.intern} + + when /mswin|mingw/ =~ RUBY_PLATFORM + require 'fiddle/import' + require 'fiddle/types' + + module Win32 + extend Fiddle::Importer + dlload "kernel32.dll", "psapi.dll" + include Fiddle::Win32Types + typealias "SIZE_T", "size_t" + + PROCESS_MEMORY_COUNTERS = struct [ + "DWORD cb", + "DWORD PageFaultCount", + "SIZE_T PeakWorkingSetSize", + "SIZE_T WorkingSetSize", + "SIZE_T QuotaPeakPagedPoolUsage", + "SIZE_T QuotaPagedPoolUsage", + "SIZE_T QuotaPeakNonPagedPoolUsage", + "SIZE_T QuotaNonPagedPoolUsage", + "SIZE_T PagefileUsage", + "SIZE_T PeakPagefileUsage", + ] + + typealias "PPROCESS_MEMORY_COUNTERS", "PROCESS_MEMORY_COUNTERS*" + + extern "HANDLE GetCurrentProcess()", :stdcall + extern "BOOL GetProcessMemoryInfo(HANDLE, PPROCESS_MEMORY_COUNTERS, DWORD)", :stdcall + + module_function + def memory_info + size = PROCESS_MEMORY_COUNTERS.size + data = PROCESS_MEMORY_COUNTERS.malloc + data.cb = size + data if GetProcessMemoryInfo(GetCurrentProcess(), data, size) + end + end + + keys.push(:size, :rss, :peak) + def self.read_status + if info = Win32.memory_info + yield :size, info.PagefileUsage + yield :rss, info.WorkingSetSize + yield :peak, info.PeakWorkingSetSize + end + end + when (require_relative 'find_executable' + pat = /^\s*(\d+)\s+(\d+)$/ + pscmd = EnvUtil.find_executable("ps", "-ovsz=", "-orss=", "-p", $$.to_s) {|out| pat =~ out}) + pscmd.pop + PAT = pat + PSCMD = pscmd + + keys << :size << :rss + def self.read_status + if PAT =~ IO.popen(PSCMD + [$$.to_s], "r", err: [:child, :out], &:read) + yield :size, $1.to_i*1024 + yield :rss, $2.to_i*1024 + end + end + else + def self.read_status + raise NotImplementedError, "unsupported platform" + end + end + + if !keys.empty? + Status = Struct.new(*keys) + end +end unless defined?(Memory::Status) + +if defined?(Memory::Status) + class Memory::Status + def _update + Memory.read_status do |key, val| + self[key] = val + end + self + end unless method_defined?(:_update) + + Header = members.map {|k| k.to_s.upcase.rjust(6)}.join('') + Format = "%6d" + + def initialize + _update + end + + def to_s + status = each_pair.map {|n,v| + "#{n}:#{v}" + } + "{#{status.join(",")}}" + end + + def self.parse(str) + status = allocate + str.scan(/(?:\A\{|\G,)(#{members.join('|')}):(\d+)(?=,|\}\z)/) do + status[$1] = $2.to_i + end + status + end + end + + # On some platforms (e.g. Solaris), libc malloc does not return + # freed memory to OS because of efficiency, and linking with extra + # malloc library is needed to detect memory leaks. + # + case RUBY_PLATFORM + when /solaris2\.(?:9|[1-9][0-9])/i # Solaris 9, 10, 11,... + bits = [nil].pack('p').size == 8 ? 64 : 32 + if ENV['LD_PRELOAD'].to_s.empty? && + ENV["LD_PRELOAD_#{bits}"].to_s.empty? && + (ENV['UMEM_OPTIONS'].to_s.empty? || + ENV['UMEM_OPTIONS'] == 'backend=mmap') then + envs = { + 'LD_PRELOAD' => 'libumem.so', + 'UMEM_OPTIONS' => 'backend=mmap' + } + args = [ + envs, + "--disable=gems", + "-v", "-", + ] + _, err, status = EnvUtil.invoke_ruby(args, "exit(0)", true, true) + if status.exitstatus == 0 && err.to_s.empty? then + Memory::NO_MEMORY_LEAK_ENVS = envs + end + end + end #case RUBY_PLATFORM + +end diff --git a/tool/lib/profile_test_all.rb b/tool/lib/profile_test_all.rb new file mode 100644 index 0000000000..fb434e314d --- /dev/null +++ b/tool/lib/profile_test_all.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true +# +# purpose: +# Profile memory usage of each tests. +# +# usage: +# RUBY_TEST_ALL_PROFILE=[file] make test-all +# +# output: +# [file] specified by RUBY_TEST_ALL_PROFILE +# If [file] is 'true', then it is ./test_all_profile +# +# collected information: +# - ObjectSpace.memsize_of_all +# - GC.stat +# - /proc/meminfo (some fields, if exists) +# - /proc/self/status (some fields, if exists) +# - /proc/self/statm (if exists) +# + +require 'objspace' + +class Test::Unit::TestCase + alias orig_run run + + file = ENV['RUBY_TEST_ALL_PROFILE'] + file = 'test-all-profile-result' if file == 'true' + TEST_ALL_PROFILE_OUT = open(file, 'w') + TEST_ALL_PROFILE_GC_STAT_HASH = {} + TEST_ALL_PROFILE_BANNER = ['name'] + TEST_ALL_PROFILE_PROCS = [] + + def self.add *name, &b + TEST_ALL_PROFILE_BANNER.concat name + TEST_ALL_PROFILE_PROCS << b + end + + add 'failed?' do |result, tc| + result << (tc.passed? ? 0 : 1) + end + + add 'memsize_of_all' do |result, *| + result << ObjectSpace.memsize_of_all + end + + add(*GC.stat.keys) do |result, *| + GC.stat(TEST_ALL_PROFILE_GC_STAT_HASH) + result.concat TEST_ALL_PROFILE_GC_STAT_HASH.values + end + + def self.add_proc_meminfo file, fields + return unless FileTest.exist?(file) + regexp = /(#{fields.join("|")}):\s*(\d+) kB/ + # check = {}; fields.each{|e| check[e] = true} + add(*fields) do |result, *| + text = File.read(file) + text.scan(regexp){ + # check.delete $1 + result << $2 + '' + } + # raise check.inspect unless check.empty? + end + end + + add_proc_meminfo '/proc/meminfo', %w(MemTotal MemFree) + add_proc_meminfo '/proc/self/status', %w(VmPeak VmSize VmHWM VmRSS) + + if FileTest.exist?('/proc/self/statm') + add 'size', 'resident', 'share', 'text', 'lib', 'data', 'dt' do |result, *| + result.concat File.read('/proc/self/statm').split(/\s+/) + end + end + + def memprofile_test_all_result_result + result = ["#{self.class}\##{self.__name__.to_s.gsub(/\s+/, '')}"] + TEST_ALL_PROFILE_PROCS.each{|proc| + proc.call(result, self) + } + result.join("\t") + end + + def run runner + result = orig_run(runner) + TEST_ALL_PROFILE_OUT.puts memprofile_test_all_result_result + TEST_ALL_PROFILE_OUT.flush + result + end + + TEST_ALL_PROFILE_OUT.puts TEST_ALL_PROFILE_BANNER.join("\t") +end diff --git a/tool/lib/test/unit.rb b/tool/lib/test/unit.rb new file mode 100644 index 0000000000..3bb2692b43 --- /dev/null +++ b/tool/lib/test/unit.rb @@ -0,0 +1,1762 @@ +# frozen_string_literal: true + +require_relative '../envutil' +require_relative '../colorize' +require_relative '../leakchecker' +require_relative '../test/unit/testcase' +require 'optparse' + +# See Test::Unit +module Test + + class << self + ## + # Filter object for backtraces. + + attr_accessor :backtrace_filter + end + + class BacktraceFilter # :nodoc: + def filter bt + return ["No backtrace"] unless bt + + new_bt = [] + pattern = %r[/(?:lib\/test/|core_assertions\.rb:)] + + unless $DEBUG then + bt.each do |line| + break if pattern.match?(line) + new_bt << line + end + + new_bt = bt.reject { |line| pattern.match?(line) } if new_bt.empty? + new_bt = bt.dup if new_bt.empty? + else + new_bt = bt.dup + end + + new_bt + end + end + + self.backtrace_filter = BacktraceFilter.new + + def self.filter_backtrace bt # :nodoc: + backtrace_filter.filter bt + end + + ## + # Test::Unit is an implementation of the xUnit testing framework for Ruby. + module Unit + ## + # Assertion base class + + class AssertionFailedError < Exception; end + + ## + # Assertion raised when skipping a test + + class PendedError < AssertionFailedError; end + + module Order + class NoSort + def initialize(seed) + end + + def sort_by_name(list) + list + end + + alias sort_by_string sort_by_name + + def group(list) + list + end + end + + module JITFirst + def group(list) + # JIT first + jit, others = list.partition {|e| /test_jit/ =~ e} + jit + others + end + end + + class Alpha < NoSort + include JITFirst + + def sort_by_name(list) + list.sort_by(&:name) + end + + def sort_by_string(list) + list.sort + end + + end + + # shuffle test suites based on CRC32 of their names + Shuffle = Struct.new(:seed, :salt) do + include JITFirst + + def initialize(seed) + self.class::CRC_TBL ||= (0..255).map {|i| + (0..7).inject(i) {|c,| (c & 1 == 1) ? (0xEDB88320 ^ (c >> 1)) : (c >> 1) } + }.freeze + + salt = [seed].pack("V").unpack1("H*") + super(seed, "\n#{salt}".freeze).freeze + end + + def sort_by_name(list) + list.sort_by {|e| randomize_key(e.name)} + end + + def sort_by_string(list) + list.sort_by {|e| randomize_key(e)} + end + + private + + def crc32(str, crc32 = 0xffffffff) + crc_tbl = self.class::CRC_TBL + str.each_byte do |data| + crc32 = crc_tbl[(crc32 ^ data) & 0xff] ^ (crc32 >> 8) + end + crc32 + end + + def randomize_key(name) + crc32(salt, crc32(name)) ^ 0xffffffff + end + end + + Types = { + random: Shuffle, + alpha: Alpha, + sorted: Alpha, + nosort: NoSort, + } + Types.default_proc = proc {|_, order| + raise "Unknown test_order: #{order.inspect}" + } + end + + module RunCount # :nodoc: all + @@run_count = 0 + + def self.have_run? + @@run_count.nonzero? + end + + def run(*) + @@run_count += 1 + super + end + + def run_once + return if have_run? + return if $! # don't run if there was an exception + yield + end + module_function :run_once + end + + module Options # :nodoc: all + def initialize(*, &block) + @init_hook = block + @options = nil + super(&nil) + end + + def option_parser + @option_parser ||= OptionParser.new + end + + def process_args(args = []) + return @options if @options + orig_args = args.dup + options = {} + opts = option_parser + setup_options(opts, options) + opts.parse!(args) + orig_args -= args + args = @init_hook.call(args, options) if @init_hook + non_options(args, options) + @run_options = orig_args + + order = options[:test_order] + if seed = options[:seed] + order ||= :random + elsif (order ||= :random) == :random + seed = options[:seed] = rand(0x10000) + orig_args.unshift "--seed=#{seed}" + end + Test::Unit::TestCase.test_order = order if order + order = Test::Unit::TestCase.test_order + @order = Test::Unit::Order::Types[order].new(seed) + + @help = "\n" + orig_args.map { |s| + " " + (s =~ /[\s|&<>$()]/ ? s.inspect : s) + }.join("\n") + @options = options + end + + private + def setup_options(opts, options) + opts.separator 'test-unit options:' + + opts.on '-h', '--help', 'Display this help.' do + puts opts + exit + end + + opts.on '-s', '--seed SEED', Integer, "Sets random seed" do |m| + options[:seed] = m.to_i + end + + opts.on '-v', '--verbose', "Verbose. Show progress processing files." do + options[:verbose] = true + self.verbose = options[:verbose] + end + + opts.on '-n', '--name PATTERN', "Filter test method names on pattern: /REGEXP/, !/REGEXP/ or STRING" do |a| + (options[:filter] ||= []) << a + end + + orders = Test::Unit::Order::Types.keys + opts.on "--test-order=#{orders.join('|')}", orders do |a| + options[:test_order] = a + end + end + + def non_options(files, options) + filter = options[:filter] + if filter + pos_pat = /\A\/(.*)\/\z/ + neg_pat = /\A!\/(.*)\/\z/ + negative, positive = filter.partition {|s| neg_pat =~ s} + if positive.empty? + filter = nil + elsif negative.empty? and positive.size == 1 and pos_pat !~ positive[0] + filter = positive[0] + unless /\A[A-Z]\w*(?:::[A-Z]\w*)*#/ =~ filter + filter = /##{Regexp.quote(filter)}\z/ + end + else + filter = Regexp.union(*positive.map! {|s| Regexp.new(s[pos_pat, 1] || "\\A#{Regexp.quote(s)}\\z")}) + end + unless negative.empty? + negative = Regexp.union(*negative.map! {|s| Regexp.new(s[neg_pat, 1])}) + filter = /\A(?=.*#{filter})(?!.*#{negative})/ + end + options[:filter] = filter + end + true + end + end + + module Parallel # :nodoc: all + def process_args(args = []) + return @options if @options + options = super + if @options[:parallel] + @files = args + end + options + end + + def non_options(files, options) + @jobserver = nil + makeflags = ENV.delete("MAKEFLAGS") + if !options[:parallel] and + /(?:\A|\s)--jobserver-(?:auth|fds)=(\d+),(\d+)/ =~ makeflags + begin + r = IO.for_fd($1.to_i(10), "rb", autoclose: false) + w = IO.for_fd($2.to_i(10), "wb", autoclose: false) + rescue + r.close if r + nil + else + r.close_on_exec = true + w.close_on_exec = true + @jobserver = [r, w] + options[:parallel] ||= 1 + end + end + @worker_timeout = EnvUtil.apply_timeout_scale(options[:worker_timeout] || 180) + super + end + + def status(*args) + result = super + raise @interrupt if @interrupt + result + end + + private + def setup_options(opts, options) + super + + opts.separator "parallel test options:" + + options[:retry] = true + + opts.on '-j N', '--jobs N', /\A(t)?(\d+)\z/, "Allow run tests with N jobs at once" do |_, t, a| + options[:testing] = true & t # For testing + options[:parallel] = a.to_i + end + + opts.on '--worker-timeout=N', Integer, "Timeout workers not responding in N seconds" do |a| + options[:worker_timeout] = a + end + + opts.on '--separate', "Restart job process after one testcase has done" do + options[:parallel] ||= 1 + options[:separate] = true + end + + opts.on '--retry', "Retry running testcase when --jobs specified" do + options[:retry] = true + end + + opts.on '--no-retry', "Disable --retry" do + options[:retry] = false + end + + opts.on '--ruby VAL', "Path to ruby which is used at -j option" do |a| + options[:ruby] = a.split(/ /).reject(&:empty?) + end + + opts.on '--timetable-data=FILE', "Path to timetable data" do |a| + options[:timetable_data] = a + end + end + + class Worker + def self.launch(ruby,args=[]) + scale = EnvUtil.timeout_scale + io = IO.popen([*ruby, "-W1", + "#{__dir__}/unit/parallel.rb", + *("--timeout-scale=#{scale}" if scale), + *args], "rb+") + new(io, io.pid, :waiting) + end + + attr_reader :quit_called + attr_accessor :start_time + attr_accessor :response_at + attr_accessor :current + + @@worker_number = 0 + + def initialize(io, pid, status) + @num = (@@worker_number += 1) + @io = io + @pid = pid + @status = status + @file = nil + @real_file = nil + @loadpath = [] + @hooks = {} + @quit_called = false + @response_at = nil + end + + def name + "Worker #{@num}" + end + + def puts(*args) + @io.puts(*args) + end + + def run(task,type) + @file = File.basename(task, ".rb") + @real_file = task + begin + puts "loadpath #{[Marshal.dump($:-@loadpath)].pack("m0")}" + @loadpath = $:.dup + puts "run #{task} #{type}" + @status = :prepare + @start_time = Time.now + @response_at = @start_time + rescue Errno::EPIPE + died + rescue IOError + raise unless /stream closed|closed stream/ =~ $!.message + died + end + end + + def hook(id,&block) + @hooks[id] ||= [] + @hooks[id] << block + self + end + + def read + res = (@status == :quit) ? @io.read : @io.gets + @response_at = Time.now + res && res.chomp + end + + def close + @io.close unless @io.closed? + self + rescue IOError + end + + def quit + return if @io.closed? + @quit_called = true + @io.puts "quit" + rescue Errno::EPIPE => e + warn "#{@pid}:#{@status.to_s.ljust(7)}:#{@file}: #{e.message}" + end + + def kill + Process.kill(:KILL, @pid) + rescue Errno::ESRCH + end + + def died(*additional) + @status = :quit + @io.close + status = $? + if status and status.signaled? + additional[0] ||= SignalException.new(status.termsig) + end + + call_hook(:dead,*additional) + end + + def to_s + if @file and @status != :ready + "#{@pid}=#{@file}" + else + "#{@pid}:#{@status.to_s.ljust(7)}" + end + end + + attr_reader :io, :pid + attr_accessor :status, :file, :real_file, :loadpath + + private + + def call_hook(id,*additional) + @hooks[id] ||= [] + @hooks[id].each{|hook| hook[self,additional] } + self + end + + end + + def flush_job_tokens + if @jobserver + r, w = @jobserver.shift(2) + @jobserver = nil + w << @job_tokens.slice!(0..-1) + r.close + w.close + end + end + + def after_worker_down(worker, e=nil, c=false) + return unless @options[:parallel] + return if @interrupt + flush_job_tokens + warn e if e + real_file = worker.real_file and warn "running file: #{real_file}" + @need_quit = true + warn "" + warn "Some worker was crashed. It seems ruby interpreter's bug" + warn "or, a bug of test/unit/parallel.rb. try again without -j" + warn "option." + warn "" + if File.exist?('core') + require 'fileutils' + require 'time' + Dir.glob('/tmp/test-unit-core.*').each do |f| + if Time.now - File.mtime(f) > 7 * 24 * 60 * 60 # 7 days + warn "Deleting an old core file: #{f}" + FileUtils.rm(f) + end + end + core_path = "/tmp/test-unit-core.#{Time.now.utc.iso8601}" + warn "A core file is found. Saving it at: #{core_path.dump}" + FileUtils.mv('core', core_path) + cmd = ['gdb', RbConfig.ruby, '-c', core_path, '-ex', 'bt', '-batch'] + p cmd # debugging why it's not working + system(*cmd) + end + STDERR.flush + exit c + end + + def after_worker_quit(worker) + return unless @options[:parallel] + return if @interrupt + worker.close + if @jobserver and (token = @job_tokens.slice!(0)) + @jobserver[1] << token + end + @workers.delete(worker) + @dead_workers << worker + @ios = @workers.map(&:io) + end + + def launch_worker + begin + worker = Worker.launch(@options[:ruby], @run_options) + rescue => e + abort "ERROR: Failed to launch job process - #{e.class}: #{e.message}" + end + worker.hook(:dead) do |w,info| + after_worker_quit w + after_worker_down w, *info if !info.empty? && !worker.quit_called + end + @workers << worker + @ios << worker.io + @workers_hash[worker.io] = worker + worker + end + + def delete_worker(worker) + @workers_hash.delete worker.io + @workers.delete worker + @ios.delete worker.io + end + + def quit_workers(&cond) + return if @workers.empty? + closed = [] if cond + @workers.reject! do |worker| + next unless cond&.call(worker) + begin + Timeout.timeout(1) do + worker.quit + end + rescue Errno::EPIPE + rescue Timeout::Error + end + closed&.push worker + begin + Timeout.timeout(0.2) do + worker.close + end + rescue Timeout::Error + worker.kill + retry + end + @ios.delete worker.io + end + + return if (closed ||= @workers).empty? + pids = closed.map(&:pid) + begin + Timeout.timeout(0.2 * closed.size) do + Process.waitall + end + rescue Timeout::Error + if pids + Process.kill(:KILL, *pids) rescue nil + pids = nil + retry + end + end + @workers.clear unless cond + closed + end + + FakeClass = Struct.new(:name) + def fake_class(name) + (@fake_classes ||= {})[name] ||= FakeClass.new(name) + end + + def deal(io, type, result, rep, shutting_down = false) + worker = @workers_hash[io] + cmd = worker.read + cmd.sub!(/\A\.+/, '') if cmd # read may return nil + + case cmd + when '' + # just only dots, ignore + when /^okay$/ + worker.status = :running + when /^ready(!)?$/ + bang = $1 + worker.status = :ready + + unless task = @tasks.shift + worker.quit + return nil + end + if @options[:separate] and not bang + worker.quit + worker = launch_worker + end + worker.run(task, type) + @test_count += 1 + + jobs_status(worker) + when /^start (.+?)$/ + worker.current = Marshal.load($1.unpack1("m")) + when /^done (.+?)$/ + begin + r = Marshal.load($1.unpack1("m")) + rescue + print "unknown object: #{$1.unpack1("m").dump}" + return true + end + result << r[0..1] unless r[0..1] == [nil,nil] + rep << {file: worker.real_file, report: r[2], result: r[3], testcase: r[5]} + $:.push(*r[4]).uniq! + jobs_status(worker) if @options[:job_status] == :replace + + return true + when /^record (.+?)$/ + begin + r = Marshal.load($1.unpack1("m")) + + suite = r.first + key = [worker.name, suite] + if @records[key] + @records[key][1] = worker.start_time = Time.now + else + @records[key] = [worker.start_time, Time.now] + end + rescue => e + print "unknown record: #{e.message} #{$1.unpack1("m").dump}" + return true + end + record(fake_class(r[0]), *r[1..-1]) + when /^p (.+?)$/ + del_jobs_status + print $1.unpack1("m") + jobs_status(worker) if @options[:job_status] == :replace + when /^after (.+?)$/ + @warnings << Marshal.load($1.unpack1("m")) + when /^bye (.+?)$/ + after_worker_down worker, Marshal.load($1.unpack1("m")) + when /^bye$/, nil + if shutting_down || worker.quit_called + after_worker_quit worker + else + after_worker_down worker + end + else + print "unknown command: #{cmd.dump}\n" + end + return false + end + + def _run_parallel suites, type, result + @records = {} + + if @options[:parallel] < 1 + warn "Error: parameter of -j option should be greater than 0." + return + end + + # Require needed thing for parallel running + require 'timeout' + @tasks = @order.group(@order.sort_by_string(@files)) # Array of filenames. + + @need_quit = false + @dead_workers = [] # Array of dead workers. + @warnings = [] + @total_tests = @tasks.size.to_s(10) + rep = [] # FIXME: more good naming + + @workers = [] # Array of workers. + @workers_hash = {} # out-IO => worker + @ios = [] # Array of worker IOs + @job_tokens = String.new(encoding: Encoding::ASCII_8BIT) if @jobserver + begin + [@tasks.size, @options[:parallel]].min.times {launch_worker} + + while true + timeout = [(@workers.filter_map {|w| w.response_at}.min&.-(Time.now) || 0) + @worker_timeout, 1].max + + if !(_io = IO.select(@ios, nil, nil, timeout)) + timeout = Time.now - @worker_timeout + quit_workers {|w| w.response_at < timeout}&.map {|w| + rep << {file: w.real_file, result: nil, testcase: w.current[0], error: w.current} + } + elsif _io.first.any? {|io| + @need_quit or + (deal(io, type, result, rep).nil? and + !@workers.any? {|x| [:running, :prepare].include? x.status}) + } + break + end + break if @tasks.empty? and @workers.empty? + if @jobserver and @job_tokens and !@tasks.empty? and + ((newjobs = [@tasks.size, @options[:parallel]].min) > @workers.size or + !@workers.any? {|x| x.status == :ready}) + t = @jobserver[0].read_nonblock(newjobs, exception: false) + if String === t + @job_tokens << t + t.size.times {launch_worker} + end + end + end + rescue Interrupt => ex + @interrupt = ex + return result + ensure + if file = @options[:timetable_data] + open(file, 'w'){|f| + @records.each{|(worker, suite), (st, ed)| + f.puts '[' + [worker.dump, suite.dump, st.to_f * 1_000, ed.to_f * 1_000].join(", ") + '],' + } + } + end + + if @interrupt + @ios.select!{|x| @workers_hash[x].status == :running } + while !@ios.empty? && (__io = IO.select(@ios,[],[],10)) + __io[0].reject! {|io| deal(io, type, result, rep, true)} + end + end + + quit_workers + flush_job_tokens + + unless @interrupt || !@options[:retry] || @need_quit + parallel = @options[:parallel] + @options[:parallel] = false + suites, rep = rep.partition {|r| + r[:testcase] && r[:file] && + (!r.key?(:report) || r[:report].any? {|e| !e[2].is_a?(Test::Unit::PendedError)}) + } + suites.map {|r| File.realpath(r[:file])}.uniq.each {|file| require file} + del_status_line or puts + error, suites = suites.partition {|r| r[:error]} + unless suites.empty? + puts "\n""Retrying..." + @verbose = options[:verbose] + suites.map! {|r| ::Object.const_get(r[:testcase])} + _run_suites(suites, type) + end + unless error.empty? + puts "\n""Retrying hung up testcases..." + error.map! {|r| ::Object.const_get(r[:testcase])} + verbose = @verbose + job_status = options[:job_status] + options[:verbose] = @verbose = true + options[:job_status] = :normal + result.concat _run_suites(error, type) + options[:verbose] = @verbose = verbose + options[:job_status] = job_status + end + @options[:parallel] = parallel + end + unless @options[:retry] + del_status_line or puts + end + unless rep.empty? + rep.each do |r| + if r[:error] + puke(*r[:error], Timeout::Error) + next + end + r[:report]&.each do |f| + puke(*f) if f + end + end + if @options[:retry] + rep.each do |x| + (e, f, s = x[:result]) or next + @errors += e + @failures += f + @skips += s + end + end + end + unless @warnings.empty? + warn "" + @warnings.uniq! {|w| w[1].message} + @warnings.each do |w| + warn "#{w[0]}: #{w[1].message} (#{w[1].class})" + end + warn "" + end + end + end + + def _run_suites suites, type + _prepare_run(suites, type) + @interrupt = nil + result = [] + GC.start + if @options[:parallel] + _run_parallel suites, type, result + else + suites.each {|suite| + begin + result << _run_suite(suite, type) + rescue Interrupt => e + @interrupt = e + break + end + } + end + del_status_line + result + end + end + + module Skipping # :nodoc: all + def failed(s) + super if !s or @options[:hide_skip] + end + + private + def setup_options(opts, options) + super + + opts.separator "skipping options:" + + options[:hide_skip] = true + + opts.on '-q', '--hide-skip', 'Hide skipped tests' do + options[:hide_skip] = true + end + + opts.on '--show-skip', 'Show skipped tests' do + options[:hide_skip] = false + end + end + + def _run_suites(suites, type) + result = super + report.reject!{|r| r.start_with? "Skipped:" } if @options[:hide_skip] + report.sort_by!{|r| r.start_with?("Skipped:") ? 0 : \ + (r.start_with?("Failure:") ? 1 : 2) } + failed(nil) + result + end + end + + module Statistics + def update_list(list, rec, max) + if i = list.empty? ? 0 : list.bsearch_index {|*a| yield(*a)} + list[i, 0] = [rec] + list[max..-1] = [] if list.size >= max + end + end + + def record(suite, method, assertions, time, error) + if @options.values_at(:longest, :most_asserted).any? + @tops ||= {} + rec = [suite.name, method, assertions, time, error] + if max = @options[:longest] + update_list(@tops[:longest] ||= [], rec, max) {|_,_,_,t,_|t<time} + end + if max = @options[:most_asserted] + update_list(@tops[:most_asserted] ||= [], rec, max) {|_,_,a,_,_|a<assertions} + end + end + # (((@record ||= {})[suite] ||= {})[method]) = [assertions, time, error] + super + end + + def run(*args) + result = super + if @tops ||= nil + @tops.each do |t, list| + if list + puts "#{t.to_s.tr('_', ' ')} tests:" + list.each {|suite, method, assertions, time, error| + printf "%5.2fsec(%d): %s#%s\n", time, assertions, suite, method + } + end + end + end + result + end + + private + def setup_options(opts, options) + super + opts.separator "statistics options:" + opts.on '--longest=N', Integer, 'Show longest N tests' do |n| + options[:longest] = n + end + opts.on '--most-asserted=N', Integer, 'Show most asserted N tests' do |n| + options[:most_asserted] = n + end + end + end + + module StatusLine # :nodoc: all + def terminal_width + unless @terminal_width ||= nil + begin + require 'io/console' + width = $stdout.winsize[1] + rescue LoadError, NoMethodError, Errno::ENOTTY, Errno::EBADF, Errno::EINVAL + width = ENV["COLUMNS"].to_i.nonzero? || 80 + end + width -= 1 if /mswin|mingw/ =~ RUBY_PLATFORM + @terminal_width = width + end + @terminal_width + end + + def del_status_line(flush = true) + @status_line_size ||= 0 + if @options[:job_status] == :replace + $stdout.print "\r"+" "*@status_line_size+"\r" + else + $stdout.puts if @status_line_size > 0 + end + $stdout.flush if flush + @status_line_size = 0 + end + + def add_status(line) + @status_line_size ||= 0 + if @options[:job_status] == :replace + line = line[0...(terminal_width-@status_line_size)] + end + print line + @status_line_size += line.size + end + + def jobs_status(worker) + return if !@options[:job_status] or @verbose + if @options[:job_status] == :replace + status_line = @workers.map(&:to_s).join(" ") + else + status_line = worker.to_s + end + update_status(status_line) or (puts; nil) + end + + def del_jobs_status + return unless @options[:job_status] == :replace && @status_line_size.nonzero? + del_status_line + end + + def output + (@output ||= nil) || super + end + + def _prepare_run(suites, type) + options[:job_status] ||= :replace if @tty && !@verbose + case options[:color] + when :always + color = true + when :auto, nil + color = true if @tty || @options[:job_status] == :replace + else + color = false + end + @colorize = Colorize.new(color, colors_file: File.join(__dir__, "../../colors")) + if color or @options[:job_status] == :replace + @verbose = !options[:parallel] + end + @output = Output.new(self) unless @options[:testing] + filter = options[:filter] + type = "#{type}_methods" + total = if filter + suites.inject(0) {|n, suite| n + suite.send(type).grep(filter).size} + else + suites.inject(0) {|n, suite| n + suite.send(type).size} + end + @test_count = 0 + @total_tests = total.to_s(10) + end + + def new_test(s) + @test_count += 1 + update_status(s) + end + + def update_status(s) + count = @test_count.to_s(10).rjust(@total_tests.size) + del_status_line(false) + add_status(@colorize.pass("[#{count}/#{@total_tests}]")) + add_status(" #{s}") + $stdout.print "\r" if @options[:job_status] == :replace and !@verbose + $stdout.flush + end + + def _print(s); $stdout.print(s); end + def succeed; del_status_line; end + + def failed(s) + return if s and @options[:job_status] != :replace + sep = "\n" + @report_count ||= 0 + report.each do |msg| + if msg.start_with? "Skipped:" + if @options[:hide_skip] + del_status_line + next + end + color = :skip + else + color = :fail + end + first, msg = msg.split(/$/, 2) + first = sprintf("%3d) %s", @report_count += 1, first) + $stdout.print(sep, @colorize.decorate(first, color), msg, "\n") + sep = nil + end + report.clear + end + + def initialize + super + @tty = $stdout.tty? + end + + def run(*args) + result = super + puts "\nruby -v: #{RUBY_DESCRIPTION}" + result + end + + private + def setup_options(opts, options) + super + + opts.separator "status line options:" + + options[:job_status] = nil + + opts.on '--jobs-status [TYPE]', [:normal, :replace, :none], + "Show status of jobs every file; Disabled when --jobs isn't specified." do |type| + options[:job_status] = (type || :normal if type != :none) + end + + opts.on '--color[=WHEN]', + [:always, :never, :auto], + "colorize the output. WHEN defaults to 'always'", "or can be 'never' or 'auto'." do |c| + options[:color] = c || :always + end + + opts.on '--tty[=WHEN]', + [:yes, :no], + "force to output tty control. WHEN defaults to 'yes'", "or can be 'no'." do |c| + @tty = c != :no + end + end + + class Output < Struct.new(:runner) # :nodoc: all + def puts(*a) $stdout.puts(*a) unless a.empty? end + def respond_to_missing?(*a) $stdout.respond_to?(*a) end + def method_missing(*a, &b) $stdout.__send__(*a, &b) end + + def print(s) + case s + when /\A(.*\#.*) = \z/ + runner.new_test($1) + when /\A(.* s) = \z/ + runner.add_status(" = #$1") + when /\A\.+\z/ + runner.succeed + when /\A\.*[EFS][EFS.]*\z/ + runner.failed(s) + else + $stdout.print(s) + end + end + end + end + + module LoadPathOption # :nodoc: all + def non_options(files, options) + begin + require "rbconfig" + rescue LoadError + warn "#{caller(1, 1)[0]}: warning: Parallel running disabled because can't get path to ruby; run specify with --ruby argument" + options[:parallel] = nil + else + options[:ruby] ||= [RbConfig.ruby] + end + + super + end + + def setup_options(parser, options) + super + parser.separator "load path options:" + parser.on '-Idirectory', 'Add library load path' do |dirs| + dirs.split(':').each { |d| $LOAD_PATH.unshift d } + end + end + end + + module GlobOption # :nodoc: all + @@testfile_prefix = "test" + @@testfile_suffix = "test" + + def setup_options(parser, options) + super + parser.separator "globbing options:" + parser.on '-B', '--base-directory DIR', 'Base directory to glob.' do |dir| + raise OptionParser::InvalidArgument, "not a directory: #{dir}" unless File.directory?(dir) + options[:base_directory] = dir + end + parser.on '-x', '--exclude REGEXP', 'Exclude test files on pattern.' do |pattern| + (options[:reject] ||= []) << pattern + end + end + + def complement_test_name f, orig_f + basename = File.basename(f) + + if /\.rb\z/ !~ basename + return File.join(File.dirname(f), basename+'.rb') + elsif /\Atest_/ !~ basename + return File.join(File.dirname(f), 'test_'+basename) + end if f.end_with?(basename) # otherwise basename is dirname/ + + raise ArgumentError, "file not found: #{orig_f}" + end + + def non_options(files, options) + paths = [options.delete(:base_directory), nil].uniq + if reject = options.delete(:reject) + reject_pat = Regexp.union(reject.map {|r| %r"#{r}"}) + end + files.map! {|f| + f = f.tr(File::ALT_SEPARATOR, File::SEPARATOR) if File::ALT_SEPARATOR + orig_f = f + while true + ret = ((paths if /\A\.\.?(?:\z|\/)/ !~ f) || [nil]).any? do |prefix| + if prefix + path = f.empty? ? prefix : "#{prefix}/#{f}" + else + next if f.empty? + path = f + end + if f.end_with?(File::SEPARATOR) or !f.include?(File::SEPARATOR) or File.directory?(path) + match = (Dir["#{path}/**/#{@@testfile_prefix}_*.rb"] + Dir["#{path}/**/*_#{@@testfile_suffix}.rb"]).uniq + else + match = Dir[path] + end + if !match.empty? + if reject + match.reject! {|n| + n = n[(prefix.length+1)..-1] if prefix + reject_pat =~ n + } + end + break match + elsif !reject or reject_pat !~ f and File.exist? path + break path + end + end + if !ret + f = complement_test_name(f, orig_f) + else + break ret + end + end + } + files.flatten! + super(files, options) + end + end + + module GCOption # :nodoc: all + def setup_options(parser, options) + super + parser.separator "GC options:" + parser.on '--[no-]gc-stress', 'Set GC.stress as true' do |flag| + options[:gc_stress] = flag + end + parser.on '--[no-]gc-compact', 'GC.compact every time' do |flag| + options[:gc_compact] = flag + end + end + + def non_options(files, options) + if options.delete(:gc_stress) + Test::Unit::TestCase.class_eval do + oldrun = instance_method(:run) + define_method(:run) do |runner| + begin + gc_stress, GC.stress = GC.stress, true + oldrun.bind_call(self, runner) + ensure + GC.stress = gc_stress + end + end + end + end + if options.delete(:gc_compact) + Test::Unit::TestCase.class_eval do + oldrun = instance_method(:run) + define_method(:run) do |runner| + begin + oldrun.bind_call(self, runner) + ensure + GC.compact + end + end + end + end + super + end + end + + module RequireFiles # :nodoc: all + def non_options(files, options) + return false if !super + errors = {} + result = false + files.each {|f| + d = File.dirname(path = File.realpath(f)) + unless $:.include? d + $: << d + end + begin + require path unless options[:parallel] + result = true + rescue LoadError + next if errors[$!.message] + errors[$!.message] = true + puts "#{f}: #{$!}" + end + } + result + end + end + + module RepeatOption # :nodoc: all + def setup_options(parser, options) + super + options[:repeat_count] = nil + parser.separator "repeat options:" + parser.on '--repeat-count=NUM', "Number of times to repeat", Integer do |n| + options[:repeat_count] = n + end + end + + def _run_anything(type) + @repeat_count = @options[:repeat_count] + super + end + end + + module ExcludesOption # :nodoc: all + class ExcludedMethods < Struct.new(:excludes) + def exclude(name, reason) + excludes[name] = reason + end + + def exclude_from(klass) + excludes = self.excludes + pattern = excludes.keys.grep(Regexp).tap {|k| + break (Regexp.new(k.join('|')) unless k.empty?) + } + klass.class_eval do + public_instance_methods(false).each do |method| + if excludes[method] or (pattern and pattern =~ method) + remove_method(method) + end + end + public_instance_methods(true).each do |method| + if excludes[method] or (pattern and pattern =~ method) + undef_method(method) + end + end + end + end + + def self.load(dirs, name) + return unless dirs and name + instance = nil + dirs.each do |dir| + path = File.join(dir, name.gsub(/::/, '/') + ".rb") + begin + src = File.read(path) + rescue Errno::ENOENT + nil + else + instance ||= new({}) + instance.instance_eval(src, path) + end + end + instance + end + end + + def setup_options(parser, options) + super + if excludes = ENV["EXCLUDES"] + excludes = excludes.split(File::PATH_SEPARATOR) + end + options[:excludes] = excludes || [] + parser.separator "excludes options:" + parser.on '-X', '--excludes-dir DIRECTORY', "Directory name of exclude files" do |d| + options[:excludes].concat d.split(File::PATH_SEPARATOR) + end + end + + def _run_suite(suite, type) + if ex = ExcludedMethods.load(@options[:excludes], suite.name) + ex.exclude_from(suite) + end + super + end + end + + module TimeoutOption + def setup_options(parser, options) + super + parser.separator "timeout options:" + parser.on '--timeout-scale NUM', '--subprocess-timeout-scale NUM', "Scale timeout", Float do |scale| + raise OptionParser::InvalidArgument, "timeout scale must be positive" unless scale > 0 + options[:timeout_scale] = scale + end + end + + def non_options(files, options) + if scale = options[:timeout_scale] or + (scale = ENV["RUBY_TEST_TIMEOUT_SCALE"] || ENV["RUBY_TEST_SUBPROCESS_TIMEOUT_SCALE"] and + (scale = scale.to_f) > 0) + EnvUtil.timeout_scale = scale + end + super + end + end + + class Runner # :nodoc: all + + attr_accessor :report, :failures, :errors, :skips # :nodoc: + attr_accessor :assertion_count # :nodoc: + attr_writer :test_count # :nodoc: + attr_accessor :start_time # :nodoc: + attr_accessor :help # :nodoc: + attr_accessor :verbose # :nodoc: + attr_writer :options # :nodoc: + + ## + # :attr: + # + # if true, installs an "INFO" signal handler (only available to BSD and + # OS X users) which prints diagnostic information about the test run. + # + # This is auto-detected by default but may be overridden by custom + # runners. + + attr_accessor :info_signal + + ## + # Lazy accessor for options. + + def options + @options ||= {seed: 42} + end + + @@installed_at_exit ||= false + @@out = $stdout + @@after_tests = [] + @@current_repeat_count = 0 + + ## + # A simple hook allowing you to run a block of code after _all_ of + # the tests are done. Eg: + # + # Test::Unit::Runner.after_tests { p $debugging_info } + + def self.after_tests &block + @@after_tests << block + end + + ## + # Returns the stream to use for output. + + def self.output + @@out + end + + ## + # Sets Test::Unit::Runner to write output to +stream+. $stdout is the default + # output + + def self.output= stream + @@out = stream + end + + ## + # Tells Test::Unit::Runner to delegate to +runner+, an instance of a + # Test::Unit::Runner subclass, when Test::Unit::Runner#run is called. + + def self.runner= runner + @@runner = runner + end + + ## + # Returns the Test::Unit::Runner subclass instance that will be used + # to run the tests. A Test::Unit::Runner instance is the default + # runner. + + def self.runner + @@runner ||= self.new + end + + ## + # Return all plugins' run methods (methods that start with "run_"). + + def self.plugins + @@plugins ||= (["run_tests"] + + public_instance_methods(false). + grep(/^run_/).map { |s| s.to_s }).uniq + end + + ## + # Return the IO for output. + + def output + self.class.output + end + + def puts *a # :nodoc: + output.puts(*a) + end + + def print *a # :nodoc: + output.print(*a) + end + + def test_count # :nodoc: + @test_count ||= 0 + end + + ## + # Runner for a given +type+ (eg, test vs bench). + + def self.current_repeat_count + @@current_repeat_count + end + + def _run_anything type + suites = Test::Unit::TestCase.send "#{type}_suites" + return if suites.empty? + + suites = @order.sort_by_name(suites) + + puts + puts "# Running #{type}s:" + puts + + @test_count, @assertion_count = 0, 0 + test_count = assertion_count = 0 + sync = output.respond_to? :"sync=" # stupid emacs + old_sync, output.sync = output.sync, true if sync + + @@current_repeat_count = 0 + begin + start = Time.now + + results = _run_suites suites, type + + @test_count = results.inject(0) { |sum, (tc, _)| sum + tc } + @assertion_count = results.inject(0) { |sum, (_, ac)| sum + ac } + test_count += @test_count + assertion_count += @assertion_count + t = Time.now - start + @@current_repeat_count += 1 + unless @repeat_count + puts + puts + end + puts "Finished%s %ss in %.6fs, %.4f tests/s, %.4f assertions/s.\n" % + [(@repeat_count ? "(#{@@current_repeat_count}/#{@repeat_count}) " : ""), type, + t, @test_count.fdiv(t), @assertion_count.fdiv(t)] + end while @repeat_count && @@current_repeat_count < @repeat_count && + report.empty? && failures.zero? && errors.zero? + + output.sync = old_sync if sync + + report.each_with_index do |msg, i| + puts "\n%3d) %s" % [i + 1, msg] + end + + puts + @test_count = test_count + @assertion_count = assertion_count + + status + end + + ## + # Run a single +suite+ for a given +type+. + + def _run_suite suite, type + header = "#{type}_suite_header" + puts send(header, suite) if respond_to? header + + filter = options[:filter] + + all_test_methods = suite.send "#{type}_methods" + if filter + all_test_methods.select! {|method| + filter === "#{suite}##{method}" + } + end + all_test_methods = @order.sort_by_name(all_test_methods) + + leakchecker = LeakChecker.new + if ENV["LEAK_CHECKER_TRACE_OBJECT_ALLOCATION"] + require "objspace" + trace = true + end + + assertions = all_test_methods.map { |method| + + inst = suite.new method + _start_method(inst) + inst._assertions = 0 + + print "#{suite}##{method} = " if @verbose + + start_time = Time.now if @verbose + result = + if trace + ObjectSpace.trace_object_allocations {inst.run self} + else + inst.run self + end + + print "%.2f s = " % (Time.now - start_time) if @verbose + print result + puts if @verbose + $stdout.flush + + unless defined?(RubyVM::MJIT) && RubyVM::MJIT.enabled? # compiler process is wrongly considered as leak + leakchecker.check("#{inst.class}\##{inst.__name__}") + end + + _end_method(inst) + + inst._assertions + } + return assertions.size, assertions.inject(0) { |sum, n| sum + n } + end + + def _start_method(inst) + end + def _end_method(inst) + end + + ## + # Record the result of a single test. Makes it very easy to gather + # information. Eg: + # + # class StatisticsRecorder < Test::Unit::Runner + # def record suite, method, assertions, time, error + # # ... record the results somewhere ... + # end + # end + # + # Test::Unit::Runner.runner = StatisticsRecorder.new + # + # NOTE: record might be sent more than once per test. It will be + # sent once with the results from the test itself. If there is a + # failure or error in teardown, it will be sent again with the + # error or failure. + + def record suite, method, assertions, time, error + end + + def location e # :nodoc: + last_before_assertion = "" + + return '<empty>' unless e.backtrace # SystemStackError can return nil. + + e.backtrace.reverse_each do |s| + break if s =~ /in .(assert|refute|flunk|pass|fail|raise|must|wont)/ + last_before_assertion = s + end + last_before_assertion.sub(/:in .*$/, '') + end + + ## + # Writes status for failed test +meth+ in +klass+ which finished with + # exception +e+ + + def initialize # :nodoc: + @report = [] + @errors = @failures = @skips = 0 + @verbose = false + @mutex = Thread::Mutex.new + @info_signal = Signal.list['INFO'] + @repeat_count = nil + end + + def synchronize # :nodoc: + if @mutex then + @mutex.synchronize { yield } + else + yield + end + end + + def inspect + "#<#{self.class.name}: " << + instance_variables.filter_map do |var| + next if var == :@option_parser # too big + "#{var}=#{instance_variable_get(var).inspect}" + end.join(", ") << ">" + end + + ## + # Top level driver, controls all output and filtering. + + def _run args = [] + args = process_args args # ARGH!! blame test/unit process_args + self.options.merge! args + + puts "Run options: #{help}" + + self.class.plugins.each do |plugin| + send plugin + break unless report.empty? + end + + return failures + errors if self.test_count > 0 # or return nil... + rescue Interrupt + abort 'Interrupted' + end + + ## + # Runs test suites matching +filter+. + + def run_tests + _run_anything :test + end + + ## + # Writes status to +io+ + + def status io = self.output + format = "%d tests, %d assertions, %d failures, %d errors, %d skips" + io.puts format % [test_count, assertion_count, failures, errors, skips] + end + + prepend Test::Unit::Options + prepend Test::Unit::StatusLine + prepend Test::Unit::Parallel + prepend Test::Unit::Statistics + prepend Test::Unit::Skipping + prepend Test::Unit::GlobOption + prepend Test::Unit::RepeatOption + prepend Test::Unit::LoadPathOption + prepend Test::Unit::GCOption + prepend Test::Unit::ExcludesOption + prepend Test::Unit::TimeoutOption + prepend Test::Unit::RunCount + + ## + # Begins the full test run. Delegates to +runner+'s #_run method. + + def run(argv = []) + self.class.runner._run(argv) + rescue NoMemoryError + system("cat /proc/meminfo") if File.exist?("/proc/meminfo") + system("ps x -opid,args,%cpu,%mem,nlwp,rss,vsz,wchan,stat,start,time,etime,blocked,caught,ignored,pending,f") if File.exist?("/bin/ps") + raise + end + + @@stop_auto_run = false + def self.autorun + at_exit { + Test::Unit::RunCount.run_once { + exit(Test::Unit::Runner.new.run(ARGV) || true) + } unless @@stop_auto_run + } unless @@installed_at_exit + @@installed_at_exit = true + end + + alias orig_run_suite _run_suite + + # Overriding of Test::Unit::Runner#puke + def puke klass, meth, e + n = report.size + e = case e + when Test::Unit::PendedError then + @skips += 1 + return "S" unless @verbose + "Skipped:\n#{klass}##{meth} [#{location e}]:\n#{e.message}\n" + when Test::Unit::AssertionFailedError then + @failures += 1 + "Failure:\n#{klass}##{meth} [#{location e}]:\n#{e.message}\n" + else + @errors += 1 + bt = Test::filter_backtrace(e.backtrace).join "\n " + "Error:\n#{klass}##{meth}:\n#{e.class}: #{e.message.b}\n #{bt}\n" + end + @report << e + rep = e[0, 1] + if Test::Unit::PendedError === e and /no message given\z/ =~ e.message + report.slice!(n..-1) + rep = "." + end + rep + end + end + + class AutoRunner # :nodoc: all + class Runner < Test::Unit::Runner + include Test::Unit::RequireFiles + end + + attr_accessor :to_run, :options + + def initialize(force_standalone = false, default_dir = nil, argv = ARGV) + @force_standalone = force_standalone + @runner = Runner.new do |files, options| + base = options[:base_directory] ||= default_dir + files << default_dir if files.empty? and default_dir + @to_run = files + yield self if block_given? + $LOAD_PATH.unshift base if base + files + end + Runner.runner = @runner + @options = @runner.option_parser + if @force_standalone + @options.banner.sub!(/\[options\]/, '\& tests...') + end + @argv = argv + end + + def process_args(*args) + @runner.process_args(*args) + !@to_run.empty? + end + + def run + if @force_standalone and not process_args(@argv) + abort @options.banner + end + @runner.run(@argv) || true + end + + def self.run(*args) + new(*args).run + end + end + + class ProxyError < StandardError # :nodoc: all + def initialize(ex) + @message = ex.message + @backtrace = ex.backtrace + end + + attr_accessor :message, :backtrace + end + end +end + +Test::Unit::Runner.autorun diff --git a/tool/lib/test/unit/assertions.rb b/tool/lib/test/unit/assertions.rb new file mode 100644 index 0000000000..dcf3e6fcb9 --- /dev/null +++ b/tool/lib/test/unit/assertions.rb @@ -0,0 +1,839 @@ +# frozen_string_literal: true +require 'pp' + +module Test + module Unit + module Assertions + + ## + # Returns the diff command to use in #diff. Tries to intelligently + # figure out what diff to use. + + def self.diff + unless defined? @diff + exe = RbConfig::CONFIG['EXEEXT'] + @diff = %W"gdiff#{exe} diff#{exe}".find do |diff| + if system(diff, "-u", __FILE__, __FILE__) + break "#{diff} -u" + end + end + end + + @diff + end + + ## + # Set the diff command to use in #diff. + + def self.diff= o + @diff = o + end + + ## + # Returns a diff between +exp+ and +act+. If there is no known + # diff command or if it doesn't make sense to diff the output + # (single line, short output), then it simply returns a basic + # comparison between the two. + + def diff exp, act + require "tempfile" + + expect = mu_pp_for_diff exp + butwas = mu_pp_for_diff act + result = nil + + need_to_diff = + self.class.diff && + (expect.include?("\n") || + butwas.include?("\n") || + expect.size > 30 || + butwas.size > 30 || + expect == butwas) + + return "Expected: #{mu_pp exp}\n Actual: #{mu_pp act}" unless + need_to_diff + + tempfile_a = nil + tempfile_b = nil + + Tempfile.open("expect") do |a| + tempfile_a = a + a.puts expect + a.flush + + Tempfile.open("butwas") do |b| + tempfile_b = b + b.puts butwas + b.flush + + result = `#{self.class.diff} #{a.path} #{b.path}` + result.sub!(/^\-\-\- .+/, "--- expected") + result.sub!(/^\+\+\+ .+/, "+++ actual") + + if result.empty? then + klass = exp.class + result = [ + "No visible difference in the #{klass}#inspect output.\n", + "You should look at the implementation of #== on ", + "#{klass} or its members.\n", + expect, + ].join + end + end + end + + result + ensure + tempfile_a.close! if tempfile_a + tempfile_b.close! if tempfile_b + end + + ## + # This returns a diff-able human-readable version of +obj+. This + # differs from the regular mu_pp because it expands escaped + # newlines and makes hex-values generic (like object_ids). This + # uses mu_pp to do the first pass and then cleans it up. + + def mu_pp_for_diff obj + mu_pp(obj).gsub(/(?<!\\)(?:\\\\)*\K\\n/, "\n").gsub(/:0x[a-fA-F0-9]{4,}/m, ':0xXXXXXX') + end + + ## + # Fails unless +test+ is a true value. + + def assert test, msg = nil + msg ||= "Failed assertion, no message given." + self._assertions += 1 + unless test then + msg = msg.call if Proc === msg + raise Test::Unit::AssertionFailedError, msg + end + true + end + + ## + # Fails unless +obj+ is empty. + + def assert_empty obj, msg = nil + msg = message(msg) { "Expected #{mu_pp(obj)} to be empty" } + assert_respond_to obj, :empty? + assert obj.empty?, msg + end + + ## + # For comparing Floats. Fails unless +exp+ and +act+ are within +delta+ + # of each other. + # + # assert_in_delta Math::PI, (22.0 / 7.0), 0.01 + + def assert_in_delta exp, act, delta = 0.001, msg = nil + n = (exp - act).abs + msg = message(msg) { + "Expected |#{exp} - #{act}| (#{n}) to be <= #{delta}" + } + assert delta >= n, msg + end + + ## + # For comparing Floats. Fails unless +exp+ and +act+ have a relative + # error less than +epsilon+. + + def assert_in_epsilon a, b, epsilon = 0.001, msg = nil + assert_in_delta a, b, [a.abs, b.abs].min * epsilon, msg + end + + ## + # Fails unless +collection+ includes +obj+. + + def assert_includes collection, obj, msg = nil + msg = message(msg) { + "Expected #{mu_pp(collection)} to include #{mu_pp(obj)}" + } + assert_respond_to collection, :include? + assert collection.include?(obj), msg + end + + ## + # Fails unless +obj+ is an instance of +cls+. + + def assert_instance_of cls, obj, msg = nil + msg = message(msg) { + "Expected #{mu_pp(obj)} to be an instance of #{cls}, not #{obj.class}" + } + + assert obj.instance_of?(cls), msg + end + + ## + # Fails unless +obj+ is a kind of +cls+. + + def assert_kind_of cls, obj, msg = nil # TODO: merge with instance_of + msg = message(msg) { + "Expected #{mu_pp(obj)} to be a kind of #{cls}, not #{obj.class}" } + + assert obj.kind_of?(cls), msg + end + + ## + # Fails unless +matcher+ <tt>=~</tt> +obj+. + + def assert_match matcher, obj, msg = nil + msg = message(msg) { "Expected #{mu_pp matcher} to match #{mu_pp obj}" } + assert_respond_to matcher, :"=~" + matcher = Regexp.new Regexp.escape matcher if String === matcher + assert matcher =~ obj, msg + end + + ## + # Fails unless +obj+ is nil + + def assert_nil obj, msg = nil + msg = message(msg) { "Expected #{mu_pp(obj)} to be nil" } + assert obj.nil?, msg + end + + ## + # Fails unless +obj+ is true + + def assert_true obj, msg = nil + msg = message(msg) { "Expected #{mu_pp(obj)} to be true" } + assert obj == true, msg + end + + ## + # Fails unless +obj+ is false + + def assert_false obj, msg = nil + msg = message(msg) { "Expected #{mu_pp(obj)} to be false" } + assert obj == false, msg + end + + ## + # For testing with binary operators. + # + # assert_operator 5, :<=, 4 + + def assert_operator o1, op, o2 = (predicate = true; nil), msg = nil + return assert_predicate o1, op, msg if predicate + msg = message(msg) { "Expected #{mu_pp(o1)} to be #{op} #{mu_pp(o2)}" } + assert o1.__send__(op, o2), msg + end + + ## + # Fails if stdout or stderr do not output the expected results. + # Pass in nil if you don't care about that streams output. Pass in + # "" if you require it to be silent. Pass in a regexp if you want + # to pattern match. + # + # NOTE: this uses #capture_io, not #capture_subprocess_io. + # + # See also: #assert_silent + + def assert_output stdout = nil, stderr = nil + out, err = capture_output do + yield + end + + err_msg = Regexp === stderr ? :assert_match : :assert_equal if stderr + out_msg = Regexp === stdout ? :assert_match : :assert_equal if stdout + + y = send err_msg, stderr, err, "In stderr" if err_msg + x = send out_msg, stdout, out, "In stdout" if out_msg + + (!stdout || x) && (!stderr || y) + end + + ## + # For testing with predicates. + # + # assert_predicate str, :empty? + # + # This is really meant for specs and is front-ended by assert_operator: + # + # str.must_be :empty? + + def assert_predicate o1, op, msg = nil + msg = message(msg) { "Expected #{mu_pp(o1)} to be #{op}" } + assert o1.__send__(op), msg + end + + ## + # Fails unless +obj+ responds to +meth+. + + def assert_respond_to obj, meth, msg = nil + msg = message(msg) { + "Expected #{mu_pp(obj)} (#{obj.class}) to respond to ##{meth}" + } + assert obj.respond_to?(meth), msg + end + + ## + # Fails unless +exp+ and +act+ are #equal? + + def assert_same exp, act, msg = nil + msg = message(msg) { + data = [mu_pp(act), act.object_id, mu_pp(exp), exp.object_id] + "Expected %s (oid=%d) to be the same as %s (oid=%d)" % data + } + assert exp.equal?(act), msg + end + + ## + # Fails if the block outputs anything to stderr or stdout. + # + # See also: #assert_output + + def assert_silent + assert_output "", "" do + yield + end + end + + ## + # Fails unless the block throws +sym+ + + def assert_throws sym, msg = nil + default = "Expected #{mu_pp(sym)} to have been thrown" + caught = true + catch(sym) do + begin + yield + rescue ThreadError => e # wtf?!? 1.8 + threads == suck + default += ", not \:#{e.message[/uncaught throw \`(\w+?)\'/, 1]}" + rescue ArgumentError => e # 1.9 exception + default += ", not #{e.message.split(/ /).last}" + rescue NameError => e # 1.8 exception + default += ", not #{e.name.inspect}" + end + caught = false + end + + assert caught, message(msg) { default } + end + + def assert_path_exists(path, msg = nil) + msg = message(msg) { "Expected path '#{path}' to exist" } + assert File.exist?(path), msg + end + alias assert_path_exist assert_path_exists + alias refute_path_not_exist assert_path_exists + + def refute_path_exists(path, msg = nil) + msg = message(msg) { "Expected path '#{path}' to not exist" } + refute File.exist?(path), msg + end + alias refute_path_exist refute_path_exists + alias assert_path_not_exist refute_path_exists + + ## + # Captures $stdout and $stderr into strings: + # + # out, err = capture_output do + # puts "Some info" + # warn "You did a bad thing" + # end + # + # assert_match %r%info%, out + # assert_match %r%bad%, err + + def capture_output + require 'stringio' + + captured_stdout, captured_stderr = StringIO.new, StringIO.new + + synchronize do + orig_stdout, orig_stderr = $stdout, $stderr + $stdout, $stderr = captured_stdout, captured_stderr + + begin + yield + ensure + $stdout = orig_stdout + $stderr = orig_stderr + end + end + + return captured_stdout.string, captured_stderr.string + end + + def capture_io + raise NoMethodError, "use capture_output" + end + + ## + # Fails with +msg+ + + def flunk msg = nil + msg ||= "Epic Fail!" + assert false, msg + end + + ## + # used for counting assertions + + def pass msg = nil + assert true + end + + ## + # Fails if +test+ is a true value + + def refute test, msg = nil + msg ||= "Failed refutation, no message given" + not assert(! test, msg) + end + + ## + # Fails if +obj+ is empty. + + def refute_empty obj, msg = nil + msg = message(msg) { "Expected #{mu_pp(obj)} to not be empty" } + assert_respond_to obj, :empty? + refute obj.empty?, msg + end + + ## + # Fails if <tt>exp == act</tt>. + # + # For floats use refute_in_delta. + + def refute_equal exp, act, msg = nil + msg = message(msg) { + "Expected #{mu_pp(act)} to not be equal to #{mu_pp(exp)}" + } + refute exp == act, msg + end + + ## + # For comparing Floats. Fails if +exp+ is within +delta+ of +act+. + # + # refute_in_delta Math::PI, (22.0 / 7.0) + + def refute_in_delta exp, act, delta = 0.001, msg = nil + n = (exp - act).abs + msg = message(msg) { + "Expected |#{exp} - #{act}| (#{n}) to not be <= #{delta}" + } + refute delta >= n, msg + end + + ## + # For comparing Floats. Fails if +exp+ and +act+ have a relative error + # less than +epsilon+. + + def refute_in_epsilon a, b, epsilon = 0.001, msg = nil + refute_in_delta a, b, a * epsilon, msg + end + + ## + # Fails if +collection+ includes +obj+. + + def refute_includes collection, obj, msg = nil + msg = message(msg) { + "Expected #{mu_pp(collection)} to not include #{mu_pp(obj)}" + } + assert_respond_to collection, :include? + refute collection.include?(obj), msg + end + + ## + # Fails if +obj+ is an instance of +cls+. + + def refute_instance_of cls, obj, msg = nil + msg = message(msg) { + "Expected #{mu_pp(obj)} to not be an instance of #{cls}" + } + refute obj.instance_of?(cls), msg + end + + ## + # Fails if +obj+ is a kind of +cls+. + + def refute_kind_of cls, obj, msg = nil # TODO: merge with instance_of + msg = message(msg) { "Expected #{mu_pp(obj)} to not be a kind of #{cls}" } + refute obj.kind_of?(cls), msg + end + + ## + # Fails if +matcher+ <tt>=~</tt> +obj+. + + def refute_match matcher, obj, msg = nil + msg = message(msg) {"Expected #{mu_pp matcher} to not match #{mu_pp obj}"} + assert_respond_to matcher, :"=~" + matcher = Regexp.new Regexp.escape matcher if String === matcher + refute matcher =~ obj, msg + end + + ## + # Fails if +obj+ is nil. + + def refute_nil obj, msg = nil + msg = message(msg) { "Expected #{mu_pp(obj)} to not be nil" } + refute obj.nil?, msg + end + + ## + # Fails if +o1+ is not +op+ +o2+. Eg: + # + # refute_operator 1, :>, 2 #=> pass + # refute_operator 1, :<, 2 #=> fail + + def refute_operator o1, op, o2 = (predicate = true; nil), msg = nil + return refute_predicate o1, op, msg if predicate + msg = message(msg) { "Expected #{mu_pp(o1)} to not be #{op} #{mu_pp(o2)}"} + refute o1.__send__(op, o2), msg + end + + ## + # For testing with predicates. + # + # refute_predicate str, :empty? + # + # This is really meant for specs and is front-ended by refute_operator: + # + # str.wont_be :empty? + + def refute_predicate o1, op, msg = nil + msg = message(msg) { "Expected #{mu_pp(o1)} to not be #{op}" } + refute o1.__send__(op), msg + end + + ## + # Fails if +obj+ responds to the message +meth+. + + def refute_respond_to obj, meth, msg = nil + msg = message(msg) { "Expected #{mu_pp(obj)} to not respond to #{meth}" } + + refute obj.respond_to?(meth), msg + end + + ## + # Fails if +exp+ is the same (by object identity) as +act+. + + def refute_same exp, act, msg = nil + msg = message(msg) { + data = [mu_pp(act), act.object_id, mu_pp(exp), exp.object_id] + "Expected %s (oid=%d) to not be the same as %s (oid=%d)" % data + } + refute exp.equal?(act), msg + end + + ## + # Skips the current test. Gets listed at the end of the run but + # doesn't cause a failure exit code. + + def pend msg = nil, bt = caller + msg ||= "Skipped, no message given" + @skip = true + raise Test::Unit::PendedError, msg, bt + end + alias omit pend + + # TODO: Removed this and enabled to raise NoMethodError with skip + alias skip pend + # def skip(msg = nil, bt = caller) + # raise NoMethodError, "use omit or pend", caller + # end + + ## + # Was this testcase skipped? Meant for #teardown. + + def skipped? + defined?(@skip) and @skip + end + + ## + # Takes a block and wraps it with the runner's shared mutex. + + def synchronize + Test::Unit::Runner.runner.synchronize do + yield + end + end + + # :call-seq: + # assert_block( failure_message = nil ) + # + #Tests the result of the given block. If the block does not return true, + #the assertion will fail. The optional +failure_message+ argument is the same as in + #Assertions#assert. + # + # assert_block do + # [1, 2, 3].any? { |num| num < 1 } + # end + def assert_block(*msgs) + assert yield, *msgs + end + + def assert_raises(*exp, &b) + raise NoMethodError, "use assert_raise", caller + end + + # :call-seq: + # assert_nothing_thrown( failure_message = nil, &block ) + # + #Fails if the given block uses a call to Kernel#throw, and + #returns the result of the block otherwise. + # + #An optional failure message may be provided as the final argument. + # + # assert_nothing_thrown "Something was thrown!" do + # throw :problem? + # end + def assert_nothing_thrown(msg=nil) + begin + ret = yield + rescue ArgumentError => error + raise error if /\Auncaught throw (.+)\z/m !~ error.message + msg = message(msg) { "<#{$1}> was thrown when nothing was expected" } + flunk(msg) + end + assert(true, "Expected nothing to be thrown") + ret + end + + # :call-seq: + # assert_equal( expected, actual, failure_message = nil ) + # + #Tests if +expected+ is equal to +actual+. + # + #An optional failure message may be provided as the final argument. + def assert_equal(exp, act, msg = nil) + msg = message(msg) { + exp_str = mu_pp(exp) + act_str = mu_pp(act) + exp_comment = '' + act_comment = '' + if exp_str == act_str + if (exp.is_a?(String) && act.is_a?(String)) || + (exp.is_a?(Regexp) && act.is_a?(Regexp)) + exp_comment = " (#{exp.encoding})" + act_comment = " (#{act.encoding})" + elsif exp.is_a?(Float) && act.is_a?(Float) + exp_str = "%\#.#{Float::DIG+2}g" % exp + act_str = "%\#.#{Float::DIG+2}g" % act + elsif exp.is_a?(Time) && act.is_a?(Time) + if exp.subsec * 1000_000_000 == exp.nsec + exp_comment = " (#{exp.nsec}[ns])" + else + exp_comment = " (subsec=#{exp.subsec})" + end + if act.subsec * 1000_000_000 == act.nsec + act_comment = " (#{act.nsec}[ns])" + else + act_comment = " (subsec=#{act.subsec})" + end + elsif exp.class != act.class + # a subclass of Range, for example. + exp_comment = " (#{exp.class})" + act_comment = " (#{act.class})" + end + elsif !Encoding.compatible?(exp_str, act_str) + if exp.is_a?(String) && act.is_a?(String) + exp_str = exp.dump + act_str = act.dump + exp_comment = " (#{exp.encoding})" + act_comment = " (#{act.encoding})" + else + exp_str = exp_str.dump + act_str = act_str.dump + end + end + "<#{exp_str}>#{exp_comment} expected but was\n<#{act_str}>#{act_comment}" + } + assert(exp == act, msg) + end + + # :call-seq: + # assert_not_nil( expression, failure_message = nil ) + # + #Tests if +expression+ is not nil. + # + #An optional failure message may be provided as the final argument. + def assert_not_nil(exp, msg=nil) + msg = message(msg) { "<#{mu_pp(exp)}> expected to not be nil" } + assert(!exp.nil?, msg) + end + + # :call-seq: + # assert_not_equal( expected, actual, failure_message = nil ) + # + #Tests if +expected+ is not equal to +actual+. + # + #An optional failure message may be provided as the final argument. + def assert_not_equal(exp, act, msg=nil) + msg = message(msg) { "<#{mu_pp(exp)}> expected to be != to\n<#{mu_pp(act)}>" } + assert(exp != act, msg) + end + + # :call-seq: + # assert_no_match( regexp, string, failure_message = nil ) + # + #Tests if the given Regexp does not match a given String. + # + #An optional failure message may be provided as the final argument. + def assert_no_match(regexp, string, msg=nil) + assert_instance_of(Regexp, regexp, "The first argument to assert_no_match should be a Regexp.") + self._assertions -= 1 + msg = message(msg) { "<#{mu_pp(regexp)}> expected to not match\n<#{mu_pp(string)}>" } + assert(regexp !~ string, msg) + end + + # :call-seq: + # assert_not_same( expected, actual, failure_message = nil ) + # + #Tests if +expected+ is not the same object as +actual+. + #This test uses Object#equal? to test equality. + # + #An optional failure message may be provided as the final argument. + # + # assert_not_same("x", "x") #Succeeds + def assert_not_same(expected, actual, message="") + msg = message(msg) { build_message(message, <<EOT, expected, expected.__id__, actual, actual.__id__) } +<?> +with id <?> expected to not be equal\\? to +<?> +with id <?>. +EOT + assert(!actual.equal?(expected), msg) + end + + # :call-seq: + # assert_send( +send_array+, failure_message = nil ) + # + # Passes if the method send returns a true value. + # + # +send_array+ is composed of: + # * A receiver + # * A method + # * Arguments to the method + # + # Example: + # assert_send(["Hello world", :include?, "Hello"]) # -> pass + # assert_send(["Hello world", :include?, "Goodbye"]) # -> fail + def assert_send send_ary, m = nil + recv, msg, *args = send_ary + m = message(m) { + if args.empty? + argsstr = "" + else + (argsstr = mu_pp(args)).sub!(/\A\[(.*)\]\z/m, '(\1)') + end + "Expected #{mu_pp(recv)}.#{msg}#{argsstr} to return true" + } + assert recv.__send__(msg, *args), m + end + + # :call-seq: + # assert_not_send( +send_array+, failure_message = nil ) + # + # Passes if the method send doesn't return a true value. + # + # +send_array+ is composed of: + # * A receiver + # * A method + # * Arguments to the method + # + # Example: + # assert_not_send([[1, 2], :member?, 1]) # -> fail + # assert_not_send([[1, 2], :member?, 4]) # -> pass + def assert_not_send send_ary, m = nil + recv, msg, *args = send_ary + m = message(m) { + if args.empty? + argsstr = "" + else + (argsstr = mu_pp(args)).sub!(/\A\[(.*)\]\z/m, '(\1)') + end + "Expected #{mu_pp(recv)}.#{msg}#{argsstr} to return false" + } + assert !recv.__send__(msg, *args), m + end + + ms = instance_methods(true).map {|sym| sym.to_s } + ms.grep(/\Arefute_/) do |m| + mname = ('assert_not_'.dup << m.to_s[/.*?_(.*)/, 1]) + alias_method(mname, m) unless ms.include? mname + end + alias assert_include assert_includes + alias assert_not_include assert_not_includes + + def assert_not_all?(obj, m = nil, &blk) + failed = [] + obj.each do |*a, &b| + if blk.call(*a, &b) + failed << (a.size > 1 ? a : a[0]) + end + end + assert(failed.empty?, message(m) {failed.pretty_inspect}) + end + + def assert_syntax_error(code, error, *args, **opt) + prepare_syntax_check(code, *args, **opt) do |src, fname, line, mesg| + yield if defined?(yield) + e = assert_raise(SyntaxError, mesg) do + syntax_check(src, fname, line) + end + assert_match(error, e.message, mesg) + e + end + end + + def assert_no_warning(pat, msg = nil) + result = nil + stderr = EnvUtil.verbose_warning { + EnvUtil.with_default_internal(pat.encoding) { + result = yield + } + } + msg = message(msg) {diff pat, stderr} + refute(pat === stderr, msg) + result + end + + # kernel resolution can limit the minimum time we can measure + # [ruby-core:81540] + MIN_HZ = /mswin|mingw/ =~ RUBY_PLATFORM ? 67 : 100 + MIN_MEASURABLE = 1.0 / MIN_HZ + + def assert_cpu_usage_low(msg = nil, pct: 0.05, wait: 1.0, stop: nil) + require 'benchmark' + + wait = EnvUtil.apply_timeout_scale(wait) + if wait < 0.1 # TIME_QUANTUM_USEC in thread_pthread.c + warn "test #{msg || 'assert_cpu_usage_low'} too short to be accurate" + end + tms = Benchmark.measure(msg || '') do + if stop + th = Thread.start {sleep wait; stop.call} + yield + th.join + else + begin + Timeout.timeout(wait) {yield} + rescue Timeout::Error + end + end + end + + max = pct * tms.real + min_measurable = MIN_MEASURABLE + min_measurable *= 1.30 # add a little (30%) to account for misc. overheads + if max < min_measurable + max = min_measurable + end + + assert_operator tms.total, :<=, max, msg + end + + def assert_is_minus_zero(f) + assert(1.0/f == -Float::INFINITY, "#{f} is not -0.0") + end + + def build_message(head, template=nil, *arguments) #:nodoc: + template &&= template.chomp + template.gsub(/\G((?:[^\\]|\\.)*?)(\\)?\?/) { $1 + ($2 ? "?" : mu_pp(arguments.shift)) } + end + end + end +end diff --git a/tool/lib/test/unit/parallel.rb b/tool/lib/test/unit/parallel.rb new file mode 100644 index 0000000000..b3a8957f26 --- /dev/null +++ b/tool/lib/test/unit/parallel.rb @@ -0,0 +1,212 @@ +# frozen_string_literal: true +$LOAD_PATH.unshift "#{__dir__}/../.." +require_relative '../../test/unit' + +require_relative '../../profile_test_all' if ENV.key?('RUBY_TEST_ALL_PROFILE') +require_relative '../../tracepointchecker' +require_relative '../../zombie_hunter' +require_relative '../../iseq_loader_checker' +require_relative '../../gc_checker' + +module Test + module Unit + class Worker < Runner # :nodoc: + class << self + undef autorun + end + + undef _run_suite + undef _run_suites + undef run + + def increment_io(orig) # :nodoc: + *rest, io = 32.times.inject([orig.dup]){|ios, | ios << ios.last.dup } + rest.each(&:close) + io + end + + def _run_suites(suites, type) # :nodoc: + suites.map do |suite| + _run_suite(suite, type) + end + end + + def _start_method(inst) + _report "start", Marshal.dump([inst.class.name, inst.__name__]) + end + + def _run_suite(suite, type) # :nodoc: + @partial_report = [] + orig_testout = Test::Unit::Runner.output + i,o = IO.pipe + + Test::Unit::Runner.output = o + orig_stdin, orig_stdout = $stdin, $stdout + + th = Thread.new do + begin + while buf = (self.verbose ? i.gets : i.readpartial(1024)) + _report "p", buf or break + end + rescue IOError + end + end + + e, f, s = @errors, @failures, @skips + + begin + result = orig_run_suite(suite, type) + rescue Interrupt + @need_exit = true + result = [nil,nil] + end + + Test::Unit::Runner.output = orig_testout + $stdin = orig_stdin + $stdout = orig_stdout + + o.close + begin + th.join + rescue IOError + raise unless /stream closed|closed stream/ =~ $!.message + end + i.close + + result << @partial_report + @partial_report = nil + result << [@errors-e,@failures-f,@skips-s] + result << ($: - @old_loadpath) + result << suite.name + + _report "done", Marshal.dump(result) + return result + ensure + Test::Unit::Runner.output = orig_stdout + $stdin = orig_stdin if orig_stdin + $stdout = orig_stdout if orig_stdout + o.close if o && !o.closed? + i.close if i && !i.closed? + end + + def run(args = []) # :nodoc: + process_args args + @@stop_auto_run = true + @opts = @options.dup + @need_exit = false + + @old_loadpath = [] + begin + begin + @stdout = increment_io(STDOUT) + @stdin = increment_io(STDIN) + rescue + exit 2 + end + exit 2 unless @stdout && @stdin + + @stdout.sync = true + _report "ready!" + while buf = @stdin.gets + case buf.chomp + when /^loadpath (.+?)$/ + @old_loadpath = $:.dup + $:.push(*Marshal.load($1.unpack1("m").force_encoding("ASCII-8BIT"))).uniq! + when /^run (.+?) (.+?)$/ + _report "okay" + + @options = @opts.dup + suites = Test::Unit::TestCase.test_suites + + begin + require File.realpath($1) + rescue LoadError + _report "after", Marshal.dump([$1, ProxyError.new($!)]) + _report "ready" + next + end + _run_suites Test::Unit::TestCase.test_suites-suites, $2.to_sym + + if @need_exit + _report "bye" + exit + else + _report "ready" + end + when /^quit$/ + _report "bye" + exit + end + end + rescue Exception => e + trace = e.backtrace || ['unknown method'] + err = ["#{trace.shift}: #{e.message} (#{e.class})"] + trace.map{|t| "\t" + t } + + if @stdout + _report "bye", Marshal.dump(err.join("\n")) + else + raise "failed to report a failure due to lack of @stdout" + end + exit + ensure + @stdin.close if @stdin + @stdout.close if @stdout + end + end + + def _report(res, *args) # :nodoc: + @stdout.write(args.empty? ? "#{res}\n" : "#{res} #{args.pack("m0")}\n") + true + rescue Errno::EPIPE + rescue TypeError => e + abort("#{e.inspect} in _report(#{res.inspect}, #{args.inspect})\n#{e.backtrace.join("\n")}") + end + + def puke(klass, meth, e) # :nodoc: + if e.is_a?(Test::Unit::PendedError) + new_e = Test::Unit::PendedError.new(e.message) + new_e.set_backtrace(e.backtrace) + e = new_e + end + @partial_report << [klass.name, meth, e.is_a?(Test::Unit::AssertionFailedError) ? e : ProxyError.new(e)] + super + end + + def record(suite, method, assertions, time, error) # :nodoc: + case error + when nil + when Test::Unit::AssertionFailedError, Test::Unit::PendedError + case error.cause + when nil, Test::Unit::AssertionFailedError, Test::Unit::PendedError + else + bt = error.backtrace + error = error.class.new(error.message) + error.set_backtrace(bt) + end + else + error = ProxyError.new(error) + end + _report "record", Marshal.dump([suite.name, method, assertions, time, error]) + super + end + end + end +end + +if $0 == __FILE__ + module Test + module Unit + class TestCase # :nodoc: all + undef on_parallel_worker? + def on_parallel_worker? + true + end + def self.on_parallel_worker? + true + end + end + end + end + require 'rubygems' + Test::Unit::Worker.new.run(ARGV) +end diff --git a/tool/lib/test/unit/testcase.rb b/tool/lib/test/unit/testcase.rb new file mode 100644 index 0000000000..44d9ba7fdb --- /dev/null +++ b/tool/lib/test/unit/testcase.rb @@ -0,0 +1,296 @@ +# frozen_string_literal: true +require_relative 'assertions' +require_relative '../../core_assertions' + +module Test + module Unit + + ## + # Provides a simple set of guards that you can use in your tests + # to skip execution if it is not applicable. These methods are + # mixed into TestCase as both instance and class methods so you + # can use them inside or outside of the test methods. + # + # def test_something_for_mri + # skip "bug 1234" if jruby? + # # ... + # end + # + # if windows? then + # # ... lots of test methods ... + # end + + module Guard + + ## + # Is this running on jruby? + + def jruby? platform = RUBY_PLATFORM + "java" == platform + end + + ## + # Is this running on mri? + + def mri? platform = RUBY_DESCRIPTION + /^ruby/ =~ platform + end + + ## + # Is this running on windows? + + def windows? platform = RUBY_PLATFORM + /mswin|mingw/ =~ platform + end + + ## + # Is this running on mingw? + + def mingw? platform = RUBY_PLATFORM + /mingw/ =~ platform + end + + end + + ## + # Provides before/after hooks for setup and teardown. These are + # meant for library writers, NOT for regular test authors. See + # #before_setup for an example. + + module LifecycleHooks + ## + # Runs before every test, after setup. This hook is meant for + # libraries to extend Test::Unit. It is not meant to be used by + # test developers. + # + # See #before_setup for an example. + + def after_setup; end + + ## + # Runs before every test, before setup. This hook is meant for + # libraries to extend Test::Unit. It is not meant to be used by + # test developers. + # + # As a simplistic example: + # + # module MyTestUnitPlugin + # def before_setup + # super + # # ... stuff to do before setup is run + # end + # + # def after_setup + # # ... stuff to do after setup is run + # super + # end + # + # def before_teardown + # super + # # ... stuff to do before teardown is run + # end + # + # def after_teardown + # # ... stuff to do after teardown is run + # super + # end + # end + # + # class Test::Unit::Runner::TestCase + # include MyTestUnitPlugin + # end + + def before_setup; end + + ## + # Runs after every test, before teardown. This hook is meant for + # libraries to extend Test::Unit. It is not meant to be used by + # test developers. + # + # See #before_setup for an example. + + def before_teardown; end + + ## + # Runs after every test, after teardown. This hook is meant for + # libraries to extend Test::Unit. It is not meant to be used by + # test developers. + # + # See #before_setup for an example. + + def after_teardown; end + end + + ## + # Subclass TestCase to create your own tests. Typically you'll want a + # TestCase subclass per implementation class. + # + # See <code>Test::Unit::AssertionFailedError</code>s + + class TestCase + include Assertions + include CoreAssertions + + include LifecycleHooks + include Guard + extend Guard + + attr_reader :__name__ # :nodoc: + + PASSTHROUGH_EXCEPTIONS = [NoMemoryError, SignalException, + Interrupt, SystemExit] # :nodoc: + + ## + # Runs the tests reporting the status to +runner+ + + def run runner + @options = runner.options + + trap "INFO" do + runner.report.each_with_index do |msg, i| + warn "\n%3d) %s" % [i + 1, msg] + end + warn '' + time = runner.start_time ? Time.now - runner.start_time : 0 + warn "Current Test: %s#%s %.2fs" % [self.class, self.__name__, time] + runner.status $stderr + end if runner.info_signal + + start_time = Time.now + + result = "" + + begin + @passed = nil + self.before_setup + self.setup + self.after_setup + self.run_test self.__name__ + result = "." unless io? + time = Time.now - start_time + runner.record self.class, self.__name__, self._assertions, time, nil + @passed = true + rescue *PASSTHROUGH_EXCEPTIONS + raise + rescue Exception => e + @passed = Test::Unit::PendedError === e + time = Time.now - start_time + runner.record self.class, self.__name__, self._assertions, time, e + result = runner.puke self.class, self.__name__, e + ensure + %w{ before_teardown teardown after_teardown }.each do |hook| + begin + self.send hook + rescue *PASSTHROUGH_EXCEPTIONS + raise + rescue Exception => e + @passed = false + runner.record self.class, self.__name__, self._assertions, time, e + result = runner.puke self.class, self.__name__, e + end + end + trap 'INFO', 'DEFAULT' if runner.info_signal + end + result + end + + RUN_TEST_TRACE = "#{__FILE__}:#{__LINE__+3}:in `run_test'".freeze + def run_test(name) + progname, $0 = $0, "#{$0}: #{self.class}##{name}" + self.__send__(name) + ensure + $@.delete(RUN_TEST_TRACE) if $@ + $0 = progname + end + + def initialize name # :nodoc: + @__name__ = name + @__io__ = nil + @passed = nil + @@current = self # FIX: make thread local + end + + def self.current # :nodoc: + @@current # FIX: make thread local + end + + ## + # Return the output IO object + + def io + @__io__ = true + Test::Unit::Runner.output + end + + ## + # Have we hooked up the IO yet? + + def io? + @__io__ + end + + def self.reset # :nodoc: + @@test_suites = {} + @@test_suites[self] = true + end + + reset + + def self.inherited klass # :nodoc: + @@test_suites[klass] = true + super + end + + @test_order = :sorted + + class << self + attr_writer :test_order + end + + def self.test_order + defined?(@test_order) ? @test_order : superclass.test_order + end + + def self.test_suites # :nodoc: + @@test_suites.keys + end + + def self.test_methods # :nodoc: + public_instance_methods(true).grep(/^test/) + end + + ## + # Returns true if the test passed. + + def passed? + @passed + end + + ## + # Runs before every test. Use this to set up before each test + # run. + + def setup; end + + ## + # Runs after every test. Use this to clean up after each test + # run. + + def teardown; end + + def on_parallel_worker? + false + end + + def self.method_added(name) + super + return unless name.to_s.start_with?("test_") + @test_methods ||= {} + if @test_methods[name] + raise AssertionFailedError, "test/unit: method #{ self }##{ name } is redefined" + end + @test_methods[name] = true + end + end + end +end diff --git a/tool/lib/tracepointchecker.rb b/tool/lib/tracepointchecker.rb new file mode 100644 index 0000000000..3254e59357 --- /dev/null +++ b/tool/lib/tracepointchecker.rb @@ -0,0 +1,126 @@ +# frozen_string_literal: true +module TracePointChecker + STATE = { + count: 0, + running: false, + } + + module ZombieTraceHunter + def tracepoint_capture_stat_get + TracePoint.stat.map{|k, (activated, deleted)| + deleted = 0 unless @tracepoint_captured_singlethread + [k, activated, deleted] + } + end + + def before_setup + @tracepoint_captured_singlethread = (Thread.list.size == 1) + @tracepoint_captured_stat = tracepoint_capture_stat_get() + super + end + + def after_teardown + super + + # detect zombie traces. + assert_equal( + @tracepoint_captured_stat, + tracepoint_capture_stat_get(), + "The number of active/deleted trace events was changed" + ) + # puts "TracePoint - deleted: #{deleted}" if deleted > 0 + + TracePointChecker.check if STATE[:running] + end + end + + MAIN_THREAD = Thread.current + TRACES = [] + + def self.prefix event + case event + when :call, :return + :n + when :c_call, :c_return + :c + when :b_call, :b_return + :b + end + end + + def self.clear_call_stack + Thread.current[:call_stack] = [] + end + + def self.call_stack + stack = Thread.current[:call_stack] + stack = clear_call_stack unless stack + stack + end + + def self.verbose_out label, method + puts label => call_stack, :count => STATE[:count], :method => method + end + + def self.method_label tp + "#{prefix(tp.event)}##{tp.method_id}" + end + + def self.start verbose: false, stop_at_failure: false + call_events = %i(a_call) + return_events = %i(a_return) + clear_call_stack + + STATE[:running] = true + + TRACES << TracePoint.new(*call_events){|tp| + next if Thread.current != MAIN_THREAD + + method = method_label(tp) + call_stack.push method + STATE[:count] += 1 + + verbose_out :push, method if verbose + } + + TRACES << TracePoint.new(*return_events){|tp| + next if Thread.current != MAIN_THREAD + STATE[:count] += 1 + + method = "#{prefix(tp.event)}##{tp.method_id}" + verbose_out :pop1, method if verbose + + stored_method = call_stack.pop + next if stored_method.nil? + + verbose_out :pop2, method if verbose + + if stored_method != method + stop if stop_at_failure + RubyVM::SDR() if defined? RubyVM::SDR() + call_stack.clear + raise "#{stored_method} is expected, but #{method} (count: #{STATE[:count]})" + end + } + + TRACES.each{|trace| trace.enable} + end + + def self.stop + STATE[:running] = true + TRACES.each{|trace| trace.disable} + TRACES.clear + end + + def self.check + TRACES.each{|trace| + raise "trace #{trace} should not be deactivated" unless trace.enabled? + } + end +end if defined?(TracePoint.stat) + +class ::Test::Unit::TestCase + include TracePointChecker::ZombieTraceHunter +end if defined?(TracePointChecker) + +# TracePointChecker.start verbose: false diff --git a/tool/lib/vcs.rb b/tool/lib/vcs.rb new file mode 100644 index 0000000000..c41276e3b4 --- /dev/null +++ b/tool/lib/vcs.rb @@ -0,0 +1,733 @@ +# vcs +require 'fileutils' +require 'optparse' + +# This library is used by several other tools/ scripts to detect the current +# VCS in use (e.g. SVN, Git) or to interact with that VCS. + +ENV.delete('PWD') + +class VCS + DEBUG_OUT = STDERR.dup +end + +unless File.respond_to? :realpath + require 'pathname' + def File.realpath(arg) + Pathname(arg).realpath.to_s + end +end + +def IO.pread(*args) + VCS::DEBUG_OUT.puts(args.inspect) if $DEBUG + popen(*args) {|f|f.read} +end + +module DebugPOpen + refine IO.singleton_class do + def popen(*args) + VCS::DEBUG_OUT.puts args.inspect if $DEBUG + super + end + end +end +using DebugPOpen +module DebugSystem + def system(*args) + VCS::DEBUG_OUT.puts args.inspect if $DEBUG + exception = false + opts = Hash.try_convert(args[-1]) + if RUBY_VERSION >= "2.6" + unless opts + opts = {} + args << opts + end + exception = opts.fetch(:exception) {opts[:exception] = true} + elsif opts + exception = opts.delete(:exception) {true} + args.pop if opts.empty? + end + ret = super(*args) + raise "Command failed with status (#$?): #{args[0]}" if exception and !ret + ret + end +end + +class VCS + prepend(DebugSystem) if defined?(DebugSystem) + class NotFoundError < RuntimeError; end + + @@dirs = [] + def self.register(dir, &pred) + @@dirs << [dir, self, pred] + end + + def self.detect(path = '.', options = {}, parser = nil, **opts) + options.update(opts) + uplevel_limit = options.fetch(:uplevel_limit, 0) + curr = path + begin + @@dirs.each do |dir, klass, pred| + if pred ? pred[curr, dir] : File.directory?(File.join(curr, dir)) + vcs = klass.new(curr) + vcs.define_options(parser) if parser + vcs.set_options(options) + return vcs + end + end + if uplevel_limit + break if uplevel_limit.zero? + uplevel_limit -= 1 + end + prev, curr = curr, File.realpath(File.join(curr, '..')) + end until curr == prev # stop at the root directory + raise VCS::NotFoundError, "does not seem to be under a vcs: #{path}" + end + + def self.local_path?(path) + String === path or path.respond_to?(:to_path) + end + + def self.define_options(parser, opts = {}) + parser.separator(" VCS common options:") + parser.define("--[no-]dryrun") {|v| opts[:dryrun] = v} + parser.define("--[no-]debug") {|v| opts[:debug] = v} + opts + end + + attr_reader :srcdir + + def initialize(path) + @srcdir = path + super() + end + + def chdir(path) + @srcdir = path + end + + def define_options(parser) + end + + def set_options(opts) + @debug = opts.fetch(:debug) {$DEBUG} + @dryrun = opts.fetch(:dryrun) {@debug} + end + + attr_reader :dryrun, :debug + alias dryrun? dryrun + alias debug? debug + + NullDevice = defined?(IO::NULL) ? IO::NULL : + %w[/dev/null NUL NIL: NL:].find {|dev| File.exist?(dev)} + + # returns + # * the last revision of the current branch + # * the last revision in which +path+ was modified + # * the last modified time of +path+ + # * the last commit title since the latest upstream + def get_revisions(path) + if self.class.local_path?(path) + path = relative_to(path) + end + last, changed, modified, *rest = ( + begin + if NullDevice + save_stderr = STDERR.dup + STDERR.reopen NullDevice, 'w' + end + _get_revisions(path, @srcdir) + rescue Errno::ENOENT => e + raise VCS::NotFoundError, e.message + ensure + if save_stderr + STDERR.reopen save_stderr + save_stderr.close + end + end + ) + last or raise VCS::NotFoundError, "last revision not found" + changed or raise VCS::NotFoundError, "changed revision not found" + if modified + /\A(\d+)-(\d+)-(\d+)\D(\d+):(\d+):(\d+(?:\.\d+)?)\s*(?:Z|([-+]\d\d)(\d\d))\z/ =~ modified or + raise "unknown time format - #{modified}" + match = $~[1..6].map { |x| x.to_i } + off = $7 ? "#{$7}:#{$8}" : "+00:00" + match << off + begin + modified = Time.new(*match) + rescue ArgumentError + modified = Time.utc(*$~[1..6]) + $7.to_i * 3600 + $8.to_i * 60 + end + end + return last, changed, modified, *rest + end + + def modified(path) + _, _, modified, * = get_revisions(path) + modified + end + + def relative_to(path) + if path + srcdir = File.realpath(@srcdir) + path = File.realdirpath(path) + list1 = srcdir.split(%r{/}) + list2 = path.split(%r{/}) + while !list1.empty? && !list2.empty? && list1.first == list2.first + list1.shift + list2.shift + end + if list1.empty? && list2.empty? + "." + else + ([".."] * list1.length + list2).join("/") + end + else + '.' + end + end + + def after_export(dir) + FileUtils.rm_rf(Dir.glob("#{dir}/.git*")) + end + + def revision_handler(rev) + self.class + end + + def revision_name(rev) + revision_handler(rev).revision_name(rev) + end + + def short_revision(rev) + revision_handler(rev).short_revision(rev) + end + + class SVN < self + register(".svn") + COMMAND = ENV['SVN'] || 'svn' + + def self.revision_name(rev) + "r#{rev}" + end + + def self.short_revision(rev) + rev + end + + def _get_revisions(path, srcdir = nil) + if srcdir and self.class.local_path?(path) + path = File.join(srcdir, path) + end + if srcdir + info_xml = IO.pread(%W"#{COMMAND} info --xml #{srcdir}") + info_xml = nil unless info_xml[/<url>(.*)<\/url>/, 1] == path.to_s + end + info_xml ||= IO.pread(%W"#{COMMAND} info --xml #{path}") + _, last, _, changed, _ = info_xml.split(/revision="(\d+)"/) + modified = info_xml[/<date>([^<>]*)/, 1] + branch = info_xml[%r'<relative-url>\^/(?:branches/|tags/)?([^<>]+)', 1] + [Integer(last), Integer(changed), modified, branch] + end + + def self.search_root(path) + return unless local_path?(path) + parent = File.realpath(path) + begin + parent = File.dirname(wkdir = parent) + return wkdir if File.directory?(wkdir + "/.svn") + end until parent == wkdir + end + + def get_info + @info ||= IO.pread(%W"#{COMMAND} info --xml #{@srcdir}") + end + + def url + @url ||= begin + url = get_info[/<root>(.*)<\/root>/, 1] + @url = URI.parse(url+"/") if url + end + end + + def wcroot + @wcroot ||= begin + info = get_info + @wcroot = info[/<wcroot-abspath>(.*)<\/wcroot-abspath>/, 1] + @wcroot ||= self.class.search_root(@srcdir) + end + end + + def branch(name) + return trunk if name == "trunk" + url + "branches/#{name}" + end + + def tag(name) + url + "tags/#{name}" + end + + def trunk + url + "trunk" + end + alias master trunk + + def branch_list(pat) + IO.popen(%W"#{COMMAND} ls #{branch('')}") do |f| + f.each do |line| + line.chomp! + line.chomp!('/') + yield(line) if File.fnmatch?(pat, line) + end + end + end + + def grep(pat, tag, *files, &block) + cmd = %W"#{COMMAND} cat" + files.map! {|n| File.join(tag, n)} if tag + set = block.binding.eval("proc {|match| $~ = match}") + IO.popen([cmd, *files]) do |f| + f.grep(pat) do |s| + set[$~] + yield s + end + end + end + + def export(revision, url, dir, keep_temp = false) + if @srcdir and (rootdir = wcroot) + srcdir = File.realpath(@srcdir) + rootdir << "/" + if srcdir.start_with?(rootdir) + subdir = srcdir[rootdir.size..-1] + subdir = nil if subdir.empty? + FileUtils.mkdir_p(svndir = dir+"/.svn") + FileUtils.ln_s(Dir.glob(rootdir+"/.svn/*"), svndir) + system(COMMAND, "-q", "revert", "-R", subdir || ".", :chdir => dir) or return false + FileUtils.rm_rf(svndir) unless keep_temp + if subdir + tmpdir = Dir.mktmpdir("tmp-co.", "#{dir}/#{subdir}") + File.rename(tmpdir, tmpdir = "#{dir}/#{File.basename(tmpdir)}") + FileUtils.mv(Dir.glob("#{dir}/#{subdir}/{.[^.]*,..?*,*}"), tmpdir) + begin + Dir.rmdir("#{dir}/#{subdir}") + end until (subdir = File.dirname(subdir)) == '.' + FileUtils.mv(Dir.glob("#{tmpdir}/#{subdir}/{.[^.]*,..?*,*}"), dir) + Dir.rmdir(tmpdir) + end + return self + end + end + IO.popen(%W"#{COMMAND} export -r #{revision} #{url} #{dir}") do |pipe| + pipe.each {|line| /^A/ =~ line or yield line} + end + self if $?.success? + end + + def after_export(dir) + super + FileUtils.rm_rf(dir+"/.svn") + end + + def branch_beginning(url) + # `--limit` of svn-log is useless in this case, because it is + # applied before `--search`. + rev = IO.pread(%W[ #{COMMAND} log --xml + --search=matz --search-and=has\ started + -- #{url}/version.h])[/<logentry\s+revision="(\d+)"/m, 1] + rev.to_i if rev + end + + def export_changelog(url = '.', from = nil, to = nil, _path = nil, path: _path) + range = [to || 'HEAD', (from ? from+1 : branch_beginning(url))].compact.join(':') + IO.popen({'TZ' => 'JST-9', 'LANG' => 'C', 'LC_ALL' => 'C'}, + %W"#{COMMAND} log -r#{range} #{url}") do |r| + IO.copy_stream(r, path) + end + end + + def commit + args = %W"#{COMMAND} commit" + if dryrun? + VCS::DEBUG_OUT.puts(args.inspect) + return true + end + system(*args) + end + end + + class GIT < self + register(".git") {|path, dir| File.exist?(File.join(path, dir))} + COMMAND = ENV["GIT"] || 'git' + + def cmd_args(cmds, srcdir = nil) + (opts = cmds.last).kind_of?(Hash) or cmds << (opts = {}) + opts[:external_encoding] ||= "UTF-8" + if srcdir + opts[:chdir] ||= srcdir + end + VCS::DEBUG_OUT.puts cmds.inspect if debug? + cmds + end + + def cmd_pipe_at(srcdir, cmds, &block) + without_gitconfig { IO.popen(*cmd_args(cmds, srcdir), &block) } + end + + def cmd_read_at(srcdir, cmds) + result = without_gitconfig { IO.pread(*cmd_args(cmds, srcdir)) } + VCS::DEBUG_OUT.puts result.inspect if debug? + result + end + + def cmd_pipe(*cmds, &block) + cmd_pipe_at(@srcdir, cmds, &block) + end + + def cmd_read(*cmds) + cmd_read_at(@srcdir, cmds) + end + + def svn_revision(log) + if /^ *git-svn-id: .*@(\d+) .*\n+\z/ =~ log + $1.to_i + end + end + + def _get_revisions(path, srcdir = nil) + ref = Branch === path ? path.to_str : 'HEAD' + gitcmd = [COMMAND] + last = cmd_read_at(srcdir, [[*gitcmd, 'rev-parse', ref]]).rstrip + log = cmd_read_at(srcdir, [[*gitcmd, 'log', '-n1', '--date=iso', '--pretty=fuller', *path]]) + changed = log[/\Acommit (\h+)/, 1] + modified = log[/^CommitDate:\s+(.*)/, 1] + if rev = svn_revision(log) + if changed == last + last = rev + else + svn_rev = svn_revision(cmd_read_at(srcdir, [[*gitcmd, 'log', '-n1', '--format=%B', last]])) + last = svn_rev if svn_rev + end + changed = rev + end + branch = cmd_read_at(srcdir, [gitcmd + %W[symbolic-ref --short #{ref}]]) + if branch.empty? + branch = cmd_read_at(srcdir, [gitcmd + %W[describe --contains #{ref}]]).strip + end + if branch.empty? + branch_list = cmd_read_at(srcdir, [gitcmd + %W[branch --list --contains #{ref}]]).lines.to_a + branch, = branch_list.grep(/\A\*/) + case branch + when /\A\* *\(\S+ detached at (.*)\)\Z/ + branch = $1 + branch = nil if last.start_with?(branch) + when /\A\* (\S+)\Z/ + branch = $1 + else + branch = nil + end + unless branch + branch_list.each {|b| b.strip!} + branch_list.delete_if {|b| / / =~ b} + branch = branch_list.min_by(&:length) || "" + end + end + branch.chomp! + branch = ":detached:" if branch.empty? + upstream = cmd_read_at(srcdir, [gitcmd + %W[branch --list --format=%(upstream:short) #{branch}]]) + upstream.chomp! + title = cmd_read_at(srcdir, [gitcmd + %W[log --format=%s -n1 #{upstream}..#{ref}]]) + title = nil if title.empty? + [last, changed, modified, branch, title] + end + + def self.revision_name(rev) + short_revision(rev) + end + + def self.short_revision(rev) + rev[0, 10] + end + + def revision_handler(rev) + case rev + when Integer + SVN + else + super + end + end + + def without_gitconfig + home = ENV.delete('HOME') + yield + ensure + ENV['HOME'] = home if home + end + + def initialize(*) + super + @srcdir = File.realpath(@srcdir) + VCS::DEBUG_OUT.puts @srcdir.inspect if debug? + self + end + + Branch = Struct.new(:to_str) + + def branch(name) + Branch.new(name) + end + + alias tag branch + + def master + branch("master") + end + alias trunk master + + def stable + cmd = %W"#{COMMAND} for-each-ref --format=\%(refname:short) refs/heads/ruby_[0-9]*" + branch(cmd_read(cmd)[/.*^(ruby_\d+_\d+)$/m, 1]) + end + + def branch_list(pat) + cmd = %W"#{COMMAND} for-each-ref --format=\%(refname:short) refs/heads/#{pat}" + cmd_pipe(cmd) {|f| + f.each {|line| + line.chomp! + yield line + } + } + end + + def grep(pat, tag, *files, &block) + cmd = %W[#{COMMAND} grep -h --perl-regexp #{tag} --] + set = block.binding.eval("proc {|match| $~ = match}") + cmd_pipe(cmd+files) do |f| + f.grep(pat) do |s| + set[$~] + yield s + end + end + end + + def export(revision, url, dir, keep_temp = false) + system(COMMAND, "clone", "-c", "advice.detachedHead=false", "-s", (@srcdir || '.').to_s, "-b", url, dir) or return + (Integer === revision ? GITSVN : GIT).new(File.expand_path(dir)) + end + + def branch_beginning(url) + cmd_read(%W[ #{COMMAND} log -n1 --format=format:%H + --author=matz --committer=matz --grep=started\\.$ + #{url.to_str} -- version.h include/ruby/version.h]) + end + + def export_changelog(url = '@', from = nil, to = nil, _path = nil, path: _path, base_url: nil) + svn = nil + from, to = [from, to].map do |rev| + rev or next + if Integer === rev + svn = true + rev = cmd_read({'LANG' => 'C', 'LC_ALL' => 'C'}, + %W"#{COMMAND} log -n1 --format=format:%H" << + "--grep=^ *git-svn-id: .*@#{rev} ") + end + rev unless rev.empty? + end + unless (from && /./.match(from)) or ((from = branch_beginning(url)) && /./.match(from)) + warn "no starting commit found", uplevel: 1 + from = nil + end + if svn or system(*%W"#{COMMAND} fetch origin refs/notes/commits:refs/notes/commits", + chdir: @srcdir, exception: false) + system(*%W"#{COMMAND} fetch origin refs/notes/log-fix:refs/notes/log-fix", + chdir: @srcdir, exception: false) + else + warn "Could not fetch notes/commits tree", uplevel: 1 + end + to ||= url.to_str + if from + arg = ["#{from}^..#{to}"] + else + arg = ["--since=25 Dec 00:00:00", to] + end + writer = + if svn + format_changelog_as_svn(path, arg) + else + if base_url == true + remote, = upstream + if remote &&= cmd_read(env, %W[#{COMMAND} remote get-url --no-push #{remote}]) + remote.chomp! + # hack to redirect git.r-l.o to github + remote.sub!(/\Agit@git\.ruby-lang\.org:/, 'git@github.com:ruby/') + remote.sub!(/\Agit@(.*?):(.*?)(?:\.git)?\z/, 'https://\1/\2/commit/') + end + base_url = remote + end + format_changelog(path, arg, base_url) + end + if !path or path == '-' + writer[$stdout] + else + File.open(path, 'wb', &writer) + end + end + + LOG_FIX_REGEXP_SEPARATORS = '/!:;|,#%&' + + def format_changelog(path, arg, base_url = nil) + env = {'TZ' => 'JST-9', 'LANG' => 'C', 'LC_ALL' => 'C'} + cmd = %W"#{COMMAND} log --format=fuller --notes=commits --notes=log-fix --topo-order --no-merges" + date = "--date=iso-local" + unless system(env, *cmd, date, chdir: @srcdir, out: NullDevice, exception: false) + date = "--date=iso" + end + cmd << date + cmd.concat(arg) + proc do |w| + w.print "-*- coding: utf-8 -*-\n\n" + w.print "base-url = #{base_url}\n\n" if base_url + cmd_pipe(env, cmd, chdir: @srcdir) do |r| + while s = r.gets("\ncommit ") + h, s = s.split(/^$/, 2) + h.gsub!(/^(?:(?:Author|Commit)(?:Date)?|Date): /, ' \&') + if s.sub!(/\nNotes \(log-fix\):\n((?: +.*\n)+)/, '') + fix = $1 + s = s.lines + fix.each_line do |x| + case x + when %r[^ +(\d+)s([#{LOG_FIX_REGEXP_SEPARATORS}])(.+)\2(.*)\2]o + n = $1.to_i + wrong = $3 + correct = $4 + begin + s[n][wrong] = correct + rescue IndexError + message = ["format_changelog failed to replace #{wrong.dump} with #{correct.dump} at #$1\n"] + from = [1, n-2].max + to = [s.size-1, n+2].min + s.each_with_index do |e, i| + next if i < from + break if to < i + message << "#{i}:#{e}" + end + raise message.join('') + end + when %r[^( +)(\d+)i([#{LOG_FIX_REGEXP_SEPARATORS}])(.*)\3]o + s[$2.to_i, 0] = "#{$1}#{$4}\n" + when %r[^ +(\d+)(?:,(\d+))?d] + n = $1.to_i + e = $2 + s[n..(e ? e.to_i : n)] = [] + end + end + s = s.join('') + end + + if %r[^ +(https://github\.com/[^/]+/[^/]+/)commit/\h+\n(?=(?: +\n(?i: +Co-authored-by: .*\n)+)?(?:\n|\Z))] =~ s + issue = "#{$1}pull/" + s.gsub!(/\b[Ff]ix(?:e[sd])? \K#(?=\d+)/) {issue} + end + + s.gsub!(/ +\n/, "\n") + s.sub!(/^Notes:/, ' \&') + w.print h, s + end + end + end + end + + def format_changelog_as_svn(path, arg) + cmd = %W"#{COMMAND} log --topo-order --no-notes -z --format=%an%n%at%n%B" + cmd.concat(arg) + proc do |w| + sep = "-"*72 + "\n" + w.print sep + cmd_pipe(cmd) do |r| + while s = r.gets("\0") + s.chomp!("\0") + author, time, s = s.split("\n", 3) + s.sub!(/\n\ngit-svn-id: .*@(\d+) .*\n\Z/, '') + rev = $1 + time = Time.at(time.to_i).getlocal("+09:00").strftime("%F %T %z (%a, %d %b %Y)") + lines = s.count("\n") + 1 + lines = "#{lines} line#{lines == 1 ? '' : 's'}" + w.print "r#{rev} | #{author} | #{time} | #{lines}\n\n", s, "\n", sep + end + end + end + end + + def upstream + (branch = cmd_read(%W"#{COMMAND} symbolic-ref --short HEAD")).chomp! + (upstream = cmd_read(%W"#{COMMAND} branch --list --format=%(upstream) #{branch}")).chomp! + while ref = upstream[%r"\Arefs/heads/(.*)", 1] + upstream = cmd_read(%W"#{COMMAND} branch --list --format=%(upstream) #{ref}") + end + unless %r"\Arefs/remotes/([^/]+)/(.*)" =~ upstream + raise "Upstream not found" + end + [$1, $2] + end + + def commit(opts = {}) + args = [COMMAND, "push"] + args << "-n" if dryrun + remote, branch = upstream + args << remote + branches = %W[refs/notes/commits:refs/notes/commits HEAD:#{branch}] + if dryrun? + branches.each do |b| + VCS::DEBUG_OUT.puts((args + [b]).inspect) + end + return true + end + branches.each do |b| + system(*(args + [b])) or return false + end + true + end + end + + class GITSVN < GIT + def self.revision_name(rev) + SVN.revision_name(rev) + end + + def last_changed_revision + rev = cmd_read(%W"#{COMMAND} svn info"+[STDERR=>[:child, :out]])[/^Last Changed Rev: (\d+)/, 1] + com = cmd_read(%W"#{COMMAND} svn find-rev r#{rev}").chomp + return rev, com + end + + def commit(opts = {}) + rev, com = last_changed_revision + head = cmd_read(%W"#{COMMAND} symbolic-ref --short HEAD").chomp + + commits = cmd_read([COMMAND, "log", "--reverse", "--format=%H %ae %ce", "#{com}..@"], "rb").split("\n") + commits.each_with_index do |l, i| + r, a, c = l.split(' ') + dcommit = [COMMAND, "svn", "dcommit"] + dcommit.insert(-2, "-n") if dryrun + dcommit << "--add-author-from" unless a == c + dcommit << r + system(*dcommit) or return false + system(COMMAND, "checkout", head) or return false + system(COMMAND, "rebase") or return false + end + + if rev + old = [cmd_read(%W"#{COMMAND} log -1 --format=%H").chomp] + old << cmd_read(%W"#{COMMAND} svn reset -r#{rev}")[/^r#{rev} = (\h+)/, 1] + 3.times do + sleep 2 + system(*%W"#{COMMAND} pull --no-edit --rebase") + break unless old.include?(cmd_read(%W"#{COMMAND} log -1 --format=%H").chomp) + end + end + true + end + end +end diff --git a/tool/lib/vpath.rb b/tool/lib/vpath.rb new file mode 100644 index 0000000000..48ab148405 --- /dev/null +++ b/tool/lib/vpath.rb @@ -0,0 +1,87 @@ +# -*- coding: us-ascii -*- + +class VPath + attr_accessor :separator + + def initialize(*list) + @list = list + @additional = [] + @separator = nil + end + + def inspect + list.inspect + end + + def search(meth, base, *rest) + begin + meth.call(base, *rest) + rescue Errno::ENOENT => error + list.each do |dir| + return meth.call(File.join(dir, base), *rest) rescue nil + end + raise error + end + end + + def process(*args, &block) + search(File.method(__callee__), *args, &block) + end + + alias stat process + alias lstat process + + def open(*args) + f = search(File.method(:open), *args) + if block_given? + begin + yield f + ensure + f.close unless f.closed? + end + else + f + end + end + + def read(*args) + open(*args) {|f| f.read} + end + + def foreach(file, *args, &block) + open(file) {|f| f.each(*args, &block)} + end + + def def_options(opt) + opt.on("-I", "--srcdir=DIR", "add a directory to search path") {|dir| + @additional << dir + } + opt.on("-L", "--vpath=PATH LIST", "add directories to search path") {|dirs| + @additional << [dirs] + } + opt.on("--path-separator=SEP", /\A(?:\W\z|\.(\W).+)/, "separator for vpath") {|sep, vsep| + # hack for msys make. + @separator = vsep || sep + } + end + + def list + @additional.reject! do |dirs| + case dirs + when String + @list << dirs + when Array + raise "--path-separator option is needed for vpath list" unless @separator + # @separator ||= (require 'rbconfig'; RbConfig::CONFIG["PATH_SEPARATOR"]) + @list.concat(dirs[0].split(@separator)) + end + true + end + @list + end + + def strip(path) + prefix = list.map {|dir| Regexp.quote(dir)} + path.sub(/\A#{prefix.join('|')}(?:\/|\z)/, '') + end +end diff --git a/tool/lib/webrick.rb b/tool/lib/webrick.rb new file mode 100644 index 0000000000..b854b68db4 --- /dev/null +++ b/tool/lib/webrick.rb @@ -0,0 +1,232 @@ +# frozen_string_literal: false +## +# = WEB server toolkit. +# +# WEBrick is an HTTP server toolkit that can be configured as an HTTPS server, +# a proxy server, and a virtual-host server. WEBrick features complete +# logging of both server operations and HTTP access. WEBrick supports both +# basic and digest authentication in addition to algorithms not in RFC 2617. +# +# A WEBrick server can be composed of multiple WEBrick servers or servlets to +# provide differing behavior on a per-host or per-path basis. WEBrick +# includes servlets for handling CGI scripts, ERB pages, Ruby blocks and +# directory listings. +# +# WEBrick also includes tools for daemonizing a process and starting a process +# at a higher privilege level and dropping permissions. +# +# == Security +# +# *Warning:* WEBrick is not recommended for production. It only implements +# basic security checks. +# +# == Starting an HTTP server +# +# To create a new WEBrick::HTTPServer that will listen to connections on port +# 8000 and serve documents from the current user's public_html folder: +# +# require 'webrick' +# +# root = File.expand_path '~/public_html' +# server = WEBrick::HTTPServer.new :Port => 8000, :DocumentRoot => root +# +# To run the server you will need to provide a suitable shutdown hook as +# starting the server blocks the current thread: +# +# trap 'INT' do server.shutdown end +# +# server.start +# +# == Custom Behavior +# +# The easiest way to have a server perform custom operations is through +# WEBrick::HTTPServer#mount_proc. The block given will be called with a +# WEBrick::HTTPRequest with request info and a WEBrick::HTTPResponse which +# must be filled in appropriately: +# +# server.mount_proc '/' do |req, res| +# res.body = 'Hello, world!' +# end +# +# Remember that +server.mount_proc+ must precede +server.start+. +# +# == Servlets +# +# Advanced custom behavior can be obtained through mounting a subclass of +# WEBrick::HTTPServlet::AbstractServlet. Servlets provide more modularity +# when writing an HTTP server than mount_proc allows. Here is a simple +# servlet: +# +# class Simple < WEBrick::HTTPServlet::AbstractServlet +# def do_GET request, response +# status, content_type, body = do_stuff_with request +# +# response.status = 200 +# response['Content-Type'] = 'text/plain' +# response.body = 'Hello, World!' +# end +# end +# +# To initialize the servlet you mount it on the server: +# +# server.mount '/simple', Simple +# +# See WEBrick::HTTPServlet::AbstractServlet for more details. +# +# == Virtual Hosts +# +# A server can act as a virtual host for multiple host names. After creating +# the listening host, additional hosts that do not listen can be created and +# attached as virtual hosts: +# +# server = WEBrick::HTTPServer.new # ... +# +# vhost = WEBrick::HTTPServer.new :ServerName => 'vhost.example', +# :DoNotListen => true, # ... +# vhost.mount '/', ... +# +# server.virtual_host vhost +# +# If no +:DocumentRoot+ is provided and no servlets or procs are mounted on the +# main server it will return 404 for all URLs. +# +# == HTTPS +# +# To create an HTTPS server you only need to enable SSL and provide an SSL +# certificate name: +# +# require 'webrick' +# require 'webrick/https' +# +# cert_name = [ +# %w[CN localhost], +# ] +# +# server = WEBrick::HTTPServer.new(:Port => 8000, +# :SSLEnable => true, +# :SSLCertName => cert_name) +# +# This will start the server with a self-generated self-signed certificate. +# The certificate will be changed every time the server is restarted. +# +# To create a server with a pre-determined key and certificate you can provide +# them: +# +# require 'webrick' +# require 'webrick/https' +# require 'openssl' +# +# cert = OpenSSL::X509::Certificate.new File.read '/path/to/cert.pem' +# pkey = OpenSSL::PKey::RSA.new File.read '/path/to/pkey.pem' +# +# server = WEBrick::HTTPServer.new(:Port => 8000, +# :SSLEnable => true, +# :SSLCertificate => cert, +# :SSLPrivateKey => pkey) +# +# == Proxy Server +# +# WEBrick can act as a proxy server: +# +# require 'webrick' +# require 'webrick/httpproxy' +# +# proxy = WEBrick::HTTPProxyServer.new :Port => 8000 +# +# trap 'INT' do proxy.shutdown end +# +# See WEBrick::HTTPProxy for further details including modifying proxied +# responses. +# +# == Basic and Digest authentication +# +# WEBrick provides both Basic and Digest authentication for regular and proxy +# servers. See WEBrick::HTTPAuth, WEBrick::HTTPAuth::BasicAuth and +# WEBrick::HTTPAuth::DigestAuth. +# +# == WEBrick as a daemonized Web Server +# +# WEBrick can be run as a daemonized server for small loads. +# +# === Daemonizing +# +# To start a WEBrick server as a daemon simple run WEBrick::Daemon.start +# before starting the server. +# +# === Dropping Permissions +# +# WEBrick can be started as one user to gain permission to bind to port 80 or +# 443 for serving HTTP or HTTPS traffic then can drop these permissions for +# regular operation. To listen on all interfaces for HTTP traffic: +# +# sockets = WEBrick::Utils.create_listeners nil, 80 +# +# Then drop privileges: +# +# WEBrick::Utils.su 'www' +# +# Then create a server that does not listen by default: +# +# server = WEBrick::HTTPServer.new :DoNotListen => true, # ... +# +# Then overwrite the listening sockets with the port 80 sockets: +# +# server.listeners.replace sockets +# +# === Logging +# +# WEBrick can separately log server operations and end-user access. For +# server operations: +# +# log_file = File.open '/var/log/webrick.log', 'a+' +# log = WEBrick::Log.new log_file +# +# For user access logging: +# +# access_log = [ +# [log_file, WEBrick::AccessLog::COMBINED_LOG_FORMAT], +# ] +# +# server = WEBrick::HTTPServer.new :Logger => log, :AccessLog => access_log +# +# See WEBrick::AccessLog for further log formats. +# +# === Log Rotation +# +# To rotate logs in WEBrick on a HUP signal (like syslogd can send), open the +# log file in 'a+' mode (as above) and trap 'HUP' to reopen the log file: +# +# trap 'HUP' do log_file.reopen '/path/to/webrick.log', 'a+' +# +# == Copyright +# +# Author: IPR -- Internet Programming with Ruby -- writers +# +# Copyright (c) 2000 TAKAHASHI Masayoshi, GOTOU YUUZOU +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +#-- +# $IPR: webrick.rb,v 1.12 2002/10/01 17:16:31 gotoyuzo Exp $ + +module WEBrick +end + +require 'webrick/compat.rb' + +require 'webrick/version.rb' +require 'webrick/config.rb' +require 'webrick/log.rb' +require 'webrick/server.rb' +require_relative 'webrick/utils.rb' +require 'webrick/accesslog' + +require 'webrick/htmlutils.rb' +require 'webrick/httputils.rb' +require 'webrick/cookie.rb' +require 'webrick/httpversion.rb' +require 'webrick/httpstatus.rb' +require 'webrick/httprequest.rb' +require 'webrick/httpresponse.rb' +require 'webrick/httpserver.rb' +require 'webrick/httpservlet.rb' +require 'webrick/httpauth.rb' diff --git a/tool/lib/webrick/.document b/tool/lib/webrick/.document new file mode 100644 index 0000000000..c62f89083b --- /dev/null +++ b/tool/lib/webrick/.document @@ -0,0 +1,6 @@ +# Add files to this as they become documented + +*.rb + +httpauth +httpservlet diff --git a/tool/lib/webrick/accesslog.rb b/tool/lib/webrick/accesslog.rb new file mode 100644 index 0000000000..e4849637f3 --- /dev/null +++ b/tool/lib/webrick/accesslog.rb @@ -0,0 +1,157 @@ +# frozen_string_literal: false +#-- +# accesslog.rb -- Access log handling utilities +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2002 keita yamaguchi +# Copyright (c) 2002 Internet Programming with Ruby writers +# +# $IPR: accesslog.rb,v 1.1 2002/10/01 17:16:32 gotoyuzo Exp $ + +module WEBrick + + ## + # AccessLog provides logging to various files in various formats. + # + # Multiple logs may be written to at the same time: + # + # access_log = [ + # [$stderr, WEBrick::AccessLog::COMMON_LOG_FORMAT], + # [$stderr, WEBrick::AccessLog::REFERER_LOG_FORMAT], + # ] + # + # server = WEBrick::HTTPServer.new :AccessLog => access_log + # + # Custom log formats may be defined. WEBrick::AccessLog provides a subset + # of the formatting from Apache's mod_log_config + # http://httpd.apache.org/docs/mod/mod_log_config.html#formats. See + # AccessLog::setup_params for a list of supported options + + module AccessLog + + ## + # Raised if a parameter such as %e, %i, %o or %n is used without fetching + # a specific field. + + class AccessLogError < StandardError; end + + ## + # The Common Log Format's time format + + CLF_TIME_FORMAT = "[%d/%b/%Y:%H:%M:%S %Z]" + + ## + # Common Log Format + + COMMON_LOG_FORMAT = "%h %l %u %t \"%r\" %s %b" + + ## + # Short alias for Common Log Format + + CLF = COMMON_LOG_FORMAT + + ## + # Referer Log Format + + REFERER_LOG_FORMAT = "%{Referer}i -> %U" + + ## + # User-Agent Log Format + + AGENT_LOG_FORMAT = "%{User-Agent}i" + + ## + # Combined Log Format + + COMBINED_LOG_FORMAT = "#{CLF} \"%{Referer}i\" \"%{User-agent}i\"" + + module_function + + # This format specification is a subset of mod_log_config of Apache: + # + # %a:: Remote IP address + # %b:: Total response size + # %e{variable}:: Given variable in ENV + # %f:: Response filename + # %h:: Remote host name + # %{header}i:: Given request header + # %l:: Remote logname, always "-" + # %m:: Request method + # %{attr}n:: Given request attribute from <tt>req.attributes</tt> + # %{header}o:: Given response header + # %p:: Server's request port + # %{format}p:: The canonical port of the server serving the request or the + # actual port or the client's actual port. Valid formats are + # canonical, local or remote. + # %q:: Request query string + # %r:: First line of the request + # %s:: Request status + # %t:: Time the request was received + # %T:: Time taken to process the request + # %u:: Remote user from auth + # %U:: Unparsed URI + # %%:: Literal % + + def setup_params(config, req, res) + params = Hash.new("") + params["a"] = req.peeraddr[3] + params["b"] = res.sent_size + params["e"] = ENV + params["f"] = res.filename || "" + params["h"] = req.peeraddr[2] + params["i"] = req + params["l"] = "-" + params["m"] = req.request_method + params["n"] = req.attributes + params["o"] = res + params["p"] = req.port + params["q"] = req.query_string + params["r"] = req.request_line.sub(/\x0d?\x0a\z/o, '') + params["s"] = res.status # won't support "%>s" + params["t"] = req.request_time + params["T"] = Time.now - req.request_time + params["u"] = req.user || "-" + params["U"] = req.unparsed_uri + params["v"] = config[:ServerName] + params + end + + ## + # Formats +params+ according to +format_string+ which is described in + # setup_params. + + def format(format_string, params) + format_string.gsub(/\%(?:\{(.*?)\})?>?([a-zA-Z%])/){ + param, spec = $1, $2 + case spec[0] + when ?e, ?i, ?n, ?o + raise AccessLogError, + "parameter is required for \"#{spec}\"" unless param + (param = params[spec][param]) ? escape(param) : "-" + when ?t + params[spec].strftime(param || CLF_TIME_FORMAT) + when ?p + case param + when 'remote' + escape(params["i"].peeraddr[1].to_s) + else + escape(params["p"].to_s) + end + when ?% + "%" + else + escape(params[spec].to_s) + end + } + end + + ## + # Escapes control characters in +data+ + + def escape(data) + data = data.gsub(/[[:cntrl:]\\]+/) {$&.dump[1...-1]} + data.untaint if RUBY_VERSION < '2.7' + data + end + end +end diff --git a/tool/lib/webrick/cgi.rb b/tool/lib/webrick/cgi.rb new file mode 100644 index 0000000000..bb0ae2fc84 --- /dev/null +++ b/tool/lib/webrick/cgi.rb @@ -0,0 +1,313 @@ +# frozen_string_literal: false +# +# cgi.rb -- Yet another CGI library +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2003 Internet Programming with Ruby writers. All rights +# reserved. +# +# $Id$ + +require_relative "httprequest" +require_relative "httpresponse" +require_relative "config" +require "stringio" + +module WEBrick + + # A CGI library using WEBrick requests and responses. + # + # Example: + # + # class MyCGI < WEBrick::CGI + # def do_GET req, res + # res.body = 'it worked!' + # res.status = 200 + # end + # end + # + # MyCGI.new.start + + class CGI + + # The CGI error exception class + + CGIError = Class.new(StandardError) + + ## + # The CGI configuration. This is based on WEBrick::Config::HTTP + + attr_reader :config + + ## + # The CGI logger + + attr_reader :logger + + ## + # Creates a new CGI interface. + # + # The first argument in +args+ is a configuration hash which would update + # WEBrick::Config::HTTP. + # + # Any remaining arguments are stored in the <code>@options</code> instance + # variable for use by a subclass. + + def initialize(*args) + if defined?(MOD_RUBY) + unless ENV.has_key?("GATEWAY_INTERFACE") + Apache.request.setup_cgi_env + end + end + if %r{HTTP/(\d+\.\d+)} =~ ENV["SERVER_PROTOCOL"] + httpv = $1 + end + @config = WEBrick::Config::HTTP.dup.update( + :ServerSoftware => ENV["SERVER_SOFTWARE"] || "null", + :HTTPVersion => HTTPVersion.new(httpv || "1.0"), + :RunOnCGI => true, # to detect if it runs on CGI. + :NPH => false # set true to run as NPH script. + ) + if config = args.shift + @config.update(config) + end + @config[:Logger] ||= WEBrick::BasicLog.new($stderr) + @logger = @config[:Logger] + @options = args + end + + ## + # Reads +key+ from the configuration + + def [](key) + @config[key] + end + + ## + # Starts the CGI process with the given environment +env+ and standard + # input and output +stdin+ and +stdout+. + + def start(env=ENV, stdin=$stdin, stdout=$stdout) + sock = WEBrick::CGI::Socket.new(@config, env, stdin, stdout) + req = HTTPRequest.new(@config) + res = HTTPResponse.new(@config) + unless @config[:NPH] or defined?(MOD_RUBY) + def res.setup_header + unless @header["status"] + phrase = HTTPStatus::reason_phrase(@status) + @header["status"] = "#{@status} #{phrase}" + end + super + end + def res.status_line + "" + end + end + + begin + req.parse(sock) + req.script_name = (env["SCRIPT_NAME"] || File.expand_path($0)).dup + req.path_info = (env["PATH_INFO"] || "").dup + req.query_string = env["QUERY_STRING"] + req.user = env["REMOTE_USER"] + res.request_method = req.request_method + res.request_uri = req.request_uri + res.request_http_version = req.http_version + res.keep_alive = req.keep_alive? + self.service(req, res) + rescue HTTPStatus::Error => ex + res.set_error(ex) + rescue HTTPStatus::Status => ex + res.status = ex.code + rescue Exception => ex + @logger.error(ex) + res.set_error(ex, true) + ensure + req.fixup + if defined?(MOD_RUBY) + res.setup_header + Apache.request.status_line = "#{res.status} #{res.reason_phrase}" + Apache.request.status = res.status + table = Apache.request.headers_out + res.header.each{|key, val| + case key + when /^content-encoding$/i + Apache::request.content_encoding = val + when /^content-type$/i + Apache::request.content_type = val + else + table[key] = val.to_s + end + } + res.cookies.each{|cookie| + table.add("Set-Cookie", cookie.to_s) + } + Apache.request.send_http_header + res.send_body(sock) + else + res.send_response(sock) + end + end + end + + ## + # Services the request +req+ which will fill in the response +res+. See + # WEBrick::HTTPServlet::AbstractServlet#service for details. + + def service(req, res) + method_name = "do_" + req.request_method.gsub(/-/, "_") + if respond_to?(method_name) + __send__(method_name, req, res) + else + raise HTTPStatus::MethodNotAllowed, + "unsupported method `#{req.request_method}'." + end + end + + ## + # Provides HTTP socket emulation from the CGI environment + + class Socket # :nodoc: + include Enumerable + + private + + def initialize(config, env, stdin, stdout) + @config = config + @env = env + @header_part = StringIO.new + @body_part = stdin + @out_port = stdout + @out_port.binmode + + @server_addr = @env["SERVER_ADDR"] || "0.0.0.0" + @server_name = @env["SERVER_NAME"] + @server_port = @env["SERVER_PORT"] + @remote_addr = @env["REMOTE_ADDR"] + @remote_host = @env["REMOTE_HOST"] || @remote_addr + @remote_port = @env["REMOTE_PORT"] || 0 + + begin + @header_part << request_line << CRLF + setup_header + @header_part << CRLF + @header_part.rewind + rescue Exception + raise CGIError, "invalid CGI environment" + end + end + + def request_line + meth = @env["REQUEST_METHOD"] || "GET" + unless url = @env["REQUEST_URI"] + url = (@env["SCRIPT_NAME"] || File.expand_path($0)).dup + url << @env["PATH_INFO"].to_s + url = WEBrick::HTTPUtils.escape_path(url) + if query_string = @env["QUERY_STRING"] + unless query_string.empty? + url << "?" << query_string + end + end + end + # we cannot get real HTTP version of client ;) + httpv = @config[:HTTPVersion] + return "#{meth} #{url} HTTP/#{httpv}" + end + + def setup_header + @env.each{|key, value| + case key + when "CONTENT_TYPE", "CONTENT_LENGTH" + add_header(key.gsub(/_/, "-"), value) + when /^HTTP_(.*)/ + add_header($1.gsub(/_/, "-"), value) + end + } + end + + def add_header(hdrname, value) + unless value.empty? + @header_part << hdrname << ": " << value << CRLF + end + end + + def input + @header_part.eof? ? @body_part : @header_part + end + + public + + def peeraddr + [nil, @remote_port, @remote_host, @remote_addr] + end + + def addr + [nil, @server_port, @server_name, @server_addr] + end + + def gets(eol=LF, size=nil) + input.gets(eol, size) + end + + def read(size=nil) + input.read(size) + end + + def each + input.each{|line| yield(line) } + end + + def eof? + input.eof? + end + + def <<(data) + @out_port << data + end + + def write(data) + @out_port.write(data) + end + + def cert + return nil unless defined?(OpenSSL) + if pem = @env["SSL_SERVER_CERT"] + OpenSSL::X509::Certificate.new(pem) unless pem.empty? + end + end + + def peer_cert + return nil unless defined?(OpenSSL) + if pem = @env["SSL_CLIENT_CERT"] + OpenSSL::X509::Certificate.new(pem) unless pem.empty? + end + end + + def peer_cert_chain + return nil unless defined?(OpenSSL) + if @env["SSL_CLIENT_CERT_CHAIN_0"] + keys = @env.keys + certs = keys.sort.collect{|k| + if /^SSL_CLIENT_CERT_CHAIN_\d+$/ =~ k + if pem = @env[k] + OpenSSL::X509::Certificate.new(pem) unless pem.empty? + end + end + } + certs.compact + end + end + + def cipher + return nil unless defined?(OpenSSL) + if cipher = @env["SSL_CIPHER"] + ret = [ cipher ] + ret << @env["SSL_PROTOCOL"] + ret << @env["SSL_CIPHER_USEKEYSIZE"] + ret << @env["SSL_CIPHER_ALGKEYSIZE"] + ret + end + end + end + end +end diff --git a/tool/lib/webrick/compat.rb b/tool/lib/webrick/compat.rb new file mode 100644 index 0000000000..c497a1933c --- /dev/null +++ b/tool/lib/webrick/compat.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: false +# +# compat.rb -- cross platform compatibility +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2002 GOTOU Yuuzou +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: compat.rb,v 1.6 2002/10/01 17:16:32 gotoyuzo Exp $ + +## +# System call error module used by webrick for cross platform compatibility. +# +# EPROTO:: protocol error +# ECONNRESET:: remote host reset the connection request +# ECONNABORTED:: Client sent TCP reset (RST) before server has accepted the +# connection requested by client. +# +module Errno + ## + # Protocol error. + + class EPROTO < SystemCallError; end + + ## + # Remote host reset the connection request. + + class ECONNRESET < SystemCallError; end + + ## + # Client sent TCP reset (RST) before server has accepted the connection + # requested by client. + + class ECONNABORTED < SystemCallError; end +end diff --git a/tool/lib/webrick/config.rb b/tool/lib/webrick/config.rb new file mode 100644 index 0000000000..9f2ab44f49 --- /dev/null +++ b/tool/lib/webrick/config.rb @@ -0,0 +1,158 @@ +# frozen_string_literal: false +# +# config.rb -- Default configurations. +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou +# Copyright (c) 2003 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: config.rb,v 1.52 2003/07/22 19:20:42 gotoyuzo Exp $ + +require_relative 'version' +require_relative 'httpversion' +require_relative 'httputils' +require_relative 'utils' +require_relative 'log' + +module WEBrick + module Config + LIBDIR = File::dirname(__FILE__) # :nodoc: + + # for GenericServer + General = Hash.new { |hash, key| + case key + when :ServerName + hash[key] = Utils.getservername + else + nil + end + }.update( + :BindAddress => nil, # "0.0.0.0" or "::" or nil + :Port => nil, # users MUST specify this!! + :MaxClients => 100, # maximum number of the concurrent connections + :ServerType => nil, # default: WEBrick::SimpleServer + :Logger => nil, # default: WEBrick::Log.new + :ServerSoftware => "WEBrick/#{WEBrick::VERSION} " + + "(Ruby/#{RUBY_VERSION}/#{RUBY_RELEASE_DATE})", + :TempDir => ENV['TMPDIR']||ENV['TMP']||ENV['TEMP']||'/tmp', + :DoNotListen => false, + :StartCallback => nil, + :StopCallback => nil, + :AcceptCallback => nil, + :DoNotReverseLookup => true, + :ShutdownSocketWithoutClose => false, + ) + + # for HTTPServer, HTTPRequest, HTTPResponse ... + HTTP = General.dup.update( + :Port => 80, + :RequestTimeout => 30, + :HTTPVersion => HTTPVersion.new("1.1"), + :AccessLog => nil, + :MimeTypes => HTTPUtils::DefaultMimeTypes, + :DirectoryIndex => ["index.html","index.htm","index.cgi","index.rhtml"], + :DocumentRoot => nil, + :DocumentRootOptions => { :FancyIndexing => true }, + :RequestCallback => nil, + :ServerAlias => nil, + :InputBufferSize => 65536, # input buffer size in reading request body + :OutputBufferSize => 65536, # output buffer size in sending File or IO + + # for HTTPProxyServer + :ProxyAuthProc => nil, + :ProxyContentHandler => nil, + :ProxyVia => true, + :ProxyTimeout => true, + :ProxyURI => nil, + + :CGIInterpreter => nil, + :CGIPathEnv => nil, + + # workaround: if Request-URIs contain 8bit chars, + # they should be escaped before calling of URI::parse(). + :Escape8bitURI => false + ) + + ## + # Default configuration for WEBrick::HTTPServlet::FileHandler + # + # :AcceptableLanguages:: + # Array of languages allowed for accept-language. There is no default + # :DirectoryCallback:: + # Allows preprocessing of directory requests. There is no default + # callback. + # :FancyIndexing:: + # If true, show an index for directories. The default is true. + # :FileCallback:: + # Allows preprocessing of file requests. There is no default callback. + # :HandlerCallback:: + # Allows preprocessing of requests. There is no default callback. + # :HandlerTable:: + # Maps file suffixes to file handlers. DefaultFileHandler is used by + # default but any servlet can be used. + # :NondisclosureName:: + # Do not show files matching this array of globs. .ht* and *~ are + # excluded by default. + # :UserDir:: + # Directory inside ~user to serve content from for /~user requests. + # Only works if mounted on /. Disabled by default. + + FileHandler = { + :NondisclosureName => [".ht*", "*~"], + :FancyIndexing => false, + :HandlerTable => {}, + :HandlerCallback => nil, + :DirectoryCallback => nil, + :FileCallback => nil, + :UserDir => nil, # e.g. "public_html" + :AcceptableLanguages => [] # ["en", "ja", ... ] + } + + ## + # Default configuration for WEBrick::HTTPAuth::BasicAuth + # + # :AutoReloadUserDB:: Reload the user database provided by :UserDB + # automatically? + + BasicAuth = { + :AutoReloadUserDB => true, + } + + ## + # Default configuration for WEBrick::HTTPAuth::DigestAuth. + # + # :Algorithm:: MD5, MD5-sess (default), SHA1, SHA1-sess + # :Domain:: An Array of URIs that define the protected space + # :Qop:: 'auth' for authentication, 'auth-int' for integrity protection or + # both + # :UseOpaque:: Should the server send opaque values to the client? This + # helps prevent replay attacks. + # :CheckNc:: Should the server check the nonce count? This helps the + # server detect replay attacks. + # :UseAuthenticationInfoHeader:: Should the server send an + # AuthenticationInfo header? + # :AutoReloadUserDB:: Reload the user database provided by :UserDB + # automatically? + # :NonceExpirePeriod:: How long should we store used nonces? Default is + # 30 minutes. + # :NonceExpireDelta:: How long is a nonce valid? Default is 1 minute + # :InternetExplorerHack:: Hack which allows Internet Explorer to work. + # :OperaHack:: Hack which allows Opera to work. + + DigestAuth = { + :Algorithm => 'MD5-sess', # or 'MD5' + :Domain => nil, # an array includes domain names. + :Qop => [ 'auth' ], # 'auth' or 'auth-int' or both. + :UseOpaque => true, + :UseNextNonce => false, + :CheckNc => false, + :UseAuthenticationInfoHeader => true, + :AutoReloadUserDB => true, + :NonceExpirePeriod => 30*60, + :NonceExpireDelta => 60, + :InternetExplorerHack => true, + :OperaHack => true, + } + end +end diff --git a/tool/lib/webrick/cookie.rb b/tool/lib/webrick/cookie.rb new file mode 100644 index 0000000000..5fd3bfb228 --- /dev/null +++ b/tool/lib/webrick/cookie.rb @@ -0,0 +1,172 @@ +# frozen_string_literal: false +# +# cookie.rb -- Cookie class +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: cookie.rb,v 1.16 2002/09/21 12:23:35 gotoyuzo Exp $ + +require 'time' +require_relative 'httputils' + +module WEBrick + + ## + # Processes HTTP cookies + + class Cookie + + ## + # The cookie name + + attr_reader :name + + ## + # The cookie value + + attr_accessor :value + + ## + # The cookie version + + attr_accessor :version + + ## + # The cookie domain + attr_accessor :domain + + ## + # The cookie path + + attr_accessor :path + + ## + # Is this a secure cookie? + + attr_accessor :secure + + ## + # The cookie comment + + attr_accessor :comment + + ## + # The maximum age of the cookie + + attr_accessor :max_age + + #attr_accessor :comment_url, :discard, :port + + ## + # Creates a new cookie with the given +name+ and +value+ + + def initialize(name, value) + @name = name + @value = value + @version = 0 # Netscape Cookie + + @domain = @path = @secure = @comment = @max_age = + @expires = @comment_url = @discard = @port = nil + end + + ## + # Sets the cookie expiration to the time +t+. The expiration time may be + # a false value to disable expiration or a Time or HTTP format time string + # to set the expiration date. + + def expires=(t) + @expires = t && (t.is_a?(Time) ? t.httpdate : t.to_s) + end + + ## + # Retrieves the expiration time as a Time + + def expires + @expires && Time.parse(@expires) + end + + ## + # The cookie string suitable for use in an HTTP header + + def to_s + ret = "" + ret << @name << "=" << @value + ret << "; " << "Version=" << @version.to_s if @version > 0 + ret << "; " << "Domain=" << @domain if @domain + ret << "; " << "Expires=" << @expires if @expires + ret << "; " << "Max-Age=" << @max_age.to_s if @max_age + ret << "; " << "Comment=" << @comment if @comment + ret << "; " << "Path=" << @path if @path + ret << "; " << "Secure" if @secure + ret + end + + ## + # Parses a Cookie field sent from the user-agent. Returns an array of + # cookies. + + def self.parse(str) + if str + ret = [] + cookie = nil + ver = 0 + str.split(/;\s+/).each{|x| + key, val = x.split(/=/,2) + val = val ? HTTPUtils::dequote(val) : "" + case key + when "$Version"; ver = val.to_i + when "$Path"; cookie.path = val + when "$Domain"; cookie.domain = val + when "$Port"; cookie.port = val + else + ret << cookie if cookie + cookie = self.new(key, val) + cookie.version = ver + end + } + ret << cookie if cookie + ret + end + end + + ## + # Parses the cookie in +str+ + + def self.parse_set_cookie(str) + cookie_elem = str.split(/;/) + first_elem = cookie_elem.shift + first_elem.strip! + key, value = first_elem.split(/=/, 2) + cookie = new(key, HTTPUtils.dequote(value)) + cookie_elem.each{|pair| + pair.strip! + key, value = pair.split(/=/, 2) + if value + value = HTTPUtils.dequote(value.strip) + end + case key.downcase + when "domain" then cookie.domain = value + when "path" then cookie.path = value + when "expires" then cookie.expires = value + when "max-age" then cookie.max_age = Integer(value) + when "comment" then cookie.comment = value + when "version" then cookie.version = Integer(value) + when "secure" then cookie.secure = true + end + } + return cookie + end + + ## + # Parses the cookies in +str+ + + def self.parse_set_cookies(str) + return str.split(/,(?=[^;,]*=)|,$/).collect{|c| + parse_set_cookie(c) + } + end + end +end diff --git a/tool/lib/webrick/htmlutils.rb b/tool/lib/webrick/htmlutils.rb new file mode 100644 index 0000000000..ed9f4ac0d3 --- /dev/null +++ b/tool/lib/webrick/htmlutils.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: false +#-- +# htmlutils.rb -- HTMLUtils Module +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: htmlutils.rb,v 1.7 2002/09/21 12:23:35 gotoyuzo Exp $ + +module WEBrick + module HTMLUtils + + ## + # Escapes &, ", > and < in +string+ + + def escape(string) + return "" unless string + str = string.b + str.gsub!(/&/n, '&') + str.gsub!(/\"/n, '"') + str.gsub!(/>/n, '>') + str.gsub!(/</n, '<') + str.force_encoding(string.encoding) + end + module_function :escape + + end +end diff --git a/tool/lib/webrick/httpauth.rb b/tool/lib/webrick/httpauth.rb new file mode 100644 index 0000000000..f8bf09a6f1 --- /dev/null +++ b/tool/lib/webrick/httpauth.rb @@ -0,0 +1,96 @@ +# frozen_string_literal: false +# +# httpauth.rb -- HTTP access authentication +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: httpauth.rb,v 1.14 2003/07/22 19:20:42 gotoyuzo Exp $ + +require_relative 'httpauth/basicauth' +require_relative 'httpauth/digestauth' +require_relative 'httpauth/htpasswd' +require_relative 'httpauth/htdigest' +require_relative 'httpauth/htgroup' + +module WEBrick + + ## + # HTTPAuth provides both basic and digest authentication. + # + # To enable authentication for requests in WEBrick you will need a user + # database and an authenticator. To start, here's an Htpasswd database for + # use with a DigestAuth authenticator: + # + # config = { :Realm => 'DigestAuth example realm' } + # + # htpasswd = WEBrick::HTTPAuth::Htpasswd.new 'my_password_file' + # htpasswd.auth_type = WEBrick::HTTPAuth::DigestAuth + # htpasswd.set_passwd config[:Realm], 'username', 'password' + # htpasswd.flush + # + # The +:Realm+ is used to provide different access to different groups + # across several resources on a server. Typically you'll need only one + # realm for a server. + # + # This database can be used to create an authenticator: + # + # config[:UserDB] = htpasswd + # + # digest_auth = WEBrick::HTTPAuth::DigestAuth.new config + # + # To authenticate a request call #authenticate with a request and response + # object in a servlet: + # + # def do_GET req, res + # @authenticator.authenticate req, res + # end + # + # For digest authentication the authenticator must not be created every + # request, it must be passed in as an option via WEBrick::HTTPServer#mount. + + module HTTPAuth + module_function + + def _basic_auth(req, res, realm, req_field, res_field, err_type, + block) # :nodoc: + user = pass = nil + if /^Basic\s+(.*)/o =~ req[req_field] + userpass = $1 + user, pass = userpass.unpack("m*")[0].split(":", 2) + end + if block.call(user, pass) + req.user = user + return + end + res[res_field] = "Basic realm=\"#{realm}\"" + raise err_type + end + + ## + # Simple wrapper for providing basic authentication for a request. When + # called with a request +req+, response +res+, authentication +realm+ and + # +block+ the block will be called with a +username+ and +password+. If + # the block returns true the request is allowed to continue, otherwise an + # HTTPStatus::Unauthorized error is raised. + + def basic_auth(req, res, realm, &block) # :yield: username, password + _basic_auth(req, res, realm, "Authorization", "WWW-Authenticate", + HTTPStatus::Unauthorized, block) + end + + ## + # Simple wrapper for providing basic authentication for a proxied request. + # When called with a request +req+, response +res+, authentication +realm+ + # and +block+ the block will be called with a +username+ and +password+. + # If the block returns true the request is allowed to continue, otherwise + # an HTTPStatus::ProxyAuthenticationRequired error is raised. + + def proxy_basic_auth(req, res, realm, &block) # :yield: username, password + _basic_auth(req, res, realm, "Proxy-Authorization", "Proxy-Authenticate", + HTTPStatus::ProxyAuthenticationRequired, block) + end + end +end diff --git a/tool/lib/webrick/httpauth/authenticator.rb b/tool/lib/webrick/httpauth/authenticator.rb new file mode 100644 index 0000000000..8f0eaa3aca --- /dev/null +++ b/tool/lib/webrick/httpauth/authenticator.rb @@ -0,0 +1,117 @@ +# frozen_string_literal: false +#-- +# httpauth/authenticator.rb -- Authenticator mix-in module. +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2003 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: authenticator.rb,v 1.3 2003/02/20 07:15:47 gotoyuzo Exp $ + +module WEBrick + module HTTPAuth + + ## + # Module providing generic support for both Digest and Basic + # authentication schemes. + + module Authenticator + + RequestField = "Authorization" # :nodoc: + ResponseField = "WWW-Authenticate" # :nodoc: + ResponseInfoField = "Authentication-Info" # :nodoc: + AuthException = HTTPStatus::Unauthorized # :nodoc: + + ## + # Method of authentication, must be overridden by the including class + + AuthScheme = nil + + ## + # The realm this authenticator covers + + attr_reader :realm + + ## + # The user database for this authenticator + + attr_reader :userdb + + ## + # The logger for this authenticator + + attr_reader :logger + + private + + # :stopdoc: + + ## + # Initializes the authenticator from +config+ + + def check_init(config) + [:UserDB, :Realm].each{|sym| + unless config[sym] + raise ArgumentError, "Argument #{sym.inspect} missing." + end + } + @realm = config[:Realm] + @userdb = config[:UserDB] + @logger = config[:Logger] || Log::new($stderr) + @reload_db = config[:AutoReloadUserDB] + @request_field = self::class::RequestField + @response_field = self::class::ResponseField + @resp_info_field = self::class::ResponseInfoField + @auth_exception = self::class::AuthException + @auth_scheme = self::class::AuthScheme + end + + ## + # Ensures +req+ has credentials that can be authenticated. + + def check_scheme(req) + unless credentials = req[@request_field] + error("no credentials in the request.") + return nil + end + unless match = /^#{@auth_scheme}\s+/i.match(credentials) + error("invalid scheme in %s.", credentials) + info("%s: %s", @request_field, credentials) if $DEBUG + return nil + end + return match.post_match + end + + def log(meth, fmt, *args) + msg = format("%s %s: ", @auth_scheme, @realm) + msg << fmt % args + @logger.__send__(meth, msg) + end + + def error(fmt, *args) + if @logger.error? + log(:error, fmt, *args) + end + end + + def info(fmt, *args) + if @logger.info? + log(:info, fmt, *args) + end + end + + # :startdoc: + end + + ## + # Module providing generic support for both Digest and Basic + # authentication schemes for proxies. + + module ProxyAuthenticator + RequestField = "Proxy-Authorization" # :nodoc: + ResponseField = "Proxy-Authenticate" # :nodoc: + InfoField = "Proxy-Authentication-Info" # :nodoc: + AuthException = HTTPStatus::ProxyAuthenticationRequired # :nodoc: + end + end +end diff --git a/tool/lib/webrick/httpauth/basicauth.rb b/tool/lib/webrick/httpauth/basicauth.rb new file mode 100644 index 0000000000..7d0a9cfc8f --- /dev/null +++ b/tool/lib/webrick/httpauth/basicauth.rb @@ -0,0 +1,116 @@ +# frozen_string_literal: false +# +# httpauth/basicauth.rb -- HTTP basic access authentication +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2003 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: basicauth.rb,v 1.5 2003/02/20 07:15:47 gotoyuzo Exp $ + +require_relative '../config' +require_relative '../httpstatus' +require_relative 'authenticator' + +module WEBrick + module HTTPAuth + + ## + # Basic Authentication for WEBrick + # + # Use this class to add basic authentication to a WEBrick servlet. + # + # Here is an example of how to set up a BasicAuth: + # + # config = { :Realm => 'BasicAuth example realm' } + # + # htpasswd = WEBrick::HTTPAuth::Htpasswd.new 'my_password_file', password_hash: :bcrypt + # htpasswd.set_passwd config[:Realm], 'username', 'password' + # htpasswd.flush + # + # config[:UserDB] = htpasswd + # + # basic_auth = WEBrick::HTTPAuth::BasicAuth.new config + + class BasicAuth + include Authenticator + + AuthScheme = "Basic" # :nodoc: + + ## + # Used by UserDB to create a basic password entry + + def self.make_passwd(realm, user, pass) + pass ||= "" + pass.crypt(Utils::random_string(2)) + end + + attr_reader :realm, :userdb, :logger + + ## + # Creates a new BasicAuth instance. + # + # See WEBrick::Config::BasicAuth for default configuration entries + # + # You must supply the following configuration entries: + # + # :Realm:: The name of the realm being protected. + # :UserDB:: A database of usernames and passwords. + # A WEBrick::HTTPAuth::Htpasswd instance should be used. + + def initialize(config, default=Config::BasicAuth) + check_init(config) + @config = default.dup.update(config) + end + + ## + # Authenticates a +req+ and returns a 401 Unauthorized using +res+ if + # the authentication was not correct. + + def authenticate(req, res) + unless basic_credentials = check_scheme(req) + challenge(req, res) + end + userid, password = basic_credentials.unpack("m*")[0].split(":", 2) + password ||= "" + if userid.empty? + error("user id was not given.") + challenge(req, res) + end + unless encpass = @userdb.get_passwd(@realm, userid, @reload_db) + error("%s: the user is not allowed.", userid) + challenge(req, res) + end + + case encpass + when /\A\$2[aby]\$/ + password_matches = BCrypt::Password.new(encpass.sub(/\A\$2[aby]\$/, '$2a$')) == password + else + password_matches = password.crypt(encpass) == encpass + end + + unless password_matches + error("%s: password unmatch.", userid) + challenge(req, res) + end + info("%s: authentication succeeded.", userid) + req.user = userid + end + + ## + # Returns a challenge response which asks for authentication information + + def challenge(req, res) + res[@response_field] = "#{@auth_scheme} realm=\"#{@realm}\"" + raise @auth_exception + end + end + + ## + # Basic authentication for proxy servers. See BasicAuth for details. + + class ProxyBasicAuth < BasicAuth + include ProxyAuthenticator + end + end +end diff --git a/tool/lib/webrick/httpauth/digestauth.rb b/tool/lib/webrick/httpauth/digestauth.rb new file mode 100644 index 0000000000..3cf12899d2 --- /dev/null +++ b/tool/lib/webrick/httpauth/digestauth.rb @@ -0,0 +1,395 @@ +# frozen_string_literal: false +# +# httpauth/digestauth.rb -- HTTP digest access authentication +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2003 Internet Programming with Ruby writers. +# Copyright (c) 2003 H.M. +# +# The original implementation is provided by H.M. +# URL: http://rwiki.jin.gr.jp/cgi-bin/rw-cgi.rb?cmd=view;name= +# %C7%A7%BE%DA%B5%A1%C7%BD%A4%F2%B2%FE%C2%A4%A4%B7%A4%C6%A4%DF%A4%EB +# +# $IPR: digestauth.rb,v 1.5 2003/02/20 07:15:47 gotoyuzo Exp $ + +require_relative '../config' +require_relative '../httpstatus' +require_relative 'authenticator' +require 'digest/md5' +require 'digest/sha1' + +module WEBrick + module HTTPAuth + + ## + # RFC 2617 Digest Access Authentication for WEBrick + # + # Use this class to add digest authentication to a WEBrick servlet. + # + # Here is an example of how to set up DigestAuth: + # + # config = { :Realm => 'DigestAuth example realm' } + # + # htdigest = WEBrick::HTTPAuth::Htdigest.new 'my_password_file' + # htdigest.set_passwd config[:Realm], 'username', 'password' + # htdigest.flush + # + # config[:UserDB] = htdigest + # + # digest_auth = WEBrick::HTTPAuth::DigestAuth.new config + # + # When using this as with a servlet be sure not to create a new DigestAuth + # object in the servlet's #initialize. By default WEBrick creates a new + # servlet instance for every request and the DigestAuth object must be + # used across requests. + + class DigestAuth + include Authenticator + + AuthScheme = "Digest" # :nodoc: + + ## + # Struct containing the opaque portion of the digest authentication + + OpaqueInfo = Struct.new(:time, :nonce, :nc) # :nodoc: + + ## + # Digest authentication algorithm + + attr_reader :algorithm + + ## + # Quality of protection. RFC 2617 defines "auth" and "auth-int" + + attr_reader :qop + + ## + # Used by UserDB to create a digest password entry + + def self.make_passwd(realm, user, pass) + pass ||= "" + Digest::MD5::hexdigest([user, realm, pass].join(":")) + end + + ## + # Creates a new DigestAuth instance. Be sure to use the same DigestAuth + # instance for multiple requests as it saves state between requests in + # order to perform authentication. + # + # See WEBrick::Config::DigestAuth for default configuration entries + # + # You must supply the following configuration entries: + # + # :Realm:: The name of the realm being protected. + # :UserDB:: A database of usernames and passwords. + # A WEBrick::HTTPAuth::Htdigest instance should be used. + + def initialize(config, default=Config::DigestAuth) + check_init(config) + @config = default.dup.update(config) + @algorithm = @config[:Algorithm] + @domain = @config[:Domain] + @qop = @config[:Qop] + @use_opaque = @config[:UseOpaque] + @use_next_nonce = @config[:UseNextNonce] + @check_nc = @config[:CheckNc] + @use_auth_info_header = @config[:UseAuthenticationInfoHeader] + @nonce_expire_period = @config[:NonceExpirePeriod] + @nonce_expire_delta = @config[:NonceExpireDelta] + @internet_explorer_hack = @config[:InternetExplorerHack] + + case @algorithm + when 'MD5','MD5-sess' + @h = Digest::MD5 + when 'SHA1','SHA1-sess' # it is a bonus feature :-) + @h = Digest::SHA1 + else + msg = format('Algorithm "%s" is not supported.', @algorithm) + raise ArgumentError.new(msg) + end + + @instance_key = hexdigest(self.__id__, Time.now.to_i, Process.pid) + @opaques = {} + @last_nonce_expire = Time.now + @mutex = Thread::Mutex.new + end + + ## + # Authenticates a +req+ and returns a 401 Unauthorized using +res+ if + # the authentication was not correct. + + def authenticate(req, res) + unless result = @mutex.synchronize{ _authenticate(req, res) } + challenge(req, res) + end + if result == :nonce_is_stale + challenge(req, res, true) + end + return true + end + + ## + # Returns a challenge response which asks for authentication information + + def challenge(req, res, stale=false) + nonce = generate_next_nonce(req) + if @use_opaque + opaque = generate_opaque(req) + @opaques[opaque].nonce = nonce + end + + param = Hash.new + param["realm"] = HTTPUtils::quote(@realm) + param["domain"] = HTTPUtils::quote(@domain.to_a.join(" ")) if @domain + param["nonce"] = HTTPUtils::quote(nonce) + param["opaque"] = HTTPUtils::quote(opaque) if opaque + param["stale"] = stale.to_s + param["algorithm"] = @algorithm + param["qop"] = HTTPUtils::quote(@qop.to_a.join(",")) if @qop + + res[@response_field] = + "#{@auth_scheme} " + param.map{|k,v| "#{k}=#{v}" }.join(", ") + info("%s: %s", @response_field, res[@response_field]) if $DEBUG + raise @auth_exception + end + + private + + # :stopdoc: + + MustParams = ['username','realm','nonce','uri','response'] + MustParamsAuth = ['cnonce','nc'] + + def _authenticate(req, res) + unless digest_credentials = check_scheme(req) + return false + end + + auth_req = split_param_value(digest_credentials) + if auth_req['qop'] == "auth" || auth_req['qop'] == "auth-int" + req_params = MustParams + MustParamsAuth + else + req_params = MustParams + end + req_params.each{|key| + unless auth_req.has_key?(key) + error('%s: parameter missing. "%s"', auth_req['username'], key) + raise HTTPStatus::BadRequest + end + } + + if !check_uri(req, auth_req) + raise HTTPStatus::BadRequest + end + + if auth_req['realm'] != @realm + error('%s: realm unmatch. "%s" for "%s"', + auth_req['username'], auth_req['realm'], @realm) + return false + end + + auth_req['algorithm'] ||= 'MD5' + if auth_req['algorithm'].upcase != @algorithm.upcase + error('%s: algorithm unmatch. "%s" for "%s"', + auth_req['username'], auth_req['algorithm'], @algorithm) + return false + end + + if (@qop.nil? && auth_req.has_key?('qop')) || + (@qop && (! @qop.member?(auth_req['qop']))) + error('%s: the qop is not allowed. "%s"', + auth_req['username'], auth_req['qop']) + return false + end + + password = @userdb.get_passwd(@realm, auth_req['username'], @reload_db) + unless password + error('%s: the user is not allowed.', auth_req['username']) + return false + end + + nonce_is_invalid = false + if @use_opaque + info("@opaque = %s", @opaque.inspect) if $DEBUG + if !(opaque = auth_req['opaque']) + error('%s: opaque is not given.', auth_req['username']) + nonce_is_invalid = true + elsif !(opaque_struct = @opaques[opaque]) + error('%s: invalid opaque is given.', auth_req['username']) + nonce_is_invalid = true + elsif !check_opaque(opaque_struct, req, auth_req) + @opaques.delete(auth_req['opaque']) + nonce_is_invalid = true + end + elsif !check_nonce(req, auth_req) + nonce_is_invalid = true + end + + if /-sess$/i =~ auth_req['algorithm'] + ha1 = hexdigest(password, auth_req['nonce'], auth_req['cnonce']) + else + ha1 = password + end + + if auth_req['qop'] == "auth" || auth_req['qop'] == nil + ha2 = hexdigest(req.request_method, auth_req['uri']) + ha2_res = hexdigest("", auth_req['uri']) + elsif auth_req['qop'] == "auth-int" + body_digest = @h.new + req.body { |chunk| body_digest.update(chunk) } + body_digest = body_digest.hexdigest + ha2 = hexdigest(req.request_method, auth_req['uri'], body_digest) + ha2_res = hexdigest("", auth_req['uri'], body_digest) + end + + if auth_req['qop'] == "auth" || auth_req['qop'] == "auth-int" + param2 = ['nonce', 'nc', 'cnonce', 'qop'].map{|key| + auth_req[key] + }.join(':') + digest = hexdigest(ha1, param2, ha2) + digest_res = hexdigest(ha1, param2, ha2_res) + else + digest = hexdigest(ha1, auth_req['nonce'], ha2) + digest_res = hexdigest(ha1, auth_req['nonce'], ha2_res) + end + + if digest != auth_req['response'] + error("%s: digest unmatch.", auth_req['username']) + return false + elsif nonce_is_invalid + error('%s: digest is valid, but nonce is not valid.', + auth_req['username']) + return :nonce_is_stale + elsif @use_auth_info_header + auth_info = { + 'nextnonce' => generate_next_nonce(req), + 'rspauth' => digest_res + } + if @use_opaque + opaque_struct.time = req.request_time + opaque_struct.nonce = auth_info['nextnonce'] + opaque_struct.nc = "%08x" % (auth_req['nc'].hex + 1) + end + if auth_req['qop'] == "auth" || auth_req['qop'] == "auth-int" + ['qop','cnonce','nc'].each{|key| + auth_info[key] = auth_req[key] + } + end + res[@resp_info_field] = auth_info.keys.map{|key| + if key == 'nc' + key + '=' + auth_info[key] + else + key + "=" + HTTPUtils::quote(auth_info[key]) + end + }.join(', ') + end + info('%s: authentication succeeded.', auth_req['username']) + req.user = auth_req['username'] + return true + end + + def split_param_value(string) + ret = {} + string.scan(/\G\s*([\w\-.*%!]+)=\s*(?:\"((?>\\.|[^\"])*)\"|([^,\"]*))\s*,?/) do + ret[$1] = $3 || $2.gsub(/\\(.)/, "\\1") + end + ret + end + + def generate_next_nonce(req) + now = "%012d" % req.request_time.to_i + pk = hexdigest(now, @instance_key)[0,32] + nonce = [now + ":" + pk].pack("m0") # it has 60 length of chars. + nonce + end + + def check_nonce(req, auth_req) + username = auth_req['username'] + nonce = auth_req['nonce'] + + pub_time, pk = nonce.unpack("m*")[0].split(":", 2) + if (!pub_time || !pk) + error("%s: empty nonce is given", username) + return false + elsif (hexdigest(pub_time, @instance_key)[0,32] != pk) + error("%s: invalid private-key: %s for %s", + username, hexdigest(pub_time, @instance_key)[0,32], pk) + return false + end + + diff_time = req.request_time.to_i - pub_time.to_i + if (diff_time < 0) + error("%s: difference of time-stamp is negative.", username) + return false + elsif diff_time > @nonce_expire_period + error("%s: nonce is expired.", username) + return false + end + + return true + end + + def generate_opaque(req) + @mutex.synchronize{ + now = req.request_time + if now - @last_nonce_expire > @nonce_expire_delta + @opaques.delete_if{|key,val| + (now - val.time) > @nonce_expire_period + } + @last_nonce_expire = now + end + begin + opaque = Utils::random_string(16) + end while @opaques[opaque] + @opaques[opaque] = OpaqueInfo.new(now, nil, '00000001') + opaque + } + end + + def check_opaque(opaque_struct, req, auth_req) + if (@use_next_nonce && auth_req['nonce'] != opaque_struct.nonce) + error('%s: nonce unmatched. "%s" for "%s"', + auth_req['username'], auth_req['nonce'], opaque_struct.nonce) + return false + elsif !check_nonce(req, auth_req) + return false + end + if (@check_nc && auth_req['nc'] != opaque_struct.nc) + error('%s: nc unmatched."%s" for "%s"', + auth_req['username'], auth_req['nc'], opaque_struct.nc) + return false + end + true + end + + def check_uri(req, auth_req) + uri = auth_req['uri'] + if uri != req.request_uri.to_s && uri != req.unparsed_uri && + (@internet_explorer_hack && uri != req.path) + error('%s: uri unmatch. "%s" for "%s"', auth_req['username'], + auth_req['uri'], req.request_uri.to_s) + return false + end + true + end + + def hexdigest(*args) + @h.hexdigest(args.join(":")) + end + + # :startdoc: + end + + ## + # Digest authentication for proxy servers. See DigestAuth for details. + + class ProxyDigestAuth < DigestAuth + include ProxyAuthenticator + + private + def check_uri(req, auth_req) # :nodoc: + return true + end + end + end +end diff --git a/tool/lib/webrick/httpauth/htdigest.rb b/tool/lib/webrick/httpauth/htdigest.rb new file mode 100644 index 0000000000..93b18e2c75 --- /dev/null +++ b/tool/lib/webrick/httpauth/htdigest.rb @@ -0,0 +1,132 @@ +# frozen_string_literal: false +# +# httpauth/htdigest.rb -- Apache compatible htdigest file +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2003 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: htdigest.rb,v 1.4 2003/07/22 19:20:45 gotoyuzo Exp $ + +require_relative 'userdb' +require_relative 'digestauth' +require 'tempfile' + +module WEBrick + module HTTPAuth + + ## + # Htdigest accesses apache-compatible digest password files. Passwords are + # matched to a realm where they are valid. For security, the path for a + # digest password database should be stored outside of the paths available + # to the HTTP server. + # + # Htdigest is intended for use with WEBrick::HTTPAuth::DigestAuth and + # stores passwords using cryptographic hashes. + # + # htpasswd = WEBrick::HTTPAuth::Htdigest.new 'my_password_file' + # htpasswd.set_passwd 'my realm', 'username', 'password' + # htpasswd.flush + + class Htdigest + include UserDB + + ## + # Open a digest password database at +path+ + + def initialize(path) + @path = path + @mtime = Time.at(0) + @digest = Hash.new + @mutex = Thread::Mutex::new + @auth_type = DigestAuth + File.open(@path,"a").close unless File.exist?(@path) + reload + end + + ## + # Reloads passwords from the database + + def reload + mtime = File::mtime(@path) + if mtime > @mtime + @digest.clear + File.open(@path){|io| + while line = io.gets + line.chomp! + user, realm, pass = line.split(/:/, 3) + unless @digest[realm] + @digest[realm] = Hash.new + end + @digest[realm][user] = pass + end + } + @mtime = mtime + end + end + + ## + # Flush the password database. If +output+ is given the database will + # be written there instead of to the original path. + + def flush(output=nil) + output ||= @path + tmp = Tempfile.create("htpasswd", File::dirname(output)) + renamed = false + begin + each{|item| tmp.puts(item.join(":")) } + tmp.close + File::rename(tmp.path, output) + renamed = true + ensure + tmp.close + File.unlink(tmp.path) if !renamed + end + end + + ## + # Retrieves a password from the database for +user+ in +realm+. If + # +reload_db+ is true the database will be reloaded first. + + def get_passwd(realm, user, reload_db) + reload() if reload_db + if hash = @digest[realm] + hash[user] + end + end + + ## + # Sets a password in the database for +user+ in +realm+ to +pass+. + + def set_passwd(realm, user, pass) + @mutex.synchronize{ + unless @digest[realm] + @digest[realm] = Hash.new + end + @digest[realm][user] = make_passwd(realm, user, pass) + } + end + + ## + # Removes a password from the database for +user+ in +realm+. + + def delete_passwd(realm, user) + if hash = @digest[realm] + hash.delete(user) + end + end + + ## + # Iterate passwords in the database. + + def each # :yields: [user, realm, password_hash] + @digest.keys.sort.each{|realm| + hash = @digest[realm] + hash.keys.sort.each{|user| + yield([user, realm, hash[user]]) + } + } + end + end + end +end diff --git a/tool/lib/webrick/httpauth/htgroup.rb b/tool/lib/webrick/httpauth/htgroup.rb new file mode 100644 index 0000000000..e06c441b18 --- /dev/null +++ b/tool/lib/webrick/httpauth/htgroup.rb @@ -0,0 +1,97 @@ +# frozen_string_literal: false +# +# httpauth/htgroup.rb -- Apache compatible htgroup file +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2003 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: htgroup.rb,v 1.1 2003/02/16 22:22:56 gotoyuzo Exp $ + +require 'tempfile' + +module WEBrick + module HTTPAuth + + ## + # Htgroup accesses apache-compatible group files. Htgroup can be used to + # provide group-based authentication for users. Currently Htgroup is not + # directly integrated with any authenticators in WEBrick. For security, + # the path for a digest password database should be stored outside of the + # paths available to the HTTP server. + # + # Example: + # + # htgroup = WEBrick::HTTPAuth::Htgroup.new 'my_group_file' + # htgroup.add 'superheroes', %w[spiderman batman] + # + # htgroup.members('superheroes').include? 'magneto' # => false + + class Htgroup + + ## + # Open a group database at +path+ + + def initialize(path) + @path = path + @mtime = Time.at(0) + @group = Hash.new + File.open(@path,"a").close unless File.exist?(@path) + reload + end + + ## + # Reload groups from the database + + def reload + if (mtime = File::mtime(@path)) > @mtime + @group.clear + File.open(@path){|io| + while line = io.gets + line.chomp! + group, members = line.split(/:\s*/) + @group[group] = members.split(/\s+/) + end + } + @mtime = mtime + end + end + + ## + # Flush the group database. If +output+ is given the database will be + # written there instead of to the original path. + + def flush(output=nil) + output ||= @path + tmp = Tempfile.create("htgroup", File::dirname(output)) + begin + @group.keys.sort.each{|group| + tmp.puts(format("%s: %s", group, self.members(group).join(" "))) + } + ensure + tmp.close + if $! + File.unlink(tmp.path) + else + return File.rename(tmp.path, output) + end + end + end + + ## + # Retrieve the list of members from +group+ + + def members(group) + reload + @group[group] || [] + end + + ## + # Add an Array of +members+ to +group+ + + def add(group, members) + @group[group] = members(group) | members + end + end + end +end diff --git a/tool/lib/webrick/httpauth/htpasswd.rb b/tool/lib/webrick/httpauth/htpasswd.rb new file mode 100644 index 0000000000..abca30532e --- /dev/null +++ b/tool/lib/webrick/httpauth/htpasswd.rb @@ -0,0 +1,158 @@ +# frozen_string_literal: false +# +# httpauth/htpasswd -- Apache compatible htpasswd file +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2003 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: htpasswd.rb,v 1.4 2003/07/22 19:20:45 gotoyuzo Exp $ + +require_relative 'userdb' +require_relative 'basicauth' +require 'tempfile' + +module WEBrick + module HTTPAuth + + ## + # Htpasswd accesses apache-compatible password files. Passwords are + # matched to a realm where they are valid. For security, the path for a + # password database should be stored outside of the paths available to the + # HTTP server. + # + # Htpasswd is intended for use with WEBrick::HTTPAuth::BasicAuth. + # + # To create an Htpasswd database with a single user: + # + # htpasswd = WEBrick::HTTPAuth::Htpasswd.new 'my_password_file' + # htpasswd.set_passwd 'my realm', 'username', 'password' + # htpasswd.flush + + class Htpasswd + include UserDB + + ## + # Open a password database at +path+ + + def initialize(path, password_hash: nil) + @path = path + @mtime = Time.at(0) + @passwd = Hash.new + @auth_type = BasicAuth + @password_hash = password_hash + + case @password_hash + when nil + # begin + # require "string/crypt" + # rescue LoadError + # warn("Unable to load string/crypt, proceeding with deprecated use of String#crypt, consider using password_hash: :bcrypt") + # end + @password_hash = :crypt + when :crypt + # require "string/crypt" + when :bcrypt + require "bcrypt" + else + raise ArgumentError, "only :crypt and :bcrypt are supported for password_hash keyword argument" + end + + File.open(@path,"a").close unless File.exist?(@path) + reload + end + + ## + # Reload passwords from the database + + def reload + mtime = File::mtime(@path) + if mtime > @mtime + @passwd.clear + File.open(@path){|io| + while line = io.gets + line.chomp! + case line + when %r!\A[^:]+:[a-zA-Z0-9./]{13}\z! + if @password_hash == :bcrypt + raise StandardError, ".htpasswd file contains crypt password, only bcrypt passwords supported" + end + user, pass = line.split(":") + when %r!\A[^:]+:\$2[aby]\$\d{2}\$.{53}\z! + if @password_hash == :crypt + raise StandardError, ".htpasswd file contains bcrypt password, only crypt passwords supported" + end + user, pass = line.split(":") + when /:\$/, /:{SHA}/ + raise NotImplementedError, + 'MD5, SHA1 .htpasswd file not supported' + else + raise StandardError, 'bad .htpasswd file' + end + @passwd[user] = pass + end + } + @mtime = mtime + end + end + + ## + # Flush the password database. If +output+ is given the database will + # be written there instead of to the original path. + + def flush(output=nil) + output ||= @path + tmp = Tempfile.create("htpasswd", File::dirname(output)) + renamed = false + begin + each{|item| tmp.puts(item.join(":")) } + tmp.close + File::rename(tmp.path, output) + renamed = true + ensure + tmp.close + File.unlink(tmp.path) if !renamed + end + end + + ## + # Retrieves a password from the database for +user+ in +realm+. If + # +reload_db+ is true the database will be reloaded first. + + def get_passwd(realm, user, reload_db) + reload() if reload_db + @passwd[user] + end + + ## + # Sets a password in the database for +user+ in +realm+ to +pass+. + + def set_passwd(realm, user, pass) + if @password_hash == :bcrypt + # Cost of 5 to match Apache default, and because the + # bcrypt default of 10 will introduce significant delays + # for every request. + @passwd[user] = BCrypt::Password.create(pass, :cost=>5) + else + @passwd[user] = make_passwd(realm, user, pass) + end + end + + ## + # Removes a password from the database for +user+ in +realm+. + + def delete_passwd(realm, user) + @passwd.delete(user) + end + + ## + # Iterate passwords in the database. + + def each # :yields: [user, password] + @passwd.keys.sort.each{|user| + yield([user, @passwd[user]]) + } + end + end + end +end diff --git a/tool/lib/webrick/httpauth/userdb.rb b/tool/lib/webrick/httpauth/userdb.rb new file mode 100644 index 0000000000..7a17715cdf --- /dev/null +++ b/tool/lib/webrick/httpauth/userdb.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: false +#-- +# httpauth/userdb.rb -- UserDB mix-in module. +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2003 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: userdb.rb,v 1.2 2003/02/20 07:15:48 gotoyuzo Exp $ + +module WEBrick + module HTTPAuth + + ## + # User database mixin for HTTPAuth. This mixin dispatches user record + # access to the underlying auth_type for this database. + + module UserDB + + ## + # The authentication type. + # + # WEBrick::HTTPAuth::BasicAuth or WEBrick::HTTPAuth::DigestAuth are + # built-in. + + attr_accessor :auth_type + + ## + # Creates an obscured password in +realm+ with +user+ and +password+ + # using the auth_type of this database. + + def make_passwd(realm, user, pass) + @auth_type::make_passwd(realm, user, pass) + end + + ## + # Sets a password in +realm+ with +user+ and +password+ for the + # auth_type of this database. + + def set_passwd(realm, user, pass) + self[user] = pass + end + + ## + # Retrieves a password in +realm+ for +user+ for the auth_type of this + # database. +reload_db+ is a dummy value. + + def get_passwd(realm, user, reload_db=false) + make_passwd(realm, user, self[user]) + end + end + end +end diff --git a/tool/lib/webrick/httpproxy.rb b/tool/lib/webrick/httpproxy.rb new file mode 100644 index 0000000000..7607c3df88 --- /dev/null +++ b/tool/lib/webrick/httpproxy.rb @@ -0,0 +1,354 @@ +# frozen_string_literal: false +# +# httpproxy.rb -- HTTPProxy Class +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2002 GOTO Kentaro +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: httpproxy.rb,v 1.18 2003/03/08 18:58:10 gotoyuzo Exp $ +# $kNotwork: straw.rb,v 1.3 2002/02/12 15:13:07 gotoken Exp $ + +require_relative "httpserver" +require "net/http" + +module WEBrick + + NullReader = Object.new # :nodoc: + class << NullReader # :nodoc: + def read(*args) + nil + end + alias gets read + end + + FakeProxyURI = Object.new # :nodoc: + class << FakeProxyURI # :nodoc: + def method_missing(meth, *args) + if %w(scheme host port path query userinfo).member?(meth.to_s) + return nil + end + super + end + end + + # :startdoc: + + ## + # An HTTP Proxy server which proxies GET, HEAD and POST requests. + # + # To create a simple proxy server: + # + # require 'webrick' + # require 'webrick/httpproxy' + # + # proxy = WEBrick::HTTPProxyServer.new Port: 8000 + # + # trap 'INT' do proxy.shutdown end + # trap 'TERM' do proxy.shutdown end + # + # proxy.start + # + # See ::new for proxy-specific configuration items. + # + # == Modifying proxied responses + # + # To modify content the proxy server returns use the +:ProxyContentHandler+ + # option: + # + # handler = proc do |req, res| + # if res['content-type'] == 'text/plain' then + # res.body << "\nThis content was proxied!\n" + # end + # end + # + # proxy = + # WEBrick::HTTPProxyServer.new Port: 8000, ProxyContentHandler: handler + + class HTTPProxyServer < HTTPServer + + ## + # Proxy server configurations. The proxy server handles the following + # configuration items in addition to those supported by HTTPServer: + # + # :ProxyAuthProc:: Called with a request and response to authorize a + # request + # :ProxyVia:: Appended to the via header + # :ProxyURI:: The proxy server's URI + # :ProxyContentHandler:: Called with a request and response and allows + # modification of the response + # :ProxyTimeout:: Sets the proxy timeouts to 30 seconds for open and 60 + # seconds for read operations + + def initialize(config={}, default=Config::HTTP) + super(config, default) + c = @config + @via = "#{c[:HTTPVersion]} #{c[:ServerName]}:#{c[:Port]}" + end + + # :stopdoc: + def service(req, res) + if req.request_method == "CONNECT" + do_CONNECT(req, res) + elsif req.unparsed_uri =~ %r!^http://! + proxy_service(req, res) + else + super(req, res) + end + end + + def proxy_auth(req, res) + if proc = @config[:ProxyAuthProc] + proc.call(req, res) + end + req.header.delete("proxy-authorization") + end + + def proxy_uri(req, res) + # should return upstream proxy server's URI + return @config[:ProxyURI] + end + + def proxy_service(req, res) + # Proxy Authentication + proxy_auth(req, res) + + begin + public_send("do_#{req.request_method}", req, res) + rescue NoMethodError + raise HTTPStatus::MethodNotAllowed, + "unsupported method `#{req.request_method}'." + rescue => err + logger.debug("#{err.class}: #{err.message}") + raise HTTPStatus::ServiceUnavailable, err.message + end + + # Process contents + if handler = @config[:ProxyContentHandler] + handler.call(req, res) + end + end + + def do_CONNECT(req, res) + # Proxy Authentication + proxy_auth(req, res) + + ua = Thread.current[:WEBrickSocket] # User-Agent + raise HTTPStatus::InternalServerError, + "[BUG] cannot get socket" unless ua + + host, port = req.unparsed_uri.split(":", 2) + # Proxy authentication for upstream proxy server + if proxy = proxy_uri(req, res) + proxy_request_line = "CONNECT #{host}:#{port} HTTP/1.0" + if proxy.userinfo + credentials = "Basic " + [proxy.userinfo].pack("m0") + end + host, port = proxy.host, proxy.port + end + + begin + @logger.debug("CONNECT: upstream proxy is `#{host}:#{port}'.") + os = TCPSocket.new(host, port) # origin server + + if proxy + @logger.debug("CONNECT: sending a Request-Line") + os << proxy_request_line << CRLF + @logger.debug("CONNECT: > #{proxy_request_line}") + if credentials + @logger.debug("CONNECT: sending credentials") + os << "Proxy-Authorization: " << credentials << CRLF + end + os << CRLF + proxy_status_line = os.gets(LF) + @logger.debug("CONNECT: read Status-Line from the upstream server") + @logger.debug("CONNECT: < #{proxy_status_line}") + if %r{^HTTP/\d+\.\d+\s+200\s*} =~ proxy_status_line + while line = os.gets(LF) + break if /\A(#{CRLF}|#{LF})\z/om =~ line + end + else + raise HTTPStatus::BadGateway + end + end + @logger.debug("CONNECT #{host}:#{port}: succeeded") + res.status = HTTPStatus::RC_OK + rescue => ex + @logger.debug("CONNECT #{host}:#{port}: failed `#{ex.message}'") + res.set_error(ex) + raise HTTPStatus::EOFError + ensure + if handler = @config[:ProxyContentHandler] + handler.call(req, res) + end + res.send_response(ua) + access_log(@config, req, res) + + # Should clear request-line not to send the response twice. + # see: HTTPServer#run + req.parse(NullReader) rescue nil + end + + begin + while fds = IO::select([ua, os]) + if fds[0].member?(ua) + buf = ua.readpartial(1024); + @logger.debug("CONNECT: #{buf.bytesize} byte from User-Agent") + os.write(buf) + elsif fds[0].member?(os) + buf = os.readpartial(1024); + @logger.debug("CONNECT: #{buf.bytesize} byte from #{host}:#{port}") + ua.write(buf) + end + end + rescue + os.close + @logger.debug("CONNECT #{host}:#{port}: closed") + end + + raise HTTPStatus::EOFError + end + + def do_GET(req, res) + perform_proxy_request(req, res, Net::HTTP::Get) + end + + def do_HEAD(req, res) + perform_proxy_request(req, res, Net::HTTP::Head) + end + + def do_POST(req, res) + perform_proxy_request(req, res, Net::HTTP::Post, req.body_reader) + end + + def do_OPTIONS(req, res) + res['allow'] = "GET,HEAD,POST,OPTIONS,CONNECT" + end + + private + + # Some header fields should not be transferred. + HopByHop = %w( connection keep-alive proxy-authenticate upgrade + proxy-authorization te trailers transfer-encoding ) + ShouldNotTransfer = %w( set-cookie proxy-connection ) + def split_field(f) f ? f.split(/,\s+/).collect{|i| i.downcase } : [] end + + def choose_header(src, dst) + connections = split_field(src['connection']) + src.each{|key, value| + key = key.downcase + if HopByHop.member?(key) || # RFC2616: 13.5.1 + connections.member?(key) || # RFC2616: 14.10 + ShouldNotTransfer.member?(key) # pragmatics + @logger.debug("choose_header: `#{key}: #{value}'") + next + end + dst[key] = value + } + end + + # Net::HTTP is stupid about the multiple header fields. + # Here is workaround: + def set_cookie(src, dst) + if str = src['set-cookie'] + cookies = [] + str.split(/,\s*/).each{|token| + if /^[^=]+;/o =~ token + cookies[-1] << ", " << token + elsif /=/o =~ token + cookies << token + else + cookies[-1] << ", " << token + end + } + dst.cookies.replace(cookies) + end + end + + def set_via(h) + if @config[:ProxyVia] + if h['via'] + h['via'] << ", " << @via + else + h['via'] = @via + end + end + end + + def setup_proxy_header(req, res) + # Choose header fields to transfer + header = Hash.new + choose_header(req, header) + set_via(header) + return header + end + + def setup_upstream_proxy_authentication(req, res, header) + if upstream = proxy_uri(req, res) + if upstream.userinfo + header['proxy-authorization'] = + "Basic " + [upstream.userinfo].pack("m0") + end + return upstream + end + return FakeProxyURI + end + + def create_net_http(uri, upstream) + Net::HTTP.new(uri.host, uri.port, upstream.host, upstream.port) + end + + def perform_proxy_request(req, res, req_class, body_stream = nil) + uri = req.request_uri + path = uri.path.dup + path << "?" << uri.query if uri.query + header = setup_proxy_header(req, res) + upstream = setup_upstream_proxy_authentication(req, res, header) + + body_tmp = [] + http = create_net_http(uri, upstream) + req_fib = Fiber.new do + http.start do + if @config[:ProxyTimeout] + ################################## these issues are + http.open_timeout = 30 # secs # necessary (maybe because + http.read_timeout = 60 # secs # Ruby's bug, but why?) + ################################## + end + if body_stream && req['transfer-encoding'] =~ /\bchunked\b/i + header['Transfer-Encoding'] = 'chunked' + end + http_req = req_class.new(path, header) + http_req.body_stream = body_stream if body_stream + http.request(http_req) do |response| + # Persistent connection requirements are mysterious for me. + # So I will close the connection in every response. + res['proxy-connection'] = "close" + res['connection'] = "close" + + # stream Net::HTTP::HTTPResponse to WEBrick::HTTPResponse + res.status = response.code.to_i + res.chunked = response.chunked? + choose_header(response, res) + set_cookie(response, res) + set_via(res) + response.read_body do |buf| + body_tmp << buf + Fiber.yield # wait for res.body Proc#call + end + end # http.request + end + end + req_fib.resume # read HTTP response headers and first chunk of the body + res.body = ->(socket) do + while buf = body_tmp.shift + socket.write(buf) + buf.clear + req_fib.resume # continue response.read_body + end + end + end + # :stopdoc: + end +end diff --git a/tool/lib/webrick/httprequest.rb b/tool/lib/webrick/httprequest.rb new file mode 100644 index 0000000000..d34eac7ecf --- /dev/null +++ b/tool/lib/webrick/httprequest.rb @@ -0,0 +1,636 @@ +# frozen_string_literal: false +# +# httprequest.rb -- HTTPRequest Class +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: httprequest.rb,v 1.64 2003/07/13 17:18:22 gotoyuzo Exp $ + +require 'fiber' +require 'uri' +require_relative 'httpversion' +require_relative 'httpstatus' +require_relative 'httputils' +require_relative 'cookie' + +module WEBrick + + ## + # An HTTP request. This is consumed by service and do_* methods in + # WEBrick servlets + + class HTTPRequest + + BODY_CONTAINABLE_METHODS = [ "POST", "PUT" ] # :nodoc: + + # :section: Request line + + ## + # The complete request line such as: + # + # GET / HTTP/1.1 + + attr_reader :request_line + + ## + # The request method, GET, POST, PUT, etc. + + attr_reader :request_method + + ## + # The unparsed URI of the request + + attr_reader :unparsed_uri + + ## + # The HTTP version of the request + + attr_reader :http_version + + # :section: Request-URI + + ## + # The parsed URI of the request + + attr_reader :request_uri + + ## + # The request path + + attr_reader :path + + ## + # The script name (CGI variable) + + attr_accessor :script_name + + ## + # The path info (CGI variable) + + attr_accessor :path_info + + ## + # The query from the URI of the request + + attr_accessor :query_string + + # :section: Header and entity body + + ## + # The raw header of the request + + attr_reader :raw_header + + ## + # The parsed header of the request + + attr_reader :header + + ## + # The parsed request cookies + + attr_reader :cookies + + ## + # The Accept header value + + attr_reader :accept + + ## + # The Accept-Charset header value + + attr_reader :accept_charset + + ## + # The Accept-Encoding header value + + attr_reader :accept_encoding + + ## + # The Accept-Language header value + + attr_reader :accept_language + + # :section: + + ## + # The remote user (CGI variable) + + attr_accessor :user + + ## + # The socket address of the server + + attr_reader :addr + + ## + # The socket address of the client + + attr_reader :peeraddr + + ## + # Hash of request attributes + + attr_reader :attributes + + ## + # Is this a keep-alive connection? + + attr_reader :keep_alive + + ## + # The local time this request was received + + attr_reader :request_time + + ## + # Creates a new HTTP request. WEBrick::Config::HTTP is the default + # configuration. + + def initialize(config) + @config = config + @buffer_size = @config[:InputBufferSize] + @logger = config[:Logger] + + @request_line = @request_method = + @unparsed_uri = @http_version = nil + + @request_uri = @host = @port = @path = nil + @script_name = @path_info = nil + @query_string = nil + @query = nil + @form_data = nil + + @raw_header = Array.new + @header = nil + @cookies = [] + @accept = [] + @accept_charset = [] + @accept_encoding = [] + @accept_language = [] + @body = "" + + @addr = @peeraddr = nil + @attributes = {} + @user = nil + @keep_alive = false + @request_time = nil + + @remaining_size = nil + @socket = nil + + @forwarded_proto = @forwarded_host = @forwarded_port = + @forwarded_server = @forwarded_for = nil + end + + ## + # Parses a request from +socket+. This is called internally by + # WEBrick::HTTPServer. + + def parse(socket=nil) + @socket = socket + begin + @peeraddr = socket.respond_to?(:peeraddr) ? socket.peeraddr : [] + @addr = socket.respond_to?(:addr) ? socket.addr : [] + rescue Errno::ENOTCONN + raise HTTPStatus::EOFError + end + + read_request_line(socket) + if @http_version.major > 0 + read_header(socket) + @header['cookie'].each{|cookie| + @cookies += Cookie::parse(cookie) + } + @accept = HTTPUtils.parse_qvalues(self['accept']) + @accept_charset = HTTPUtils.parse_qvalues(self['accept-charset']) + @accept_encoding = HTTPUtils.parse_qvalues(self['accept-encoding']) + @accept_language = HTTPUtils.parse_qvalues(self['accept-language']) + end + return if @request_method == "CONNECT" + return if @unparsed_uri == "*" + + begin + setup_forwarded_info + @request_uri = parse_uri(@unparsed_uri) + @path = HTTPUtils::unescape(@request_uri.path) + @path = HTTPUtils::normalize_path(@path) + @host = @request_uri.host + @port = @request_uri.port + @query_string = @request_uri.query + @script_name = "" + @path_info = @path.dup + rescue + raise HTTPStatus::BadRequest, "bad URI `#{@unparsed_uri}'." + end + + if /\Aclose\z/io =~ self["connection"] + @keep_alive = false + elsif /\Akeep-alive\z/io =~ self["connection"] + @keep_alive = true + elsif @http_version < "1.1" + @keep_alive = false + else + @keep_alive = true + end + end + + ## + # Generate HTTP/1.1 100 continue response if the client expects it, + # otherwise does nothing. + + def continue # :nodoc: + if self['expect'] == '100-continue' && @config[:HTTPVersion] >= "1.1" + @socket << "HTTP/#{@config[:HTTPVersion]} 100 continue#{CRLF}#{CRLF}" + @header.delete('expect') + end + end + + ## + # Returns the request body. + + def body(&block) # :yields: body_chunk + block ||= Proc.new{|chunk| @body << chunk } + read_body(@socket, block) + @body.empty? ? nil : @body + end + + ## + # Prepares the HTTPRequest object for use as the + # source for IO.copy_stream + + def body_reader + @body_tmp = [] + @body_rd = Fiber.new do + body do |buf| + @body_tmp << buf + Fiber.yield + end + end + @body_rd.resume # grab the first chunk and yield + self + end + + # for IO.copy_stream. + def readpartial(size, buf = ''.b) # :nodoc + res = @body_tmp.shift or raise EOFError, 'end of file reached' + if res.length > size + @body_tmp.unshift(res[size..-1]) + res = res[0..size - 1] + end + buf.replace(res) + res.clear + # get more chunks - check alive? because we can take a partial chunk + @body_rd.resume if @body_rd.alive? + buf + end + + ## + # Request query as a Hash + + def query + unless @query + parse_query() + end + @query + end + + ## + # The content-length header + + def content_length + return Integer(self['content-length']) + end + + ## + # The content-type header + + def content_type + return self['content-type'] + end + + ## + # Retrieves +header_name+ + + def [](header_name) + if @header + value = @header[header_name.downcase] + value.empty? ? nil : value.join(", ") + end + end + + ## + # Iterates over the request headers + + def each + if @header + @header.each{|k, v| + value = @header[k] + yield(k, value.empty? ? nil : value.join(", ")) + } + end + end + + ## + # The host this request is for + + def host + return @forwarded_host || @host + end + + ## + # The port this request is for + + def port + return @forwarded_port || @port + end + + ## + # The server name this request is for + + def server_name + return @forwarded_server || @config[:ServerName] + end + + ## + # The client's IP address + + def remote_ip + return self["client-ip"] || @forwarded_for || @peeraddr[3] + end + + ## + # Is this an SSL request? + + def ssl? + return @request_uri.scheme == "https" + end + + ## + # Should the connection this request was made on be kept alive? + + def keep_alive? + @keep_alive + end + + def to_s # :nodoc: + ret = @request_line.dup + @raw_header.each{|line| ret << line } + ret << CRLF + ret << body if body + ret + end + + ## + # Consumes any remaining body and updates keep-alive status + + def fixup() # :nodoc: + begin + body{|chunk| } # read remaining body + rescue HTTPStatus::Error => ex + @logger.error("HTTPRequest#fixup: #{ex.class} occurred.") + @keep_alive = false + rescue => ex + @logger.error(ex) + @keep_alive = false + end + end + + # This method provides the metavariables defined by the revision 3 + # of "The WWW Common Gateway Interface Version 1.1" + # To browse the current document of CGI Version 1.1, see below: + # http://tools.ietf.org/html/rfc3875 + + def meta_vars + meta = Hash.new + + cl = self["Content-Length"] + ct = self["Content-Type"] + meta["CONTENT_LENGTH"] = cl if cl.to_i > 0 + meta["CONTENT_TYPE"] = ct.dup if ct + meta["GATEWAY_INTERFACE"] = "CGI/1.1" + meta["PATH_INFO"] = @path_info ? @path_info.dup : "" + #meta["PATH_TRANSLATED"] = nil # no plan to be provided + meta["QUERY_STRING"] = @query_string ? @query_string.dup : "" + meta["REMOTE_ADDR"] = @peeraddr[3] + meta["REMOTE_HOST"] = @peeraddr[2] + #meta["REMOTE_IDENT"] = nil # no plan to be provided + meta["REMOTE_USER"] = @user + meta["REQUEST_METHOD"] = @request_method.dup + meta["REQUEST_URI"] = @request_uri.to_s + meta["SCRIPT_NAME"] = @script_name.dup + meta["SERVER_NAME"] = @host + meta["SERVER_PORT"] = @port.to_s + meta["SERVER_PROTOCOL"] = "HTTP/" + @config[:HTTPVersion].to_s + meta["SERVER_SOFTWARE"] = @config[:ServerSoftware].dup + + self.each{|key, val| + next if /^content-type$/i =~ key + next if /^content-length$/i =~ key + name = "HTTP_" + key + name.gsub!(/-/o, "_") + name.upcase! + meta[name] = val + } + + meta + end + + private + + # :stopdoc: + + MAX_URI_LENGTH = 2083 # :nodoc: + + # same as Mongrel, Thin and Puma + MAX_HEADER_LENGTH = (112 * 1024) # :nodoc: + + def read_request_line(socket) + @request_line = read_line(socket, MAX_URI_LENGTH) if socket + raise HTTPStatus::EOFError unless @request_line + + @request_bytes = @request_line.bytesize + if @request_bytes >= MAX_URI_LENGTH and @request_line[-1, 1] != LF + raise HTTPStatus::RequestURITooLarge + end + + @request_time = Time.now + if /^(\S+)\s+(\S++)(?:\s+HTTP\/(\d+\.\d+))?\r?\n/mo =~ @request_line + @request_method = $1 + @unparsed_uri = $2 + @http_version = HTTPVersion.new($3 ? $3 : "0.9") + else + rl = @request_line.sub(/\x0d?\x0a\z/o, '') + raise HTTPStatus::BadRequest, "bad Request-Line `#{rl}'." + end + end + + def read_header(socket) + if socket + while line = read_line(socket) + break if /\A(#{CRLF}|#{LF})\z/om =~ line + if (@request_bytes += line.bytesize) > MAX_HEADER_LENGTH + raise HTTPStatus::RequestEntityTooLarge, 'headers too large' + end + @raw_header << line + end + end + @header = HTTPUtils::parse_header(@raw_header.join) + end + + def parse_uri(str, scheme="http") + if @config[:Escape8bitURI] + str = HTTPUtils::escape8bit(str) + end + str.sub!(%r{\A/+}o, '/') + uri = URI::parse(str) + return uri if uri.absolute? + if @forwarded_host + host, port = @forwarded_host, @forwarded_port + elsif self["host"] + pattern = /\A(#{URI::REGEXP::PATTERN::HOST})(?::(\d+))?\z/n + host, port = *self['host'].scan(pattern)[0] + elsif @addr.size > 0 + host, port = @addr[2], @addr[1] + else + host, port = @config[:ServerName], @config[:Port] + end + uri.scheme = @forwarded_proto || scheme + uri.host = host + uri.port = port ? port.to_i : nil + return URI::parse(uri.to_s) + end + + def read_body(socket, block) + return unless socket + if tc = self['transfer-encoding'] + case tc + when /\Achunked\z/io then read_chunked(socket, block) + else raise HTTPStatus::NotImplemented, "Transfer-Encoding: #{tc}." + end + elsif self['content-length'] || @remaining_size + @remaining_size ||= self['content-length'].to_i + while @remaining_size > 0 + sz = [@buffer_size, @remaining_size].min + break unless buf = read_data(socket, sz) + @remaining_size -= buf.bytesize + block.call(buf) + end + if @remaining_size > 0 && @socket.eof? + raise HTTPStatus::BadRequest, "invalid body size." + end + elsif BODY_CONTAINABLE_METHODS.member?(@request_method) && !@socket.eof + raise HTTPStatus::LengthRequired + end + return @body + end + + def read_chunk_size(socket) + line = read_line(socket) + if /^([0-9a-fA-F]+)(?:;(\S+))?/ =~ line + chunk_size = $1.hex + chunk_ext = $2 + [ chunk_size, chunk_ext ] + else + raise HTTPStatus::BadRequest, "bad chunk `#{line}'." + end + end + + def read_chunked(socket, block) + chunk_size, = read_chunk_size(socket) + while chunk_size > 0 + begin + sz = [ chunk_size, @buffer_size ].min + data = read_data(socket, sz) # read chunk-data + if data.nil? || data.bytesize != sz + raise HTTPStatus::BadRequest, "bad chunk data size." + end + block.call(data) + end while (chunk_size -= sz) > 0 + + read_line(socket) # skip CRLF + chunk_size, = read_chunk_size(socket) + end + read_header(socket) # trailer + CRLF + @header.delete("transfer-encoding") + @remaining_size = 0 + end + + def _read_data(io, method, *arg) + begin + WEBrick::Utils.timeout(@config[:RequestTimeout]){ + return io.__send__(method, *arg) + } + rescue Errno::ECONNRESET + return nil + rescue Timeout::Error + raise HTTPStatus::RequestTimeout + end + end + + def read_line(io, size=4096) + _read_data(io, :gets, LF, size) + end + + def read_data(io, size) + _read_data(io, :read, size) + end + + def parse_query() + begin + if @request_method == "GET" || @request_method == "HEAD" + @query = HTTPUtils::parse_query(@query_string) + elsif self['content-type'] =~ /^application\/x-www-form-urlencoded/ + @query = HTTPUtils::parse_query(body) + elsif self['content-type'] =~ /^multipart\/form-data; boundary=(.+)/ + boundary = HTTPUtils::dequote($1) + @query = HTTPUtils::parse_form_data(body, boundary) + else + @query = Hash.new + end + rescue => ex + raise HTTPStatus::BadRequest, ex.message + end + end + + PrivateNetworkRegexp = / + ^unknown$| + ^((::ffff:)?127.0.0.1|::1)$| + ^(::ffff:)?(10|172\.(1[6-9]|2[0-9]|3[01])|192\.168)\. + /ixo + + # It's said that all X-Forwarded-* headers will contain more than one + # (comma-separated) value if the original request already contained one of + # these headers. Since we could use these values as Host header, we choose + # the initial(first) value. (apr_table_mergen() adds new value after the + # existing value with ", " prefix) + def setup_forwarded_info + if @forwarded_server = self["x-forwarded-server"] + @forwarded_server = @forwarded_server.split(",", 2).first + end + if @forwarded_proto = self["x-forwarded-proto"] + @forwarded_proto = @forwarded_proto.split(",", 2).first + end + if host_port = self["x-forwarded-host"] + host_port = host_port.split(",", 2).first + if host_port =~ /\A(\[[0-9a-fA-F:]+\])(?::(\d+))?\z/ + @forwarded_host = $1 + tmp = $2 + else + @forwarded_host, tmp = host_port.split(":", 2) + end + @forwarded_port = (tmp || (@forwarded_proto == "https" ? 443 : 80)).to_i + end + if addrs = self["x-forwarded-for"] + addrs = addrs.split(",").collect(&:strip) + addrs.reject!{|ip| PrivateNetworkRegexp =~ ip } + @forwarded_for = addrs.first + end + end + + # :startdoc: + end +end diff --git a/tool/lib/webrick/httpresponse.rb b/tool/lib/webrick/httpresponse.rb new file mode 100644 index 0000000000..ba4494ab74 --- /dev/null +++ b/tool/lib/webrick/httpresponse.rb @@ -0,0 +1,564 @@ +# frozen_string_literal: false +# +# httpresponse.rb -- HTTPResponse Class +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: httpresponse.rb,v 1.45 2003/07/11 11:02:25 gotoyuzo Exp $ + +require 'time' +require 'uri' +require_relative 'httpversion' +require_relative 'htmlutils' +require_relative 'httputils' +require_relative 'httpstatus' + +module WEBrick + ## + # An HTTP response. This is filled in by the service or do_* methods of a + # WEBrick HTTP Servlet. + + class HTTPResponse + class InvalidHeader < StandardError + end + + ## + # HTTP Response version + + attr_reader :http_version + + ## + # Response status code (200) + + attr_reader :status + + ## + # Response header + + attr_reader :header + + ## + # Response cookies + + attr_reader :cookies + + ## + # Response reason phrase ("OK") + + attr_accessor :reason_phrase + + ## + # Body may be: + # * a String; + # * an IO-like object that responds to +#read+ and +#readpartial+; + # * a Proc-like object that responds to +#call+. + # + # In the latter case, either #chunked= should be set to +true+, + # or <code>header['content-length']</code> explicitly provided. + # Example: + # + # server.mount_proc '/' do |req, res| + # res.chunked = true + # # or + # # res.header['content-length'] = 10 + # res.body = proc { |out| out.write(Time.now.to_s) } + # end + + attr_accessor :body + + ## + # Request method for this response + + attr_accessor :request_method + + ## + # Request URI for this response + + attr_accessor :request_uri + + ## + # Request HTTP version for this response + + attr_accessor :request_http_version + + ## + # Filename of the static file in this response. Only used by the + # FileHandler servlet. + + attr_accessor :filename + + ## + # Is this a keep-alive response? + + attr_accessor :keep_alive + + ## + # Configuration for this response + + attr_reader :config + + ## + # Bytes sent in this response + + attr_reader :sent_size + + ## + # Creates a new HTTP response object. WEBrick::Config::HTTP is the + # default configuration. + + def initialize(config) + @config = config + @buffer_size = config[:OutputBufferSize] + @logger = config[:Logger] + @header = Hash.new + @status = HTTPStatus::RC_OK + @reason_phrase = nil + @http_version = HTTPVersion::convert(@config[:HTTPVersion]) + @body = '' + @keep_alive = true + @cookies = [] + @request_method = nil + @request_uri = nil + @request_http_version = @http_version # temporary + @chunked = false + @filename = nil + @sent_size = 0 + @bodytempfile = nil + end + + ## + # The response's HTTP status line + + def status_line + "HTTP/#@http_version #@status #@reason_phrase".rstrip << CRLF + end + + ## + # Sets the response's status to the +status+ code + + def status=(status) + @status = status + @reason_phrase = HTTPStatus::reason_phrase(status) + end + + ## + # Retrieves the response header +field+ + + def [](field) + @header[field.downcase] + end + + ## + # Sets the response header +field+ to +value+ + + def []=(field, value) + @chunked = value.to_s.downcase == 'chunked' if field.downcase == 'transfer-encoding' + @header[field.downcase] = value.to_s + end + + ## + # The content-length header + + def content_length + if len = self['content-length'] + return Integer(len) + end + end + + ## + # Sets the content-length header to +len+ + + def content_length=(len) + self['content-length'] = len.to_s + end + + ## + # The content-type header + + def content_type + self['content-type'] + end + + ## + # Sets the content-type header to +type+ + + def content_type=(type) + self['content-type'] = type + end + + ## + # Iterates over each header in the response + + def each + @header.each{|field, value| yield(field, value) } + end + + ## + # Will this response body be returned using chunked transfer-encoding? + + def chunked? + @chunked + end + + ## + # Enables chunked transfer encoding. + + def chunked=(val) + @chunked = val ? true : false + end + + ## + # Will this response's connection be kept alive? + + def keep_alive? + @keep_alive + end + + ## + # Sends the response on +socket+ + + def send_response(socket) # :nodoc: + begin + setup_header() + send_header(socket) + send_body(socket) + rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ENOTCONN => ex + @logger.debug(ex) + @keep_alive = false + rescue Exception => ex + @logger.error(ex) + @keep_alive = false + end + end + + ## + # Sets up the headers for sending + + def setup_header() # :nodoc: + @reason_phrase ||= HTTPStatus::reason_phrase(@status) + @header['server'] ||= @config[:ServerSoftware] + @header['date'] ||= Time.now.httpdate + + # HTTP/0.9 features + if @request_http_version < "1.0" + @http_version = HTTPVersion.new("0.9") + @keep_alive = false + end + + # HTTP/1.0 features + if @request_http_version < "1.1" + if chunked? + @chunked = false + ver = @request_http_version.to_s + msg = "chunked is set for an HTTP/#{ver} request. (ignored)" + @logger.warn(msg) + end + end + + # Determine the message length (RFC2616 -- 4.4 Message Length) + if @status == 304 || @status == 204 || HTTPStatus::info?(@status) + @header.delete('content-length') + @body = "" + elsif chunked? + @header["transfer-encoding"] = "chunked" + @header.delete('content-length') + elsif %r{^multipart/byteranges} =~ @header['content-type'] + @header.delete('content-length') + elsif @header['content-length'].nil? + if @body.respond_to? :readpartial + elsif @body.respond_to? :call + make_body_tempfile + else + @header['content-length'] = (@body ? @body.bytesize : 0).to_s + end + end + + # Keep-Alive connection. + if @header['connection'] == "close" + @keep_alive = false + elsif keep_alive? + if chunked? || @header['content-length'] || @status == 304 || @status == 204 || HTTPStatus.info?(@status) + @header['connection'] = "Keep-Alive" + else + msg = "Could not determine content-length of response body. Set content-length of the response or set Response#chunked = true" + @logger.warn(msg) + @header['connection'] = "close" + @keep_alive = false + end + else + @header['connection'] = "close" + end + + # Location is a single absoluteURI. + if location = @header['location'] + if @request_uri + @header['location'] = @request_uri.merge(location).to_s + end + end + end + + def make_body_tempfile # :nodoc: + return if @bodytempfile + bodytempfile = Tempfile.create("webrick") + if @body.nil? + # nothing + elsif @body.respond_to? :readpartial + IO.copy_stream(@body, bodytempfile) + @body.close + elsif @body.respond_to? :call + @body.call(bodytempfile) + else + bodytempfile.write @body + end + bodytempfile.rewind + @body = @bodytempfile = bodytempfile + @header['content-length'] = bodytempfile.stat.size.to_s + end + + def remove_body_tempfile # :nodoc: + if @bodytempfile + @bodytempfile.close + File.unlink @bodytempfile.path + @bodytempfile = nil + end + end + + + ## + # Sends the headers on +socket+ + + def send_header(socket) # :nodoc: + if @http_version.major > 0 + data = status_line() + @header.each{|key, value| + tmp = key.gsub(/\bwww|^te$|\b\w/){ $&.upcase } + data << "#{tmp}: #{check_header(value)}" << CRLF + } + @cookies.each{|cookie| + data << "Set-Cookie: " << check_header(cookie.to_s) << CRLF + } + data << CRLF + socket.write(data) + end + rescue InvalidHeader => e + @header.clear + @cookies.clear + set_error e + retry + end + + ## + # Sends the body on +socket+ + + def send_body(socket) # :nodoc: + if @body.respond_to? :readpartial then + send_body_io(socket) + elsif @body.respond_to?(:call) then + send_body_proc(socket) + else + send_body_string(socket) + end + end + + ## + # Redirects to +url+ with a WEBrick::HTTPStatus::Redirect +status+. + # + # Example: + # + # res.set_redirect WEBrick::HTTPStatus::TemporaryRedirect + + def set_redirect(status, url) + url = URI(url).to_s + @body = "<HTML><A HREF=\"#{url}\">#{url}</A>.</HTML>\n" + @header['location'] = url + raise status + end + + ## + # Creates an error page for exception +ex+ with an optional +backtrace+ + + def set_error(ex, backtrace=false) + case ex + when HTTPStatus::Status + @keep_alive = false if HTTPStatus::error?(ex.code) + self.status = ex.code + else + @keep_alive = false + self.status = HTTPStatus::RC_INTERNAL_SERVER_ERROR + end + @header['content-type'] = "text/html; charset=ISO-8859-1" + + if respond_to?(:create_error_page) + create_error_page() + return + end + + if @request_uri + host, port = @request_uri.host, @request_uri.port + else + host, port = @config[:ServerName], @config[:Port] + end + + error_body(backtrace, ex, host, port) + end + + private + + def check_header(header_value) + header_value = header_value.to_s + if /[\r\n]/ =~ header_value + raise InvalidHeader + else + header_value + end + end + + # :stopdoc: + + def error_body(backtrace, ex, host, port) + @body = '' + @body << <<-_end_of_html_ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0//EN"> +<HTML> + <HEAD><TITLE>#{HTMLUtils::escape(@reason_phrase)}</TITLE></HEAD> + <BODY> + <H1>#{HTMLUtils::escape(@reason_phrase)}</H1> + #{HTMLUtils::escape(ex.message)} + <HR> + _end_of_html_ + + if backtrace && $DEBUG + @body << "backtrace of `#{HTMLUtils::escape(ex.class.to_s)}' " + @body << "#{HTMLUtils::escape(ex.message)}" + @body << "<PRE>" + ex.backtrace.each{|line| @body << "\t#{line}\n"} + @body << "</PRE><HR>" + end + + @body << <<-_end_of_html_ + <ADDRESS> + #{HTMLUtils::escape(@config[:ServerSoftware])} at + #{host}:#{port} + </ADDRESS> + </BODY> +</HTML> + _end_of_html_ + end + + def send_body_io(socket) + begin + if @request_method == "HEAD" + # do nothing + elsif chunked? + buf = '' + begin + @body.readpartial(@buffer_size, buf) + size = buf.bytesize + data = "#{size.to_s(16)}#{CRLF}#{buf}#{CRLF}" + socket.write(data) + data.clear + @sent_size += size + rescue EOFError + break + end while true + buf.clear + socket.write("0#{CRLF}#{CRLF}") + else + if %r{\Abytes (\d+)-(\d+)/\d+\z} =~ @header['content-range'] + offset = $1.to_i + size = $2.to_i - offset + 1 + else + offset = nil + size = @header['content-length'] + size = size.to_i if size + end + begin + @sent_size = IO.copy_stream(@body, socket, size, offset) + rescue NotImplementedError + @body.seek(offset, IO::SEEK_SET) + @sent_size = IO.copy_stream(@body, socket, size) + end + end + ensure + @body.close + end + remove_body_tempfile + end + + def send_body_string(socket) + if @request_method == "HEAD" + # do nothing + elsif chunked? + body ? @body.bytesize : 0 + while buf = @body[@sent_size, @buffer_size] + break if buf.empty? + size = buf.bytesize + data = "#{size.to_s(16)}#{CRLF}#{buf}#{CRLF}" + buf.clear + socket.write(data) + @sent_size += size + end + socket.write("0#{CRLF}#{CRLF}") + else + if @body && @body.bytesize > 0 + socket.write(@body) + @sent_size = @body.bytesize + end + end + end + + def send_body_proc(socket) + if @request_method == "HEAD" + # do nothing + elsif chunked? + @body.call(ChunkedWrapper.new(socket, self)) + socket.write("0#{CRLF}#{CRLF}") + else + size = @header['content-length'].to_i + if @bodytempfile + @bodytempfile.rewind + IO.copy_stream(@bodytempfile, socket) + else + @body.call(socket) + end + @sent_size = size + end + end + + class ChunkedWrapper + def initialize(socket, resp) + @socket = socket + @resp = resp + end + + def write(buf) + return 0 if buf.empty? + socket = @socket + @resp.instance_eval { + size = buf.bytesize + data = "#{size.to_s(16)}#{CRLF}#{buf}#{CRLF}" + socket.write(data) + data.clear + @sent_size += size + size + } + end + + def <<(*buf) + write(buf) + self + end + end + + # preserved for compatibility with some 3rd-party handlers + def _write_data(socket, data) + socket << data + end + + # :startdoc: + end + +end diff --git a/tool/lib/webrick/https.rb b/tool/lib/webrick/https.rb new file mode 100644 index 0000000000..b0a49bc40b --- /dev/null +++ b/tool/lib/webrick/https.rb @@ -0,0 +1,152 @@ +# frozen_string_literal: false +# +# https.rb -- SSL/TLS enhancement for HTTPServer +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2001 GOTOU Yuuzou +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: https.rb,v 1.15 2003/07/22 19:20:42 gotoyuzo Exp $ + +require_relative 'ssl' +require_relative 'httpserver' + +module WEBrick + module Config + HTTP.update(SSL) + end + + ## + #-- + # Adds SSL functionality to WEBrick::HTTPRequest + + class HTTPRequest + + ## + # HTTP request SSL cipher + + attr_reader :cipher + + ## + # HTTP request server certificate + + attr_reader :server_cert + + ## + # HTTP request client certificate + + attr_reader :client_cert + + # :stopdoc: + + alias orig_parse parse + + def parse(socket=nil) + if socket.respond_to?(:cert) + @server_cert = socket.cert || @config[:SSLCertificate] + @client_cert = socket.peer_cert + @client_cert_chain = socket.peer_cert_chain + @cipher = socket.cipher + end + orig_parse(socket) + end + + alias orig_parse_uri parse_uri + + def parse_uri(str, scheme="https") + if server_cert + return orig_parse_uri(str, scheme) + end + return orig_parse_uri(str) + end + private :parse_uri + + alias orig_meta_vars meta_vars + + def meta_vars + meta = orig_meta_vars + if server_cert + meta["HTTPS"] = "on" + meta["SSL_SERVER_CERT"] = @server_cert.to_pem + meta["SSL_CLIENT_CERT"] = @client_cert ? @client_cert.to_pem : "" + if @client_cert_chain + @client_cert_chain.each_with_index{|cert, i| + meta["SSL_CLIENT_CERT_CHAIN_#{i}"] = cert.to_pem + } + end + meta["SSL_CIPHER"] = @cipher[0] + meta["SSL_PROTOCOL"] = @cipher[1] + meta["SSL_CIPHER_USEKEYSIZE"] = @cipher[2].to_s + meta["SSL_CIPHER_ALGKEYSIZE"] = @cipher[3].to_s + end + meta + end + + # :startdoc: + end + + ## + #-- + # Fake WEBrick::HTTPRequest for lookup_server + + class SNIRequest + + ## + # The SNI hostname + + attr_reader :host + + ## + # The socket address of the server + + attr_reader :addr + + ## + # The port this request is for + + attr_reader :port + + ## + # Creates a new SNIRequest. + + def initialize(sslsocket, hostname) + @host = hostname + @addr = sslsocket.addr + @port = @addr[1] + end + end + + + ## + #-- + # Adds SSL functionality to WEBrick::HTTPServer + + class HTTPServer < ::WEBrick::GenericServer + ## + # ServerNameIndication callback + + def ssl_servername_callback(sslsocket, hostname = nil) + req = SNIRequest.new(sslsocket, hostname) + server = lookup_server(req) + server ? server.ssl_context : nil + end + + # :stopdoc: + + ## + # Check whether +server+ is also SSL server. + # Also +server+'s SSL context will be created. + + alias orig_virtual_host virtual_host + + def virtual_host(server) + if @config[:SSLEnable] && !server.ssl_context + raise ArgumentError, "virtual host must set SSLEnable to true" + end + orig_virtual_host(server) + end + + # :startdoc: + end +end diff --git a/tool/lib/webrick/httpserver.rb b/tool/lib/webrick/httpserver.rb new file mode 100644 index 0000000000..e85d059319 --- /dev/null +++ b/tool/lib/webrick/httpserver.rb @@ -0,0 +1,294 @@ +# frozen_string_literal: false +# +# httpserver.rb -- HTTPServer Class +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: httpserver.rb,v 1.63 2002/10/01 17:16:32 gotoyuzo Exp $ + +require 'io/wait' +require_relative 'server' +require_relative 'httputils' +require_relative 'httpstatus' +require_relative 'httprequest' +require_relative 'httpresponse' +require_relative 'httpservlet' +require_relative 'accesslog' + +module WEBrick + class HTTPServerError < ServerError; end + + ## + # An HTTP Server + + class HTTPServer < ::WEBrick::GenericServer + ## + # Creates a new HTTP server according to +config+ + # + # An HTTP server uses the following attributes: + # + # :AccessLog:: An array of access logs. See WEBrick::AccessLog + # :BindAddress:: Local address for the server to bind to + # :DocumentRoot:: Root path to serve files from + # :DocumentRootOptions:: Options for the default HTTPServlet::FileHandler + # :HTTPVersion:: The HTTP version of this server + # :Port:: Port to listen on + # :RequestCallback:: Called with a request and response before each + # request is serviced. + # :RequestTimeout:: Maximum time to wait between requests + # :ServerAlias:: Array of alternate names for this server for virtual + # hosting + # :ServerName:: Name for this server for virtual hosting + + def initialize(config={}, default=Config::HTTP) + super(config, default) + @http_version = HTTPVersion::convert(@config[:HTTPVersion]) + + @mount_tab = MountTable.new + if @config[:DocumentRoot] + mount("/", HTTPServlet::FileHandler, @config[:DocumentRoot], + @config[:DocumentRootOptions]) + end + + unless @config[:AccessLog] + @config[:AccessLog] = [ + [ $stderr, AccessLog::COMMON_LOG_FORMAT ], + [ $stderr, AccessLog::REFERER_LOG_FORMAT ] + ] + end + + @virtual_hosts = Array.new + end + + ## + # Processes requests on +sock+ + + def run(sock) + while true + req = create_request(@config) + res = create_response(@config) + server = self + begin + timeout = @config[:RequestTimeout] + while timeout > 0 + break if sock.to_io.wait_readable(0.5) + break if @status != :Running + timeout -= 0.5 + end + raise HTTPStatus::EOFError if timeout <= 0 || @status != :Running + raise HTTPStatus::EOFError if sock.eof? + req.parse(sock) + res.request_method = req.request_method + res.request_uri = req.request_uri + res.request_http_version = req.http_version + res.keep_alive = req.keep_alive? + server = lookup_server(req) || self + if callback = server[:RequestCallback] + callback.call(req, res) + elsif callback = server[:RequestHandler] + msg = ":RequestHandler is deprecated, please use :RequestCallback" + @logger.warn(msg) + callback.call(req, res) + end + server.service(req, res) + rescue HTTPStatus::EOFError, HTTPStatus::RequestTimeout => ex + res.set_error(ex) + rescue HTTPStatus::Error => ex + @logger.error(ex.message) + res.set_error(ex) + rescue HTTPStatus::Status => ex + res.status = ex.code + rescue StandardError => ex + @logger.error(ex) + res.set_error(ex, true) + ensure + if req.request_line + if req.keep_alive? && res.keep_alive? + req.fixup() + end + res.send_response(sock) + server.access_log(@config, req, res) + end + end + break if @http_version < "1.1" + break unless req.keep_alive? + break unless res.keep_alive? + end + end + + ## + # Services +req+ and fills in +res+ + + def service(req, res) + if req.unparsed_uri == "*" + if req.request_method == "OPTIONS" + do_OPTIONS(req, res) + raise HTTPStatus::OK + end + raise HTTPStatus::NotFound, "`#{req.unparsed_uri}' not found." + end + + servlet, options, script_name, path_info = search_servlet(req.path) + raise HTTPStatus::NotFound, "`#{req.path}' not found." unless servlet + req.script_name = script_name + req.path_info = path_info + si = servlet.get_instance(self, *options) + @logger.debug(format("%s is invoked.", si.class.name)) + si.service(req, res) + end + + ## + # The default OPTIONS request handler says GET, HEAD, POST and OPTIONS + # requests are allowed. + + def do_OPTIONS(req, res) + res["allow"] = "GET,HEAD,POST,OPTIONS" + end + + ## + # Mounts +servlet+ on +dir+ passing +options+ to the servlet at creation + # time + + def mount(dir, servlet, *options) + @logger.debug(sprintf("%s is mounted on %s.", servlet.inspect, dir)) + @mount_tab[dir] = [ servlet, options ] + end + + ## + # Mounts +proc+ or +block+ on +dir+ and calls it with a + # WEBrick::HTTPRequest and WEBrick::HTTPResponse + + def mount_proc(dir, proc=nil, &block) + proc ||= block + raise HTTPServerError, "must pass a proc or block" unless proc + mount(dir, HTTPServlet::ProcHandler.new(proc)) + end + + ## + # Unmounts +dir+ + + def unmount(dir) + @logger.debug(sprintf("unmount %s.", dir)) + @mount_tab.delete(dir) + end + alias umount unmount + + ## + # Finds a servlet for +path+ + + def search_servlet(path) + script_name, path_info = @mount_tab.scan(path) + servlet, options = @mount_tab[script_name] + if servlet + [ servlet, options, script_name, path_info ] + end + end + + ## + # Adds +server+ as a virtual host. + + def virtual_host(server) + @virtual_hosts << server + @virtual_hosts = @virtual_hosts.sort_by{|s| + num = 0 + num -= 4 if s[:BindAddress] + num -= 2 if s[:Port] + num -= 1 if s[:ServerName] + num + } + end + + ## + # Finds the appropriate virtual host to handle +req+ + + def lookup_server(req) + @virtual_hosts.find{|s| + (s[:BindAddress].nil? || req.addr[3] == s[:BindAddress]) && + (s[:Port].nil? || req.port == s[:Port]) && + ((s[:ServerName].nil? || req.host == s[:ServerName]) || + (!s[:ServerAlias].nil? && s[:ServerAlias].find{|h| h === req.host})) + } + end + + ## + # Logs +req+ and +res+ in the access logs. +config+ is used for the + # server name. + + def access_log(config, req, res) + param = AccessLog::setup_params(config, req, res) + @config[:AccessLog].each{|logger, fmt| + logger << AccessLog::format(fmt+"\n", param) + } + end + + ## + # Creates the HTTPRequest used when handling the HTTP + # request. Can be overridden by subclasses. + def create_request(with_webrick_config) + HTTPRequest.new(with_webrick_config) + end + + ## + # Creates the HTTPResponse used when handling the HTTP + # request. Can be overridden by subclasses. + def create_response(with_webrick_config) + HTTPResponse.new(with_webrick_config) + end + + ## + # Mount table for the path a servlet is mounted on in the directory space + # of the server. Users of WEBrick can only access this indirectly via + # WEBrick::HTTPServer#mount, WEBrick::HTTPServer#unmount and + # WEBrick::HTTPServer#search_servlet + + class MountTable # :nodoc: + def initialize + @tab = Hash.new + compile + end + + def [](dir) + dir = normalize(dir) + @tab[dir] + end + + def []=(dir, val) + dir = normalize(dir) + @tab[dir] = val + compile + val + end + + def delete(dir) + dir = normalize(dir) + res = @tab.delete(dir) + compile + res + end + + def scan(path) + @scanner =~ path + [ $&, $' ] + end + + private + + def compile + k = @tab.keys + k.sort! + k.reverse! + k.collect!{|path| Regexp.escape(path) } + @scanner = Regexp.new("\\A(" + k.join("|") +")(?=/|\\z)") + end + + def normalize(dir) + ret = dir ? dir.dup : "" + ret.sub!(%r|/+\z|, "") + ret + end + end + end +end diff --git a/tool/lib/webrick/httpservlet.rb b/tool/lib/webrick/httpservlet.rb new file mode 100644 index 0000000000..da49a1405b --- /dev/null +++ b/tool/lib/webrick/httpservlet.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: false +# +# httpservlet.rb -- HTTPServlet Utility File +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: httpservlet.rb,v 1.21 2003/02/23 12:24:46 gotoyuzo Exp $ + +require_relative 'httpservlet/abstract' +require_relative 'httpservlet/filehandler' +require_relative 'httpservlet/cgihandler' +require_relative 'httpservlet/erbhandler' +require_relative 'httpservlet/prochandler' + +module WEBrick + module HTTPServlet + FileHandler.add_handler("cgi", CGIHandler) + FileHandler.add_handler("rhtml", ERBHandler) + end +end diff --git a/tool/lib/webrick/httpservlet/abstract.rb b/tool/lib/webrick/httpservlet/abstract.rb new file mode 100644 index 0000000000..bccb091861 --- /dev/null +++ b/tool/lib/webrick/httpservlet/abstract.rb @@ -0,0 +1,152 @@ +# frozen_string_literal: false +# +# httpservlet.rb -- HTTPServlet Module +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2000 TAKAHASHI Masayoshi, GOTOU Yuuzou +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: abstract.rb,v 1.24 2003/07/11 11:16:46 gotoyuzo Exp $ + +require_relative '../htmlutils' +require_relative '../httputils' +require_relative '../httpstatus' + +module WEBrick + module HTTPServlet + class HTTPServletError < StandardError; end + + ## + # AbstractServlet allows HTTP server modules to be reused across multiple + # servers and allows encapsulation of functionality. + # + # By default a servlet will respond to GET, HEAD (through an alias to GET) + # and OPTIONS requests. + # + # By default a new servlet is initialized for every request. A servlet + # instance can be reused by overriding ::get_instance in the + # AbstractServlet subclass. + # + # == A Simple Servlet + # + # class Simple < WEBrick::HTTPServlet::AbstractServlet + # def do_GET request, response + # status, content_type, body = do_stuff_with request + # + # response.status = status + # response['Content-Type'] = content_type + # response.body = body + # end + # + # def do_stuff_with request + # return 200, 'text/plain', 'you got a page' + # end + # end + # + # This servlet can be mounted on a server at a given path: + # + # server.mount '/simple', Simple + # + # == Servlet Configuration + # + # Servlets can be configured via initialize. The first argument is the + # HTTP server the servlet is being initialized for. + # + # class Configurable < Simple + # def initialize server, color, size + # super server + # @color = color + # @size = size + # end + # + # def do_stuff_with request + # content = "<p " \ + # %q{style="color: #{@color}; font-size: #{@size}"} \ + # ">Hello, World!" + # + # return 200, "text/html", content + # end + # end + # + # This servlet must be provided two arguments at mount time: + # + # server.mount '/configurable', Configurable, 'red', '2em' + + class AbstractServlet + + ## + # Factory for servlet instances that will handle a request from +server+ + # using +options+ from the mount point. By default a new servlet + # instance is created for every call. + + def self.get_instance(server, *options) + self.new(server, *options) + end + + ## + # Initializes a new servlet for +server+ using +options+ which are + # stored as-is in +@options+. +@logger+ is also provided. + + def initialize(server, *options) + @server = @config = server + @logger = @server[:Logger] + @options = options + end + + ## + # Dispatches to a +do_+ method based on +req+ if such a method is + # available. (+do_GET+ for a GET request). Raises a MethodNotAllowed + # exception if the method is not implemented. + + def service(req, res) + method_name = "do_" + req.request_method.gsub(/-/, "_") + if respond_to?(method_name) + __send__(method_name, req, res) + else + raise HTTPStatus::MethodNotAllowed, + "unsupported method `#{req.request_method}'." + end + end + + ## + # Raises a NotFound exception + + def do_GET(req, res) + raise HTTPStatus::NotFound, "not found." + end + + ## + # Dispatches to do_GET + + def do_HEAD(req, res) + do_GET(req, res) + end + + ## + # Returns the allowed HTTP request methods + + def do_OPTIONS(req, res) + m = self.methods.grep(/\Ado_([A-Z]+)\z/) {$1} + m.sort! + res["allow"] = m.join(",") + end + + private + + ## + # Redirects to a path ending in / + + def redirect_to_directory_uri(req, res) + if req.path[-1] != ?/ + location = WEBrick::HTTPUtils.escape_path(req.path + "/") + if req.query_string && req.query_string.bytesize > 0 + location << "?" << req.query_string + end + res.set_redirect(HTTPStatus::MovedPermanently, location) + end + end + end + + end +end diff --git a/tool/lib/webrick/httpservlet/cgi_runner.rb b/tool/lib/webrick/httpservlet/cgi_runner.rb new file mode 100644 index 0000000000..0398c16749 --- /dev/null +++ b/tool/lib/webrick/httpservlet/cgi_runner.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: false +# +# cgi_runner.rb -- CGI launcher. +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2000 TAKAHASHI Masayoshi, GOTOU YUUZOU +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: cgi_runner.rb,v 1.9 2002/09/25 11:33:15 gotoyuzo Exp $ + +def sysread(io, size) + buf = "" + while size > 0 + tmp = io.sysread(size) + buf << tmp + size -= tmp.bytesize + end + return buf +end + +STDIN.binmode + +len = sysread(STDIN, 8).to_i +out = sysread(STDIN, len) +STDOUT.reopen(File.open(out, "w")) + +len = sysread(STDIN, 8).to_i +err = sysread(STDIN, len) +STDERR.reopen(File.open(err, "w")) + +len = sysread(STDIN, 8).to_i +dump = sysread(STDIN, len) +hash = Marshal.restore(dump) +ENV.keys.each{|name| ENV.delete(name) } +hash.each{|k, v| ENV[k] = v if v } + +dir = File::dirname(ENV["SCRIPT_FILENAME"]) +Dir::chdir dir + +if ARGV[0] + argv = ARGV.dup + argv << ENV["SCRIPT_FILENAME"] + exec(*argv) + # NOTREACHED +end +exec ENV["SCRIPT_FILENAME"] diff --git a/tool/lib/webrick/httpservlet/cgihandler.rb b/tool/lib/webrick/httpservlet/cgihandler.rb new file mode 100644 index 0000000000..4457770b7a --- /dev/null +++ b/tool/lib/webrick/httpservlet/cgihandler.rb @@ -0,0 +1,126 @@ +# frozen_string_literal: false +# +# cgihandler.rb -- CGIHandler Class +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: cgihandler.rb,v 1.27 2003/03/21 19:56:01 gotoyuzo Exp $ + +require 'rbconfig' +require 'tempfile' +require_relative '../config' +require_relative 'abstract' + +module WEBrick + module HTTPServlet + + ## + # Servlet for handling CGI scripts + # + # Example: + # + # server.mount('/cgi/my_script', WEBrick::HTTPServlet::CGIHandler, + # '/path/to/my_script') + + class CGIHandler < AbstractServlet + Ruby = RbConfig.ruby # :nodoc: + CGIRunner = "\"#{Ruby}\" \"#{WEBrick::Config::LIBDIR}/httpservlet/cgi_runner.rb\"" # :nodoc: + CGIRunnerArray = [Ruby, "#{WEBrick::Config::LIBDIR}/httpservlet/cgi_runner.rb".freeze].freeze # :nodoc: + + ## + # Creates a new CGI script servlet for the script at +name+ + + def initialize(server, name) + super(server, name) + @script_filename = name + @tempdir = server[:TempDir] + interpreter = server[:CGIInterpreter] + if interpreter.is_a?(Array) + @cgicmd = CGIRunnerArray + interpreter + else + @cgicmd = "#{CGIRunner} #{interpreter}" + end + end + + # :stopdoc: + + def do_GET(req, res) + cgi_in = IO::popen(@cgicmd, "wb") + cgi_out = Tempfile.new("webrick.cgiout.", @tempdir, mode: IO::BINARY) + cgi_out.set_encoding("ASCII-8BIT") + cgi_err = Tempfile.new("webrick.cgierr.", @tempdir, mode: IO::BINARY) + cgi_err.set_encoding("ASCII-8BIT") + begin + cgi_in.sync = true + meta = req.meta_vars + meta["SCRIPT_FILENAME"] = @script_filename + meta["PATH"] = @config[:CGIPathEnv] + meta.delete("HTTP_PROXY") + if /mswin|bccwin|mingw/ =~ RUBY_PLATFORM + meta["SystemRoot"] = ENV["SystemRoot"] + end + dump = Marshal.dump(meta) + + cgi_in.write("%8d" % cgi_out.path.bytesize) + cgi_in.write(cgi_out.path) + cgi_in.write("%8d" % cgi_err.path.bytesize) + cgi_in.write(cgi_err.path) + cgi_in.write("%8d" % dump.bytesize) + cgi_in.write(dump) + + req.body { |chunk| cgi_in.write(chunk) } + ensure + cgi_in.close + status = $?.exitstatus + sleep 0.1 if /mswin|bccwin|mingw/ =~ RUBY_PLATFORM + data = cgi_out.read + cgi_out.close(true) + if errmsg = cgi_err.read + if errmsg.bytesize > 0 + @logger.error("CGIHandler: #{@script_filename}:\n" + errmsg) + end + end + cgi_err.close(true) + end + + if status != 0 + @logger.error("CGIHandler: #{@script_filename} exit with #{status}") + end + + data = "" unless data + raw_header, body = data.split(/^[\xd\xa]+/, 2) + raise HTTPStatus::InternalServerError, + "Premature end of script headers: #{@script_filename}" if body.nil? + + begin + header = HTTPUtils::parse_header(raw_header) + if /^(\d+)/ =~ header['status'][0] + res.status = $1.to_i + header.delete('status') + end + if header.has_key?('location') + # RFC 3875 6.2.3, 6.2.4 + res.status = 302 unless (300...400) === res.status + end + if header.has_key?('set-cookie') + header['set-cookie'].each{|k| + res.cookies << Cookie.parse_set_cookie(k) + } + header.delete('set-cookie') + end + header.each{|key, val| res[key] = val.join(", ") } + rescue => ex + raise HTTPStatus::InternalServerError, ex.message + end + res.body = body + end + alias do_POST do_GET + + # :startdoc: + end + + end +end diff --git a/tool/lib/webrick/httpservlet/erbhandler.rb b/tool/lib/webrick/httpservlet/erbhandler.rb new file mode 100644 index 0000000000..cd09e5f216 --- /dev/null +++ b/tool/lib/webrick/httpservlet/erbhandler.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: false +# +# erbhandler.rb -- ERBHandler Class +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: erbhandler.rb,v 1.25 2003/02/24 19:25:31 gotoyuzo Exp $ + +require_relative 'abstract' + +require 'erb' + +module WEBrick + module HTTPServlet + + ## + # ERBHandler evaluates an ERB file and returns the result. This handler + # is automatically used if there are .rhtml files in a directory served by + # the FileHandler. + # + # ERBHandler supports GET and POST methods. + # + # The ERB file is evaluated with the local variables +servlet_request+ and + # +servlet_response+ which are a WEBrick::HTTPRequest and + # WEBrick::HTTPResponse respectively. + # + # Example .rhtml file: + # + # Request to <%= servlet_request.request_uri %> + # + # Query params <%= servlet_request.query.inspect %> + + class ERBHandler < AbstractServlet + + ## + # Creates a new ERBHandler on +server+ that will evaluate and serve the + # ERB file +name+ + + def initialize(server, name) + super(server, name) + @script_filename = name + end + + ## + # Handles GET requests + + def do_GET(req, res) + unless defined?(ERB) + @logger.warn "#{self.class}: ERB not defined." + raise HTTPStatus::Forbidden, "ERBHandler cannot work." + end + begin + data = File.open(@script_filename, &:read) + res.body = evaluate(ERB.new(data), req, res) + res['content-type'] ||= + HTTPUtils::mime_type(@script_filename, @config[:MimeTypes]) + rescue StandardError + raise + rescue Exception => ex + @logger.error(ex) + raise HTTPStatus::InternalServerError, ex.message + end + end + + ## + # Handles POST requests + + alias do_POST do_GET + + private + + ## + # Evaluates +erb+ providing +servlet_request+ and +servlet_response+ as + # local variables. + + def evaluate(erb, servlet_request, servlet_response) + Module.new.module_eval{ + servlet_request.meta_vars + servlet_request.query + erb.result(binding) + } + end + end + end +end diff --git a/tool/lib/webrick/httpservlet/filehandler.rb b/tool/lib/webrick/httpservlet/filehandler.rb new file mode 100644 index 0000000000..010df0e918 --- /dev/null +++ b/tool/lib/webrick/httpservlet/filehandler.rb @@ -0,0 +1,552 @@ +# frozen_string_literal: false +# +# filehandler.rb -- FileHandler Module +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou +# Copyright (c) 2003 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: filehandler.rb,v 1.44 2003/06/07 01:34:51 gotoyuzo Exp $ + +require 'time' + +require_relative '../htmlutils' +require_relative '../httputils' +require_relative '../httpstatus' + +module WEBrick + module HTTPServlet + + ## + # Servlet for serving a single file. You probably want to use the + # FileHandler servlet instead as it handles directories and fancy indexes. + # + # Example: + # + # server.mount('/my_page.txt', WEBrick::HTTPServlet::DefaultFileHandler, + # '/path/to/my_page.txt') + # + # This servlet handles If-Modified-Since and Range requests. + + class DefaultFileHandler < AbstractServlet + + ## + # Creates a DefaultFileHandler instance for the file at +local_path+. + + def initialize(server, local_path) + super(server, local_path) + @local_path = local_path + end + + # :stopdoc: + + def do_GET(req, res) + st = File::stat(@local_path) + mtime = st.mtime + res['etag'] = sprintf("%x-%x-%x", st.ino, st.size, st.mtime.to_i) + + if not_modified?(req, res, mtime, res['etag']) + res.body = '' + raise HTTPStatus::NotModified + elsif req['range'] + make_partial_content(req, res, @local_path, st.size) + raise HTTPStatus::PartialContent + else + mtype = HTTPUtils::mime_type(@local_path, @config[:MimeTypes]) + res['content-type'] = mtype + res['content-length'] = st.size.to_s + res['last-modified'] = mtime.httpdate + res.body = File.open(@local_path, "rb") + end + end + + def not_modified?(req, res, mtime, etag) + if ir = req['if-range'] + begin + if Time.httpdate(ir) >= mtime + return true + end + rescue + if HTTPUtils::split_header_value(ir).member?(res['etag']) + return true + end + end + end + + if (ims = req['if-modified-since']) && Time.parse(ims) >= mtime + return true + end + + if (inm = req['if-none-match']) && + HTTPUtils::split_header_value(inm).member?(res['etag']) + return true + end + + return false + end + + # returns a lambda for webrick/httpresponse.rb send_body_proc + def multipart_body(body, parts, boundary, mtype, filesize) + lambda do |socket| + begin + begin + first = parts.shift + last = parts.shift + socket.write( + "--#{boundary}#{CRLF}" \ + "Content-Type: #{mtype}#{CRLF}" \ + "Content-Range: bytes #{first}-#{last}/#{filesize}#{CRLF}" \ + "#{CRLF}" + ) + + begin + IO.copy_stream(body, socket, last - first + 1, first) + rescue NotImplementedError + body.seek(first, IO::SEEK_SET) + IO.copy_stream(body, socket, last - first + 1) + end + socket.write(CRLF) + end while parts[0] + socket.write("--#{boundary}--#{CRLF}") + ensure + body.close + end + end + end + + def make_partial_content(req, res, filename, filesize) + mtype = HTTPUtils::mime_type(filename, @config[:MimeTypes]) + unless ranges = HTTPUtils::parse_range_header(req['range']) + raise HTTPStatus::BadRequest, + "Unrecognized range-spec: \"#{req['range']}\"" + end + File.open(filename, "rb"){|io| + if ranges.size > 1 + time = Time.now + boundary = "#{time.sec}_#{time.usec}_#{Process::pid}" + parts = [] + ranges.each {|range| + prange = prepare_range(range, filesize) + next if prange[0] < 0 + parts.concat(prange) + } + raise HTTPStatus::RequestRangeNotSatisfiable if parts.empty? + res["content-type"] = "multipart/byteranges; boundary=#{boundary}" + if req.http_version < '1.1' + res['connection'] = 'close' + else + res.chunked = true + end + res.body = multipart_body(io.dup, parts, boundary, mtype, filesize) + elsif range = ranges[0] + first, last = prepare_range(range, filesize) + raise HTTPStatus::RequestRangeNotSatisfiable if first < 0 + res['content-type'] = mtype + res['content-range'] = "bytes #{first}-#{last}/#{filesize}" + res['content-length'] = (last - first + 1).to_s + res.body = io.dup + else + raise HTTPStatus::BadRequest + end + } + end + + def prepare_range(range, filesize) + first = range.first < 0 ? filesize + range.first : range.first + return -1, -1 if first < 0 || first >= filesize + last = range.last < 0 ? filesize + range.last : range.last + last = filesize - 1 if last >= filesize + return first, last + end + + # :startdoc: + end + + ## + # Serves a directory including fancy indexing and a variety of other + # options. + # + # Example: + # + # server.mount('/assets', WEBrick::HTTPServlet::FileHandler, + # '/path/to/assets') + + class FileHandler < AbstractServlet + HandlerTable = Hash.new # :nodoc: + + ## + # Allow custom handling of requests for files with +suffix+ by class + # +handler+ + + def self.add_handler(suffix, handler) + HandlerTable[suffix] = handler + end + + ## + # Remove custom handling of requests for files with +suffix+ + + def self.remove_handler(suffix) + HandlerTable.delete(suffix) + end + + ## + # Creates a FileHandler servlet on +server+ that serves files starting + # at directory +root+ + # + # +options+ may be a Hash containing keys from + # WEBrick::Config::FileHandler or +true+ or +false+. + # + # If +options+ is true or false then +:FancyIndexing+ is enabled or + # disabled respectively. + + def initialize(server, root, options={}, default=Config::FileHandler) + @config = server.config + @logger = @config[:Logger] + @root = File.expand_path(root) + if options == true || options == false + options = { :FancyIndexing => options } + end + @options = default.dup.update(options) + end + + # :stopdoc: + + def set_filesystem_encoding(str) + enc = Encoding.find('filesystem') + if enc == Encoding::US_ASCII + str.b + else + str.dup.force_encoding(enc) + end + end + + def service(req, res) + # if this class is mounted on "/" and /~username is requested. + # we're going to override path information before invoking service. + if defined?(Etc) && @options[:UserDir] && req.script_name.empty? + if %r|^(/~([^/]+))| =~ req.path_info + script_name, user = $1, $2 + path_info = $' + begin + passwd = Etc::getpwnam(user) + @root = File::join(passwd.dir, @options[:UserDir]) + req.script_name = script_name + req.path_info = path_info + rescue + @logger.debug "#{self.class}#do_GET: getpwnam(#{user}) failed" + end + end + end + prevent_directory_traversal(req, res) + super(req, res) + end + + def do_GET(req, res) + unless exec_handler(req, res) + set_dir_list(req, res) + end + end + + def do_POST(req, res) + unless exec_handler(req, res) + raise HTTPStatus::NotFound, "`#{req.path}' not found." + end + end + + def do_OPTIONS(req, res) + unless exec_handler(req, res) + super(req, res) + end + end + + # ToDo + # RFC2518: HTTP Extensions for Distributed Authoring -- WEBDAV + # + # PROPFIND PROPPATCH MKCOL DELETE PUT COPY MOVE + # LOCK UNLOCK + + # RFC3253: Versioning Extensions to WebDAV + # (Web Distributed Authoring and Versioning) + # + # VERSION-CONTROL REPORT CHECKOUT CHECK_IN UNCHECKOUT + # MKWORKSPACE UPDATE LABEL MERGE ACTIVITY + + private + + def trailing_pathsep?(path) + # check for trailing path separator: + # File.dirname("/aaaa/bbbb/") #=> "/aaaa") + # File.dirname("/aaaa/bbbb/x") #=> "/aaaa/bbbb") + # File.dirname("/aaaa/bbbb") #=> "/aaaa") + # File.dirname("/aaaa/bbbbx") #=> "/aaaa") + return File.dirname(path) != File.dirname(path+"x") + end + + def prevent_directory_traversal(req, res) + # Preventing directory traversal on Windows platforms; + # Backslashes (0x5c) in path_info are not interpreted as special + # character in URI notation. So the value of path_info should be + # normalize before accessing to the filesystem. + + # dirty hack for filesystem encoding; in nature, File.expand_path + # should not be used for path normalization. [Bug #3345] + path = req.path_info.dup.force_encoding(Encoding.find("filesystem")) + if trailing_pathsep?(req.path_info) + # File.expand_path removes the trailing path separator. + # Adding a character is a workaround to save it. + # File.expand_path("/aaa/") #=> "/aaa" + # File.expand_path("/aaa/" + "x") #=> "/aaa/x" + expanded = File.expand_path(path + "x") + expanded.chop! # remove trailing "x" + else + expanded = File.expand_path(path) + end + expanded.force_encoding(req.path_info.encoding) + req.path_info = expanded + end + + def exec_handler(req, res) + raise HTTPStatus::NotFound, "`#{req.path}' not found." unless @root + if set_filename(req, res) + handler = get_handler(req, res) + call_callback(:HandlerCallback, req, res) + h = handler.get_instance(@config, res.filename) + h.service(req, res) + return true + end + call_callback(:HandlerCallback, req, res) + return false + end + + def get_handler(req, res) + suffix1 = (/\.(\w+)\z/ =~ res.filename) && $1.downcase + if /\.(\w+)\.([\w\-]+)\z/ =~ res.filename + if @options[:AcceptableLanguages].include?($2.downcase) + suffix2 = $1.downcase + end + end + handler_table = @options[:HandlerTable] + return handler_table[suffix1] || handler_table[suffix2] || + HandlerTable[suffix1] || HandlerTable[suffix2] || + DefaultFileHandler + end + + def set_filename(req, res) + res.filename = @root + path_info = req.path_info.scan(%r|/[^/]*|) + + path_info.unshift("") # dummy for checking @root dir + while base = path_info.first + base = set_filesystem_encoding(base) + break if base == "/" + break unless File.directory?(File.expand_path(res.filename + base)) + shift_path_info(req, res, path_info) + call_callback(:DirectoryCallback, req, res) + end + + if base = path_info.first + base = set_filesystem_encoding(base) + if base == "/" + if file = search_index_file(req, res) + shift_path_info(req, res, path_info, file) + call_callback(:FileCallback, req, res) + return true + end + shift_path_info(req, res, path_info) + elsif file = search_file(req, res, base) + shift_path_info(req, res, path_info, file) + call_callback(:FileCallback, req, res) + return true + else + raise HTTPStatus::NotFound, "`#{req.path}' not found." + end + end + + return false + end + + def check_filename(req, res, name) + if nondisclosure_name?(name) || windows_ambiguous_name?(name) + @logger.warn("the request refers nondisclosure name `#{name}'.") + raise HTTPStatus::NotFound, "`#{req.path}' not found." + end + end + + def shift_path_info(req, res, path_info, base=nil) + tmp = path_info.shift + base = base || set_filesystem_encoding(tmp) + req.path_info = path_info.join + req.script_name << base + res.filename = File.expand_path(res.filename + base) + check_filename(req, res, File.basename(res.filename)) + end + + def search_index_file(req, res) + @config[:DirectoryIndex].each{|index| + if file = search_file(req, res, "/"+index) + return file + end + } + return nil + end + + def search_file(req, res, basename) + langs = @options[:AcceptableLanguages] + path = res.filename + basename + if File.file?(path) + return basename + elsif langs.size > 0 + req.accept_language.each{|lang| + path_with_lang = path + ".#{lang}" + if langs.member?(lang) && File.file?(path_with_lang) + return basename + ".#{lang}" + end + } + (langs - req.accept_language).each{|lang| + path_with_lang = path + ".#{lang}" + if File.file?(path_with_lang) + return basename + ".#{lang}" + end + } + end + return nil + end + + def call_callback(callback_name, req, res) + if cb = @options[callback_name] + cb.call(req, res) + end + end + + def windows_ambiguous_name?(name) + return true if /[. ]+\z/ =~ name + return true if /::\$DATA\z/ =~ name + return false + end + + def nondisclosure_name?(name) + @options[:NondisclosureName].each{|pattern| + if File.fnmatch(pattern, name, File::FNM_CASEFOLD) + return true + end + } + return false + end + + def set_dir_list(req, res) + redirect_to_directory_uri(req, res) + unless @options[:FancyIndexing] + raise HTTPStatus::Forbidden, "no access permission to `#{req.path}'" + end + local_path = res.filename + list = Dir::entries(local_path).collect{|name| + next if name == "." || name == ".." + next if nondisclosure_name?(name) + next if windows_ambiguous_name?(name) + st = (File::stat(File.join(local_path, name)) rescue nil) + if st.nil? + [ name, nil, -1 ] + elsif st.directory? + [ name + "/", st.mtime, -1 ] + else + [ name, st.mtime, st.size ] + end + } + list.compact! + + query = req.query + + d0 = nil + idx = nil + %w[N M S].each_with_index do |q, i| + if d = query.delete(q) + idx ||= i + d0 ||= d + end + end + d0 ||= "A" + idx ||= 0 + d1 = (d0 == "A") ? "D" : "A" + + if d0 == "A" + list.sort!{|a,b| a[idx] <=> b[idx] } + else + list.sort!{|a,b| b[idx] <=> a[idx] } + end + + namewidth = query["NameWidth"] + if namewidth == "*" + namewidth = nil + elsif !namewidth or (namewidth = namewidth.to_i) < 2 + namewidth = 25 + end + query = query.inject('') {|s, (k, v)| s << '&' << HTMLUtils::escape("#{k}=#{v}")} + + type = "text/html" + case enc = Encoding.find('filesystem') + when Encoding::US_ASCII, Encoding::ASCII_8BIT + else + type << "; charset=\"#{enc.name}\"" + end + res['content-type'] = type + + title = "Index of #{HTMLUtils::escape(req.path)}" + res.body = <<-_end_of_html_ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN"> +<HTML> + <HEAD> + <TITLE>#{title}</TITLE> + <style type="text/css"> + <!-- + .name, .mtime { text-align: left; } + .size { text-align: right; } + td { text-overflow: ellipsis; white-space: nowrap; overflow: hidden; } + table { border-collapse: collapse; } + tr th { border-bottom: 2px groove; } + //--> + </style> + </HEAD> + <BODY> + <H1>#{title}</H1> + _end_of_html_ + + res.body << "<TABLE width=\"100%\"><THEAD><TR>\n" + res.body << "<TH class=\"name\"><A HREF=\"?N=#{d1}#{query}\">Name</A></TH>" + res.body << "<TH class=\"mtime\"><A HREF=\"?M=#{d1}#{query}\">Last modified</A></TH>" + res.body << "<TH class=\"size\"><A HREF=\"?S=#{d1}#{query}\">Size</A></TH>\n" + res.body << "</TR></THEAD>\n" + res.body << "<TBODY>\n" + + query.sub!(/\A&/, '?') + list.unshift [ "..", File::mtime(local_path+"/.."), -1 ] + list.each{ |name, time, size| + if name == ".." + dname = "Parent Directory" + elsif namewidth and name.size > namewidth + dname = name[0...(namewidth - 2)] << '..' + else + dname = name + end + s = "<TR><TD class=\"name\"><A HREF=\"#{HTTPUtils::escape(name)}#{query if name.end_with?('/')}\">#{HTMLUtils::escape(dname)}</A></TD>" + s << "<TD class=\"mtime\">" << (time ? time.strftime("%Y/%m/%d %H:%M") : "") << "</TD>" + s << "<TD class=\"size\">" << (size >= 0 ? size.to_s : "-") << "</TD></TR>\n" + res.body << s + } + res.body << "</TBODY></TABLE>" + res.body << "<HR>" + + res.body << <<-_end_of_html_ + <ADDRESS> + #{HTMLUtils::escape(@config[:ServerSoftware])}<BR> + at #{req.host}:#{req.port} + </ADDRESS> + </BODY> +</HTML> + _end_of_html_ + end + + # :startdoc: + end + end +end diff --git a/tool/lib/webrick/httpservlet/prochandler.rb b/tool/lib/webrick/httpservlet/prochandler.rb new file mode 100644 index 0000000000..599ffc4340 --- /dev/null +++ b/tool/lib/webrick/httpservlet/prochandler.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: false +# +# prochandler.rb -- ProcHandler Class +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: prochandler.rb,v 1.7 2002/09/21 12:23:42 gotoyuzo Exp $ + +require_relative 'abstract' + +module WEBrick + module HTTPServlet + + ## + # Mounts a proc at a path that accepts a request and response. + # + # Instead of mounting this servlet with WEBrick::HTTPServer#mount use + # WEBrick::HTTPServer#mount_proc: + # + # server.mount_proc '/' do |req, res| + # res.body = 'it worked!' + # res.status = 200 + # end + + class ProcHandler < AbstractServlet + # :stopdoc: + def get_instance(server, *options) + self + end + + def initialize(proc) + @proc = proc + end + + def do_GET(request, response) + @proc.call(request, response) + end + + alias do_POST do_GET + # :startdoc: + end + + end +end diff --git a/tool/lib/webrick/httpstatus.rb b/tool/lib/webrick/httpstatus.rb new file mode 100644 index 0000000000..c811f21964 --- /dev/null +++ b/tool/lib/webrick/httpstatus.rb @@ -0,0 +1,194 @@ +# frozen_string_literal: false +#-- +# httpstatus.rb -- HTTPStatus Class +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: httpstatus.rb,v 1.11 2003/03/24 20:18:55 gotoyuzo Exp $ + +require_relative 'accesslog' + +module WEBrick + + ## + # This module is used to manager HTTP status codes. + # + # See http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html for more + # information. + module HTTPStatus + + ## + # Root of the HTTP status class hierarchy + class Status < StandardError + class << self + attr_reader :code, :reason_phrase # :nodoc: + end + + # Returns the HTTP status code + def code() self::class::code end + + # Returns the HTTP status description + def reason_phrase() self::class::reason_phrase end + + alias to_i code # :nodoc: + end + + # Root of the HTTP info statuses + class Info < Status; end + # Root of the HTTP success statuses + class Success < Status; end + # Root of the HTTP redirect statuses + class Redirect < Status; end + # Root of the HTTP error statuses + class Error < Status; end + # Root of the HTTP client error statuses + class ClientError < Error; end + # Root of the HTTP server error statuses + class ServerError < Error; end + + class EOFError < StandardError; end + + # HTTP status codes and descriptions + StatusMessage = { # :nodoc: + 100 => 'Continue', + 101 => 'Switching Protocols', + 200 => 'OK', + 201 => 'Created', + 202 => 'Accepted', + 203 => 'Non-Authoritative Information', + 204 => 'No Content', + 205 => 'Reset Content', + 206 => 'Partial Content', + 207 => 'Multi-Status', + 300 => 'Multiple Choices', + 301 => 'Moved Permanently', + 302 => 'Found', + 303 => 'See Other', + 304 => 'Not Modified', + 305 => 'Use Proxy', + 307 => 'Temporary Redirect', + 400 => 'Bad Request', + 401 => 'Unauthorized', + 402 => 'Payment Required', + 403 => 'Forbidden', + 404 => 'Not Found', + 405 => 'Method Not Allowed', + 406 => 'Not Acceptable', + 407 => 'Proxy Authentication Required', + 408 => 'Request Timeout', + 409 => 'Conflict', + 410 => 'Gone', + 411 => 'Length Required', + 412 => 'Precondition Failed', + 413 => 'Request Entity Too Large', + 414 => 'Request-URI Too Large', + 415 => 'Unsupported Media Type', + 416 => 'Request Range Not Satisfiable', + 417 => 'Expectation Failed', + 422 => 'Unprocessable Entity', + 423 => 'Locked', + 424 => 'Failed Dependency', + 426 => 'Upgrade Required', + 428 => 'Precondition Required', + 429 => 'Too Many Requests', + 431 => 'Request Header Fields Too Large', + 451 => 'Unavailable For Legal Reasons', + 500 => 'Internal Server Error', + 501 => 'Not Implemented', + 502 => 'Bad Gateway', + 503 => 'Service Unavailable', + 504 => 'Gateway Timeout', + 505 => 'HTTP Version Not Supported', + 507 => 'Insufficient Storage', + 511 => 'Network Authentication Required', + } + + # Maps a status code to the corresponding Status class + CodeToError = {} # :nodoc: + + # Creates a status or error class for each status code and + # populates the CodeToError map. + StatusMessage.each{|code, message| + message.freeze + var_name = message.gsub(/[ \-]/,'_').upcase + err_name = message.gsub(/[ \-]/,'') + + case code + when 100...200; parent = Info + when 200...300; parent = Success + when 300...400; parent = Redirect + when 400...500; parent = ClientError + when 500...600; parent = ServerError + end + + const_set("RC_#{var_name}", code) + err_class = Class.new(parent) + err_class.instance_variable_set(:@code, code) + err_class.instance_variable_set(:@reason_phrase, message) + const_set(err_name, err_class) + CodeToError[code] = err_class + } + + ## + # Returns the description corresponding to the HTTP status +code+ + # + # WEBrick::HTTPStatus.reason_phrase 404 + # => "Not Found" + def reason_phrase(code) + StatusMessage[code.to_i] + end + + ## + # Is +code+ an informational status? + def info?(code) + code.to_i >= 100 and code.to_i < 200 + end + + ## + # Is +code+ a successful status? + def success?(code) + code.to_i >= 200 and code.to_i < 300 + end + + ## + # Is +code+ a redirection status? + def redirect?(code) + code.to_i >= 300 and code.to_i < 400 + end + + ## + # Is +code+ an error status? + def error?(code) + code.to_i >= 400 and code.to_i < 600 + end + + ## + # Is +code+ a client error status? + def client_error?(code) + code.to_i >= 400 and code.to_i < 500 + end + + ## + # Is +code+ a server error status? + def server_error?(code) + code.to_i >= 500 and code.to_i < 600 + end + + ## + # Returns the status class corresponding to +code+ + # + # WEBrick::HTTPStatus[302] + # => WEBrick::HTTPStatus::NotFound + # + def self.[](code) + CodeToError[code] + end + + module_function :reason_phrase + module_function :info?, :success?, :redirect?, :error? + module_function :client_error?, :server_error? + end +end diff --git a/tool/lib/webrick/httputils.rb b/tool/lib/webrick/httputils.rb new file mode 100644 index 0000000000..f1b9ddf9f0 --- /dev/null +++ b/tool/lib/webrick/httputils.rb @@ -0,0 +1,512 @@ +# frozen_string_literal: false +# +# httputils.rb -- HTTPUtils Module +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: httputils.rb,v 1.34 2003/06/05 21:34:08 gotoyuzo Exp $ + +require 'socket' +require 'tempfile' + +module WEBrick + CR = "\x0d" # :nodoc: + LF = "\x0a" # :nodoc: + CRLF = "\x0d\x0a" # :nodoc: + + ## + # HTTPUtils provides utility methods for working with the HTTP protocol. + # + # This module is generally used internally by WEBrick + + module HTTPUtils + + ## + # Normalizes a request path. Raises an exception if the path cannot be + # normalized. + + def normalize_path(path) + raise "abnormal path `#{path}'" if path[0] != ?/ + ret = path.dup + + ret.gsub!(%r{/+}o, '/') # // => / + while ret.sub!(%r'/\.(?:/|\Z)', '/'); end # /. => / + while ret.sub!(%r'/(?!\.\./)[^/]+/\.\.(?:/|\Z)', '/'); end # /foo/.. => /foo + + raise "abnormal path `#{path}'" if %r{/\.\.(/|\Z)} =~ ret + ret + end + module_function :normalize_path + + ## + # Default mime types + + DefaultMimeTypes = { + "ai" => "application/postscript", + "asc" => "text/plain", + "avi" => "video/x-msvideo", + "bin" => "application/octet-stream", + "bmp" => "image/bmp", + "class" => "application/octet-stream", + "cer" => "application/pkix-cert", + "crl" => "application/pkix-crl", + "crt" => "application/x-x509-ca-cert", + #"crl" => "application/x-pkcs7-crl", + "css" => "text/css", + "dms" => "application/octet-stream", + "doc" => "application/msword", + "dvi" => "application/x-dvi", + "eps" => "application/postscript", + "etx" => "text/x-setext", + "exe" => "application/octet-stream", + "gif" => "image/gif", + "htm" => "text/html", + "html" => "text/html", + "jpe" => "image/jpeg", + "jpeg" => "image/jpeg", + "jpg" => "image/jpeg", + "js" => "application/javascript", + "json" => "application/json", + "lha" => "application/octet-stream", + "lzh" => "application/octet-stream", + "mjs" => "application/javascript", + "mov" => "video/quicktime", + "mpe" => "video/mpeg", + "mpeg" => "video/mpeg", + "mpg" => "video/mpeg", + "pbm" => "image/x-portable-bitmap", + "pdf" => "application/pdf", + "pgm" => "image/x-portable-graymap", + "png" => "image/png", + "pnm" => "image/x-portable-anymap", + "ppm" => "image/x-portable-pixmap", + "ppt" => "application/vnd.ms-powerpoint", + "ps" => "application/postscript", + "qt" => "video/quicktime", + "ras" => "image/x-cmu-raster", + "rb" => "text/plain", + "rd" => "text/plain", + "rtf" => "application/rtf", + "sgm" => "text/sgml", + "sgml" => "text/sgml", + "svg" => "image/svg+xml", + "tif" => "image/tiff", + "tiff" => "image/tiff", + "txt" => "text/plain", + "wasm" => "application/wasm", + "xbm" => "image/x-xbitmap", + "xhtml" => "text/html", + "xls" => "application/vnd.ms-excel", + "xml" => "text/xml", + "xpm" => "image/x-xpixmap", + "xwd" => "image/x-xwindowdump", + "zip" => "application/zip", + } + + ## + # Loads Apache-compatible mime.types in +file+. + + def load_mime_types(file) + # note: +file+ may be a "| command" for now; some people may + # rely on this, but currently we do not use this method by default. + open(file){ |io| + hash = Hash.new + io.each{ |line| + next if /^#/ =~ line + line.chomp! + mimetype, ext0 = line.split(/\s+/, 2) + next unless ext0 + next if ext0.empty? + ext0.split(/\s+/).each{ |ext| hash[ext] = mimetype } + } + hash + } + end + module_function :load_mime_types + + ## + # Returns the mime type of +filename+ from the list in +mime_tab+. If no + # mime type was found application/octet-stream is returned. + + def mime_type(filename, mime_tab) + suffix1 = (/\.(\w+)$/ =~ filename && $1.downcase) + suffix2 = (/\.(\w+)\.[\w\-]+$/ =~ filename && $1.downcase) + mime_tab[suffix1] || mime_tab[suffix2] || "application/octet-stream" + end + module_function :mime_type + + ## + # Parses an HTTP header +raw+ into a hash of header fields with an Array + # of values. + + def parse_header(raw) + header = Hash.new([].freeze) + field = nil + raw.each_line{|line| + case line + when /^([A-Za-z0-9!\#$%&'*+\-.^_`|~]+):\s*(.*?)\s*\z/om + field, value = $1, $2 + field.downcase! + header[field] = [] unless header.has_key?(field) + header[field] << value + when /^\s+(.*?)\s*\z/om + value = $1 + unless field + raise HTTPStatus::BadRequest, "bad header '#{line}'." + end + header[field][-1] << " " << value + else + raise HTTPStatus::BadRequest, "bad header '#{line}'." + end + } + header.each{|key, values| + values.each(&:strip!) + } + header + end + module_function :parse_header + + ## + # Splits a header value +str+ according to HTTP specification. + + def split_header_value(str) + str.scan(%r'\G((?:"(?:\\.|[^"])+?"|[^",]+)+) + (?:,\s*|\Z)'xn).flatten + end + module_function :split_header_value + + ## + # Parses a Range header value +ranges_specifier+ + + def parse_range_header(ranges_specifier) + if /^bytes=(.*)/ =~ ranges_specifier + byte_range_set = split_header_value($1) + byte_range_set.collect{|range_spec| + case range_spec + when /^(\d+)-(\d+)/ then $1.to_i .. $2.to_i + when /^(\d+)-/ then $1.to_i .. -1 + when /^-(\d+)/ then -($1.to_i) .. -1 + else return nil + end + } + end + end + module_function :parse_range_header + + ## + # Parses q values in +value+ as used in Accept headers. + + def parse_qvalues(value) + tmp = [] + if value + parts = value.split(/,\s*/) + parts.each {|part| + if m = %r{^([^\s,]+?)(?:;\s*q=(\d+(?:\.\d+)?))?$}.match(part) + val = m[1] + q = (m[2] or 1).to_f + tmp.push([val, q]) + end + } + tmp = tmp.sort_by{|val, q| -q} + tmp.collect!{|val, q| val} + end + return tmp + end + module_function :parse_qvalues + + ## + # Removes quotes and escapes from +str+ + + def dequote(str) + ret = (/\A"(.*)"\Z/ =~ str) ? $1 : str.dup + ret.gsub!(/\\(.)/, "\\1") + ret + end + module_function :dequote + + ## + # Quotes and escapes quotes in +str+ + + def quote(str) + '"' << str.gsub(/[\\\"]/o, "\\\1") << '"' + end + module_function :quote + + ## + # Stores multipart form data. FormData objects are created when + # WEBrick::HTTPUtils.parse_form_data is called. + + class FormData < String + EmptyRawHeader = [].freeze # :nodoc: + EmptyHeader = {}.freeze # :nodoc: + + ## + # The name of the form data part + + attr_accessor :name + + ## + # The filename of the form data part + + attr_accessor :filename + + attr_accessor :next_data # :nodoc: + protected :next_data + + ## + # Creates a new FormData object. + # + # +args+ is an Array of form data entries. One FormData will be created + # for each entry. + # + # This is called by WEBrick::HTTPUtils.parse_form_data for you + + def initialize(*args) + @name = @filename = @next_data = nil + if args.empty? + @raw_header = [] + @header = nil + super("") + else + @raw_header = EmptyRawHeader + @header = EmptyHeader + super(args.shift) + unless args.empty? + @next_data = self.class.new(*args) + end + end + end + + ## + # Retrieves the header at the first entry in +key+ + + def [](*key) + begin + @header[key[0].downcase].join(", ") + rescue StandardError, NameError + super + end + end + + ## + # Adds +str+ to this FormData which may be the body, a header or a + # header entry. + # + # This is called by WEBrick::HTTPUtils.parse_form_data for you + + def <<(str) + if @header + super + elsif str == CRLF + @header = HTTPUtils::parse_header(@raw_header.join) + if cd = self['content-disposition'] + if /\s+name="(.*?)"/ =~ cd then @name = $1 end + if /\s+filename="(.*?)"/ =~ cd then @filename = $1 end + end + else + @raw_header << str + end + self + end + + ## + # Adds +data+ at the end of the chain of entries + # + # This is called by WEBrick::HTTPUtils.parse_form_data for you. + + def append_data(data) + tmp = self + while tmp + unless tmp.next_data + tmp.next_data = data + break + end + tmp = tmp.next_data + end + self + end + + ## + # Yields each entry in this FormData + + def each_data + tmp = self + while tmp + next_data = tmp.next_data + yield(tmp) + tmp = next_data + end + end + + ## + # Returns all the FormData as an Array + + def list + ret = [] + each_data{|data| + ret << data.to_s + } + ret + end + + ## + # A FormData will behave like an Array + + alias :to_ary :list + + ## + # This FormData's body + + def to_s + String.new(self) + end + end + + ## + # Parses the query component of a URI in +str+ + + def parse_query(str) + query = Hash.new + if str + str.split(/[&;]/).each{|x| + next if x.empty? + key, val = x.split(/=/,2) + key = unescape_form(key) + val = unescape_form(val.to_s) + val = FormData.new(val) + val.name = key + if query.has_key?(key) + query[key].append_data(val) + next + end + query[key] = val + } + end + query + end + module_function :parse_query + + ## + # Parses form data in +io+ with the given +boundary+ + + def parse_form_data(io, boundary) + boundary_regexp = /\A--#{Regexp.quote(boundary)}(--)?#{CRLF}\z/ + form_data = Hash.new + return form_data unless io + data = nil + io.each_line{|line| + if boundary_regexp =~ line + if data + data.chop! + key = data.name + if form_data.has_key?(key) + form_data[key].append_data(data) + else + form_data[key] = data + end + end + data = FormData.new + next + else + if data + data << line + end + end + } + return form_data + end + module_function :parse_form_data + + ##### + + reserved = ';/?:@&=+$,' + num = '0123456789' + lowalpha = 'abcdefghijklmnopqrstuvwxyz' + upalpha = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' + mark = '-_.!~*\'()' + unreserved = num + lowalpha + upalpha + mark + control = (0x0..0x1f).collect{|c| c.chr }.join + "\x7f" + space = " " + delims = '<>#%"' + unwise = '{}|\\^[]`' + nonascii = (0x80..0xff).collect{|c| c.chr }.join + + module_function + + # :stopdoc: + + def _make_regex(str) /([#{Regexp.escape(str)}])/n end + def _make_regex!(str) /([^#{Regexp.escape(str)}])/n end + def _escape(str, regex) + str = str.b + str.gsub!(regex) {"%%%02X" % $1.ord} + # %-escaped string should contain US-ASCII only + str.force_encoding(Encoding::US_ASCII) + end + def _unescape(str, regex) + str = str.b + str.gsub!(regex) {$1.hex.chr} + # encoding of %-unescaped string is unknown + str + end + + UNESCAPED = _make_regex(control+space+delims+unwise+nonascii) + UNESCAPED_FORM = _make_regex(reserved+control+delims+unwise+nonascii) + NONASCII = _make_regex(nonascii) + ESCAPED = /%([0-9a-fA-F]{2})/ + UNESCAPED_PCHAR = _make_regex!(unreserved+":@&=+$,") + + # :startdoc: + + ## + # Escapes HTTP reserved and unwise characters in +str+ + + def escape(str) + _escape(str, UNESCAPED) + end + + ## + # Unescapes HTTP reserved and unwise characters in +str+ + + def unescape(str) + _unescape(str, ESCAPED) + end + + ## + # Escapes form reserved characters in +str+ + + def escape_form(str) + ret = _escape(str, UNESCAPED_FORM) + ret.gsub!(/ /, "+") + ret + end + + ## + # Unescapes form reserved characters in +str+ + + def unescape_form(str) + _unescape(str.gsub(/\+/, " "), ESCAPED) + end + + ## + # Escapes path +str+ + + def escape_path(str) + result = "" + str.scan(%r{/([^/]*)}).each{|i| + result << "/" << _escape(i[0], UNESCAPED_PCHAR) + } + return result + end + + ## + # Escapes 8 bit characters in +str+ + + def escape8bit(str) + _escape(str, NONASCII) + end + end +end diff --git a/tool/lib/webrick/httpversion.rb b/tool/lib/webrick/httpversion.rb new file mode 100644 index 0000000000..8a251944a2 --- /dev/null +++ b/tool/lib/webrick/httpversion.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: false +#-- +# HTTPVersion.rb -- presentation of HTTP version +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: httpversion.rb,v 1.5 2002/09/21 12:23:37 gotoyuzo Exp $ + +module WEBrick + + ## + # Represents an HTTP protocol version + + class HTTPVersion + include Comparable + + ## + # The major protocol version number + + attr_accessor :major + + ## + # The minor protocol version number + + attr_accessor :minor + + ## + # Converts +version+ into an HTTPVersion + + def self.convert(version) + version.is_a?(self) ? version : new(version) + end + + ## + # Creates a new HTTPVersion from +version+. + + def initialize(version) + case version + when HTTPVersion + @major, @minor = version.major, version.minor + when String + if /^(\d+)\.(\d+)$/ =~ version + @major, @minor = $1.to_i, $2.to_i + end + end + if @major.nil? || @minor.nil? + raise ArgumentError, + format("cannot convert %s into %s", version.class, self.class) + end + end + + ## + # Compares this version with +other+ according to the HTTP specification + # rules. + + def <=>(other) + unless other.is_a?(self.class) + other = self.class.new(other) + end + if (ret = @major <=> other.major) == 0 + return @minor <=> other.minor + end + return ret + end + + ## + # The HTTP version as show in the HTTP request and response. For example, + # "1.1" + + def to_s + format("%d.%d", @major, @minor) + end + end +end diff --git a/tool/lib/webrick/log.rb b/tool/lib/webrick/log.rb new file mode 100644 index 0000000000..2c1fdfe602 --- /dev/null +++ b/tool/lib/webrick/log.rb @@ -0,0 +1,156 @@ +# frozen_string_literal: false +#-- +# log.rb -- Log Class +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: log.rb,v 1.26 2002/10/06 17:06:10 gotoyuzo Exp $ + +module WEBrick + + ## + # A generic logging class + + class BasicLog + + # Fatal log level which indicates a server crash + + FATAL = 1 + + # Error log level which indicates a recoverable error + + ERROR = 2 + + # Warning log level which indicates a possible problem + + WARN = 3 + + # Information log level which indicates possibly useful information + + INFO = 4 + + # Debugging error level for messages used in server development or + # debugging + + DEBUG = 5 + + # log-level, messages above this level will be logged + attr_accessor :level + + ## + # Initializes a new logger for +log_file+ that outputs messages at +level+ + # or higher. +log_file+ can be a filename, an IO-like object that + # responds to #<< or nil which outputs to $stderr. + # + # If no level is given INFO is chosen by default + + def initialize(log_file=nil, level=nil) + @level = level || INFO + case log_file + when String + @log = File.open(log_file, "a+") + @log.sync = true + @opened = true + when NilClass + @log = $stderr + else + @log = log_file # requires "<<". (see BasicLog#log) + end + end + + ## + # Closes the logger (also closes the log device associated to the logger) + def close + @log.close if @opened + @log = nil + end + + ## + # Logs +data+ at +level+ if the given level is above the current log + # level. + + def log(level, data) + if @log && level <= @level + data += "\n" if /\n\Z/ !~ data + @log << data + end + end + + ## + # Synonym for log(INFO, obj.to_s) + def <<(obj) + log(INFO, obj.to_s) + end + + # Shortcut for logging a FATAL message + def fatal(msg) log(FATAL, "FATAL " << format(msg)); end + # Shortcut for logging an ERROR message + def error(msg) log(ERROR, "ERROR " << format(msg)); end + # Shortcut for logging a WARN message + def warn(msg) log(WARN, "WARN " << format(msg)); end + # Shortcut for logging an INFO message + def info(msg) log(INFO, "INFO " << format(msg)); end + # Shortcut for logging a DEBUG message + def debug(msg) log(DEBUG, "DEBUG " << format(msg)); end + + # Will the logger output FATAL messages? + def fatal?; @level >= FATAL; end + # Will the logger output ERROR messages? + def error?; @level >= ERROR; end + # Will the logger output WARN messages? + def warn?; @level >= WARN; end + # Will the logger output INFO messages? + def info?; @level >= INFO; end + # Will the logger output DEBUG messages? + def debug?; @level >= DEBUG; end + + private + + ## + # Formats +arg+ for the logger + # + # * If +arg+ is an Exception, it will format the error message and + # the back trace. + # * If +arg+ responds to #to_str, it will return it. + # * Otherwise it will return +arg+.inspect. + def format(arg) + if arg.is_a?(Exception) + "#{arg.class}: #{AccessLog.escape(arg.message)}\n\t" << + arg.backtrace.join("\n\t") << "\n" + elsif arg.respond_to?(:to_str) + AccessLog.escape(arg.to_str) + else + arg.inspect + end + end + end + + ## + # A logging class that prepends a timestamp to each message. + + class Log < BasicLog + # Format of the timestamp which is applied to each logged line. The + # default is <tt>"[%Y-%m-%d %H:%M:%S]"</tt> + attr_accessor :time_format + + ## + # Same as BasicLog#initialize + # + # You can set the timestamp format through #time_format + def initialize(log_file=nil, level=nil) + super(log_file, level) + @time_format = "[%Y-%m-%d %H:%M:%S]" + end + + ## + # Same as BasicLog#log + def log(level, data) + tmp = Time.now.strftime(@time_format) + tmp << " " << data + super(level, tmp) + end + end +end diff --git a/tool/lib/webrick/server.rb b/tool/lib/webrick/server.rb new file mode 100644 index 0000000000..fd6b7a61b5 --- /dev/null +++ b/tool/lib/webrick/server.rb @@ -0,0 +1,381 @@ +# frozen_string_literal: false +# +# server.rb -- GenericServer Class +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: server.rb,v 1.62 2003/07/22 19:20:43 gotoyuzo Exp $ + +require 'socket' +require_relative 'config' +require_relative 'log' + +module WEBrick + + ## + # Server error exception + + class ServerError < StandardError; end + + ## + # Base server class + + class SimpleServer + + ## + # A SimpleServer only yields when you start it + + def SimpleServer.start + yield + end + end + + ## + # A generic module for daemonizing a process + + class Daemon + + ## + # Performs the standard operations for daemonizing a process. Runs a + # block, if given. + + def Daemon.start + Process.daemon + File.umask(0) + yield if block_given? + end + end + + ## + # Base TCP server class. You must subclass GenericServer and provide a #run + # method. + + class GenericServer + + ## + # The server status. One of :Stop, :Running or :Shutdown + + attr_reader :status + + ## + # The server configuration + + attr_reader :config + + ## + # The server logger. This is independent from the HTTP access log. + + attr_reader :logger + + ## + # Tokens control the number of outstanding clients. The + # <code>:MaxClients</code> configuration sets this. + + attr_reader :tokens + + ## + # Sockets listening for connections. + + attr_reader :listeners + + ## + # Creates a new generic server from +config+. The default configuration + # comes from +default+. + + def initialize(config={}, default=Config::General) + @config = default.dup.update(config) + @status = :Stop + @config[:Logger] ||= Log::new + @logger = @config[:Logger] + + @tokens = Thread::SizedQueue.new(@config[:MaxClients]) + @config[:MaxClients].times{ @tokens.push(nil) } + + webrickv = WEBrick::VERSION + rubyv = "#{RUBY_VERSION} (#{RUBY_RELEASE_DATE}) [#{RUBY_PLATFORM}]" + @logger.info("WEBrick #{webrickv}") + @logger.info("ruby #{rubyv}") + + @listeners = [] + @shutdown_pipe = nil + unless @config[:DoNotListen] + raise ArgumentError, "Port must an integer" unless @config[:Port].to_s == @config[:Port].to_i.to_s + + @config[:Port] = @config[:Port].to_i + if @config[:Listen] + warn(":Listen option is deprecated; use GenericServer#listen", uplevel: 1) + end + listen(@config[:BindAddress], @config[:Port]) + if @config[:Port] == 0 + @config[:Port] = @listeners[0].addr[1] + end + end + end + + ## + # Retrieves +key+ from the configuration + + def [](key) + @config[key] + end + + ## + # Adds listeners from +address+ and +port+ to the server. See + # WEBrick::Utils::create_listeners for details. + + def listen(address, port) + @listeners += Utils::create_listeners(address, port) + end + + ## + # Starts the server and runs the +block+ for each connection. This method + # does not return until the server is stopped from a signal handler or + # another thread using #stop or #shutdown. + # + # If the block raises a subclass of StandardError the exception is logged + # and ignored. If an IOError or Errno::EBADF exception is raised the + # exception is ignored. If an Exception subclass is raised the exception + # is logged and re-raised which stops the server. + # + # To completely shut down a server call #shutdown from ensure: + # + # server = WEBrick::GenericServer.new + # # or WEBrick::HTTPServer.new + # + # begin + # server.start + # ensure + # server.shutdown + # end + + def start(&block) + raise ServerError, "already started." if @status != :Stop + server_type = @config[:ServerType] || SimpleServer + + setup_shutdown_pipe + + server_type.start{ + @logger.info \ + "#{self.class}#start: pid=#{$$} port=#{@config[:Port]}" + @status = :Running + call_callback(:StartCallback) + + shutdown_pipe = @shutdown_pipe + + thgroup = ThreadGroup.new + begin + while @status == :Running + begin + sp = shutdown_pipe[0] + if svrs = IO.select([sp, *@listeners]) + if svrs[0].include? sp + # swallow shutdown pipe + buf = String.new + nil while String === + sp.read_nonblock([sp.nread, 8].max, buf, exception: false) + break + end + svrs[0].each{|svr| + @tokens.pop # blocks while no token is there. + if sock = accept_client(svr) + unless config[:DoNotReverseLookup].nil? + sock.do_not_reverse_lookup = !!config[:DoNotReverseLookup] + end + th = start_thread(sock, &block) + th[:WEBrickThread] = true + thgroup.add(th) + else + @tokens.push(nil) + end + } + end + rescue Errno::EBADF, Errno::ENOTSOCK, IOError => ex + # if the listening socket was closed in GenericServer#shutdown, + # IO::select raise it. + rescue StandardError => ex + msg = "#{ex.class}: #{ex.message}\n\t#{ex.backtrace[0]}" + @logger.error msg + rescue Exception => ex + @logger.fatal ex + raise + end + end + ensure + cleanup_shutdown_pipe(shutdown_pipe) + cleanup_listener + @status = :Shutdown + @logger.info "going to shutdown ..." + thgroup.list.each{|th| th.join if th[:WEBrickThread] } + call_callback(:StopCallback) + @logger.info "#{self.class}#start done." + @status = :Stop + end + } + end + + ## + # Stops the server from accepting new connections. + + def stop + if @status == :Running + @status = :Shutdown + end + + alarm_shutdown_pipe {|f| f.write_nonblock("\0")} + end + + ## + # Shuts down the server and all listening sockets. New listeners must be + # provided to restart the server. + + def shutdown + stop + + alarm_shutdown_pipe(&:close) + end + + ## + # You must subclass GenericServer and implement \#run which accepts a TCP + # client socket + + def run(sock) + @logger.fatal "run() must be provided by user." + end + + private + + # :stopdoc: + + ## + # Accepts a TCP client socket from the TCP server socket +svr+ and returns + # the client socket. + + def accept_client(svr) + case sock = svr.to_io.accept_nonblock(exception: false) + when :wait_readable + nil + else + if svr.respond_to?(:start_immediately) + sock = OpenSSL::SSL::SSLSocket.new(sock, ssl_context) + sock.sync_close = true + # we cannot do OpenSSL::SSL::SSLSocket#accept here because + # a slow client can prevent us from accepting connections + # from other clients + end + sock + end + rescue Errno::ECONNRESET, Errno::ECONNABORTED, + Errno::EPROTO, Errno::EINVAL + nil + rescue StandardError => ex + msg = "#{ex.class}: #{ex.message}\n\t#{ex.backtrace[0]}" + @logger.error msg + nil + end + + ## + # Starts a server thread for the client socket +sock+ that runs the given + # +block+. + # + # Sets the socket to the <code>:WEBrickSocket</code> thread local variable + # in the thread. + # + # If any errors occur in the block they are logged and handled. + + def start_thread(sock, &block) + Thread.start{ + begin + Thread.current[:WEBrickSocket] = sock + begin + addr = sock.peeraddr + @logger.debug "accept: #{addr[3]}:#{addr[1]}" + rescue SocketError + @logger.debug "accept: <address unknown>" + raise + end + if sock.respond_to?(:sync_close=) && @config[:SSLStartImmediately] + WEBrick::Utils.timeout(@config[:RequestTimeout]) do + begin + sock.accept # OpenSSL::SSL::SSLSocket#accept + rescue Errno::ECONNRESET, Errno::ECONNABORTED, + Errno::EPROTO, Errno::EINVAL + Thread.exit + end + end + end + call_callback(:AcceptCallback, sock) + block ? block.call(sock) : run(sock) + rescue Errno::ENOTCONN + @logger.debug "Errno::ENOTCONN raised" + rescue ServerError => ex + msg = "#{ex.class}: #{ex.message}\n\t#{ex.backtrace[0]}" + @logger.error msg + rescue Exception => ex + @logger.error ex + ensure + @tokens.push(nil) + Thread.current[:WEBrickSocket] = nil + if addr + @logger.debug "close: #{addr[3]}:#{addr[1]}" + else + @logger.debug "close: <address unknown>" + end + sock.close + end + } + end + + ## + # Calls the callback +callback_name+ from the configuration with +args+ + + def call_callback(callback_name, *args) + @config[callback_name]&.call(*args) + end + + def setup_shutdown_pipe + return @shutdown_pipe ||= IO.pipe + end + + def cleanup_shutdown_pipe(shutdown_pipe) + @shutdown_pipe = nil + shutdown_pipe&.each(&:close) + end + + def alarm_shutdown_pipe + _, pipe = @shutdown_pipe # another thread may modify @shutdown_pipe. + if pipe + if !pipe.closed? + begin + yield pipe + rescue IOError # closed by another thread. + end + end + end + end + + def cleanup_listener + @listeners.each{|s| + if @logger.debug? + addr = s.addr + @logger.debug("close TCPSocket(#{addr[2]}, #{addr[1]})") + end + begin + s.shutdown + rescue Errno::ENOTCONN + # when `Errno::ENOTCONN: Socket is not connected' on some platforms, + # call #close instead of #shutdown. + # (ignore @config[:ShutdownSocketWithoutClose]) + s.close + else + unless @config[:ShutdownSocketWithoutClose] + s.close + end + end + } + @listeners.clear + end + end # end of GenericServer +end diff --git a/tool/lib/webrick/ssl.rb b/tool/lib/webrick/ssl.rb new file mode 100644 index 0000000000..e448095a12 --- /dev/null +++ b/tool/lib/webrick/ssl.rb @@ -0,0 +1,215 @@ +# frozen_string_literal: false +# +# ssl.rb -- SSL/TLS enhancement for GenericServer +# +# Copyright (c) 2003 GOTOU Yuuzou All rights reserved. +# +# $Id$ + +require 'webrick' +require 'openssl' + +module WEBrick + module Config + svrsoft = General[:ServerSoftware] + osslv = ::OpenSSL::OPENSSL_VERSION.split[1] + + ## + # Default SSL server configuration. + # + # WEBrick can automatically create a self-signed certificate if + # <code>:SSLCertName</code> is set. For more information on the various + # SSL options see OpenSSL::SSL::SSLContext. + # + # :ServerSoftware :: + # The server software name used in the Server: header. + # :SSLEnable :: false, + # Enable SSL for this server. Defaults to false. + # :SSLCertificate :: + # The SSL certificate for the server. + # :SSLPrivateKey :: + # The SSL private key for the server certificate. + # :SSLClientCA :: nil, + # Array of certificates that will be sent to the client. + # :SSLExtraChainCert :: nil, + # Array of certificates that will be added to the certificate chain + # :SSLCACertificateFile :: nil, + # Path to a CA certificate file + # :SSLCACertificatePath :: nil, + # Path to a directory containing CA certificates + # :SSLCertificateStore :: nil, + # OpenSSL::X509::Store used for certificate validation of the client + # :SSLTmpDhCallback :: nil, + # Callback invoked when DH parameters are required. + # :SSLVerifyClient :: + # Sets whether the client is verified. This defaults to VERIFY_NONE + # which is typical for an HTTPS server. + # :SSLVerifyDepth :: + # Number of CA certificates to walk when verifying a certificate chain + # :SSLVerifyCallback :: + # Custom certificate verification callback + # :SSLServerNameCallback:: + # Custom servername indication callback + # :SSLTimeout :: + # Maximum session lifetime + # :SSLOptions :: + # Various SSL options + # :SSLCiphers :: + # Ciphers to be used + # :SSLStartImmediately :: + # Immediately start SSL upon connection? Defaults to true + # :SSLCertName :: + # SSL certificate name. Must be set to enable automatic certificate + # creation. + # :SSLCertComment :: + # Comment used during automatic certificate creation. + + SSL = { + :ServerSoftware => "#{svrsoft} OpenSSL/#{osslv}", + :SSLEnable => false, + :SSLCertificate => nil, + :SSLPrivateKey => nil, + :SSLClientCA => nil, + :SSLExtraChainCert => nil, + :SSLCACertificateFile => nil, + :SSLCACertificatePath => nil, + :SSLCertificateStore => nil, + :SSLTmpDhCallback => nil, + :SSLVerifyClient => ::OpenSSL::SSL::VERIFY_NONE, + :SSLVerifyDepth => nil, + :SSLVerifyCallback => nil, # custom verification + :SSLTimeout => nil, + :SSLOptions => nil, + :SSLCiphers => nil, + :SSLStartImmediately => true, + # Must specify if you use auto generated certificate. + :SSLCertName => nil, + :SSLCertComment => "Generated by Ruby/OpenSSL" + } + General.update(SSL) + end + + module Utils + ## + # Creates a self-signed certificate with the given number of +bits+, + # the issuer +cn+ and a +comment+ to be stored in the certificate. + + def create_self_signed_cert(bits, cn, comment) + rsa = OpenSSL::PKey::RSA.new(bits){|p, n| + case p + when 0; $stderr.putc "." # BN_generate_prime + when 1; $stderr.putc "+" # BN_generate_prime + when 2; $stderr.putc "*" # searching good prime, + # n = #of try, + # but also data from BN_generate_prime + when 3; $stderr.putc "\n" # found good prime, n==0 - p, n==1 - q, + # but also data from BN_generate_prime + else; $stderr.putc "*" # BN_generate_prime + end + } + cert = OpenSSL::X509::Certificate.new + cert.version = 2 + cert.serial = 1 + name = (cn.kind_of? String) ? OpenSSL::X509::Name.parse(cn) + : OpenSSL::X509::Name.new(cn) + cert.subject = name + cert.issuer = name + cert.not_before = Time.now + cert.not_after = Time.now + (365*24*60*60) + cert.public_key = rsa.public_key + + ef = OpenSSL::X509::ExtensionFactory.new(nil,cert) + ef.issuer_certificate = cert + cert.extensions = [ + ef.create_extension("basicConstraints","CA:FALSE"), + ef.create_extension("keyUsage", "keyEncipherment, digitalSignature, keyAgreement, dataEncipherment"), + ef.create_extension("subjectKeyIdentifier", "hash"), + ef.create_extension("extendedKeyUsage", "serverAuth"), + ef.create_extension("nsComment", comment), + ] + aki = ef.create_extension("authorityKeyIdentifier", + "keyid:always,issuer:always") + cert.add_extension(aki) + cert.sign(rsa, "SHA256") + + return [ cert, rsa ] + end + module_function :create_self_signed_cert + end + + ## + #-- + # Updates WEBrick::GenericServer with SSL functionality + + class GenericServer + + ## + # SSL context for the server when run in SSL mode + + def ssl_context # :nodoc: + @ssl_context ||= begin + if @config[:SSLEnable] + ssl_context = setup_ssl_context(@config) + @logger.info("\n" + @config[:SSLCertificate].to_text) + ssl_context + end + end + end + + undef listen + + ## + # Updates +listen+ to enable SSL when the SSL configuration is active. + + def listen(address, port) # :nodoc: + listeners = Utils::create_listeners(address, port) + if @config[:SSLEnable] + listeners.collect!{|svr| + ssvr = ::OpenSSL::SSL::SSLServer.new(svr, ssl_context) + ssvr.start_immediately = @config[:SSLStartImmediately] + ssvr + } + end + @listeners += listeners + setup_shutdown_pipe + end + + ## + # Sets up an SSL context for +config+ + + def setup_ssl_context(config) # :nodoc: + unless config[:SSLCertificate] + cn = config[:SSLCertName] + comment = config[:SSLCertComment] + cert, key = Utils::create_self_signed_cert(2048, cn, comment) + config[:SSLCertificate] = cert + config[:SSLPrivateKey] = key + end + ctx = OpenSSL::SSL::SSLContext.new + ctx.key = config[:SSLPrivateKey] + ctx.cert = config[:SSLCertificate] + ctx.client_ca = config[:SSLClientCA] + ctx.extra_chain_cert = config[:SSLExtraChainCert] + ctx.ca_file = config[:SSLCACertificateFile] + ctx.ca_path = config[:SSLCACertificatePath] + ctx.cert_store = config[:SSLCertificateStore] + ctx.tmp_dh_callback = config[:SSLTmpDhCallback] + ctx.verify_mode = config[:SSLVerifyClient] + ctx.verify_depth = config[:SSLVerifyDepth] + ctx.verify_callback = config[:SSLVerifyCallback] + ctx.servername_cb = config[:SSLServerNameCallback] || proc { |args| ssl_servername_callback(*args) } + ctx.timeout = config[:SSLTimeout] + ctx.options = config[:SSLOptions] + ctx.ciphers = config[:SSLCiphers] + ctx + end + + ## + # ServerNameIndication callback + + def ssl_servername_callback(sslsocket, hostname = nil) + # default + end + + end +end diff --git a/tool/lib/webrick/utils.rb b/tool/lib/webrick/utils.rb new file mode 100644 index 0000000000..a96d6f03fd --- /dev/null +++ b/tool/lib/webrick/utils.rb @@ -0,0 +1,265 @@ +# frozen_string_literal: false +# +# utils.rb -- Miscellaneous utilities +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou +# Copyright (c) 2002 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: utils.rb,v 1.10 2003/02/16 22:22:54 gotoyuzo Exp $ + +require 'socket' +require 'io/nonblock' +require 'etc' + +module WEBrick + module Utils + ## + # Sets IO operations on +io+ to be non-blocking + def set_non_blocking(io) + io.nonblock = true if io.respond_to?(:nonblock=) + end + module_function :set_non_blocking + + ## + # Sets the close on exec flag for +io+ + def set_close_on_exec(io) + io.close_on_exec = true if io.respond_to?(:close_on_exec=) + end + module_function :set_close_on_exec + + ## + # Changes the process's uid and gid to the ones of +user+ + def su(user) + if pw = Etc.getpwnam(user) + Process::initgroups(user, pw.gid) + Process::Sys::setgid(pw.gid) + Process::Sys::setuid(pw.uid) + else + warn("WEBrick::Utils::su doesn't work on this platform", uplevel: 1) + end + end + module_function :su + + ## + # The server hostname + def getservername + Socket::gethostname + end + module_function :getservername + + ## + # Creates TCP server sockets bound to +address+:+port+ and returns them. + # + # It will create IPV4 and IPV6 sockets on all interfaces. + def create_listeners(address, port) + unless port + raise ArgumentError, "must specify port" + end + sockets = Socket.tcp_server_sockets(address, port) + sockets = sockets.map {|s| + s.autoclose = false + ts = TCPServer.for_fd(s.fileno) + s.close + ts + } + return sockets + end + module_function :create_listeners + + ## + # Characters used to generate random strings + RAND_CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + + "0123456789" + + "abcdefghijklmnopqrstuvwxyz" + + ## + # Generates a random string of length +len+ + def random_string(len) + rand_max = RAND_CHARS.bytesize + ret = "" + len.times{ ret << RAND_CHARS[rand(rand_max)] } + ret + end + module_function :random_string + + ########### + + require "timeout" + require "singleton" + + ## + # Class used to manage timeout handlers across multiple threads. + # + # Timeout handlers should be managed by using the class methods which are + # synchronized. + # + # id = TimeoutHandler.register(10, Timeout::Error) + # begin + # sleep 20 + # puts 'foo' + # ensure + # TimeoutHandler.cancel(id) + # end + # + # will raise Timeout::Error + # + # id = TimeoutHandler.register(10, Timeout::Error) + # begin + # sleep 5 + # puts 'foo' + # ensure + # TimeoutHandler.cancel(id) + # end + # + # will print 'foo' + # + class TimeoutHandler + include Singleton + + ## + # Mutex used to synchronize access across threads + TimeoutMutex = Thread::Mutex.new # :nodoc: + + ## + # Registers a new timeout handler + # + # +time+:: Timeout in seconds + # +exception+:: Exception to raise when timeout elapsed + def TimeoutHandler.register(seconds, exception) + at = Process.clock_gettime(Process::CLOCK_MONOTONIC) + seconds + instance.register(Thread.current, at, exception) + end + + ## + # Cancels the timeout handler +id+ + def TimeoutHandler.cancel(id) + instance.cancel(Thread.current, id) + end + + def self.terminate + instance.terminate + end + + ## + # Creates a new TimeoutHandler. You should use ::register and ::cancel + # instead of creating the timeout handler directly. + def initialize + TimeoutMutex.synchronize{ + @timeout_info = Hash.new + } + @queue = Thread::Queue.new + @watcher = nil + end + + # :nodoc: + private \ + def watch + to_interrupt = [] + while true + now = Process.clock_gettime(Process::CLOCK_MONOTONIC) + wakeup = nil + to_interrupt.clear + TimeoutMutex.synchronize{ + @timeout_info.each {|thread, ary| + next unless ary + ary.each{|info| + time, exception = *info + if time < now + to_interrupt.push [thread, info.object_id, exception] + elsif !wakeup || time < wakeup + wakeup = time + end + } + } + } + to_interrupt.each {|arg| interrupt(*arg)} + if !wakeup + @queue.pop + elsif (wakeup -= now) > 0 + begin + (th = Thread.start {@queue.pop}).join(wakeup) + ensure + th&.kill&.join + end + end + @queue.clear + end + end + + # :nodoc: + private \ + def watcher + (w = @watcher)&.alive? and return w # usual case + TimeoutMutex.synchronize{ + (w = @watcher)&.alive? and next w # pathological check + @watcher = Thread.start(&method(:watch)) + } + end + + ## + # Interrupts the timeout handler +id+ and raises +exception+ + def interrupt(thread, id, exception) + if cancel(thread, id) && thread.alive? + thread.raise(exception, "execution timeout") + end + end + + ## + # Registers a new timeout handler + # + # +time+:: Timeout in seconds + # +exception+:: Exception to raise when timeout elapsed + def register(thread, time, exception) + info = nil + TimeoutMutex.synchronize{ + (@timeout_info[thread] ||= []) << (info = [time, exception]) + } + @queue.push nil + watcher + return info.object_id + end + + ## + # Cancels the timeout handler +id+ + def cancel(thread, id) + TimeoutMutex.synchronize{ + if ary = @timeout_info[thread] + ary.delete_if{|info| info.object_id == id } + if ary.empty? + @timeout_info.delete(thread) + end + return true + end + return false + } + end + + ## + def terminate + TimeoutMutex.synchronize{ + @timeout_info.clear + @watcher&.kill&.join + } + end + end + + ## + # Executes the passed block and raises +exception+ if execution takes more + # than +seconds+. + # + # If +seconds+ is zero or nil, simply executes the block + def timeout(seconds, exception=Timeout::Error) + return yield if seconds.nil? or seconds.zero? + # raise ThreadError, "timeout within critical session" if Thread.critical + id = TimeoutHandler.register(seconds, exception) + begin + yield(seconds) + ensure + TimeoutHandler.cancel(id) + end + end + module_function :timeout + end +end diff --git a/tool/lib/webrick/version.rb b/tool/lib/webrick/version.rb new file mode 100644 index 0000000000..b62988bdbb --- /dev/null +++ b/tool/lib/webrick/version.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: false +#-- +# version.rb -- version and release date +# +# Author: IPR -- Internet Programming with Ruby -- writers +# Copyright (c) 2000 TAKAHASHI Masayoshi, GOTOU YUUZOU +# Copyright (c) 2003 Internet Programming with Ruby writers. All rights +# reserved. +# +# $IPR: version.rb,v 1.74 2003/07/22 19:20:43 gotoyuzo Exp $ + +module WEBrick + + ## + # The WEBrick version + + VERSION = "1.7.0" +end diff --git a/tool/lib/zombie_hunter.rb b/tool/lib/zombie_hunter.rb new file mode 100644 index 0000000000..33bc467941 --- /dev/null +++ b/tool/lib/zombie_hunter.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module ZombieHunter + def after_teardown + super + assert_empty(Process.waitall) + end +end + +Test::Unit::TestCase.include ZombieHunter |