diff options
Diffstat (limited to 'tool')
59 files changed, 10674 insertions, 0 deletions
diff --git a/tool/asm_parse.rb b/tool/asm_parse.rb new file mode 100644 index 0000000000..32882be3ad --- /dev/null +++ b/tool/asm_parse.rb @@ -0,0 +1,53 @@ +# YARV tool to parse assembly output. + +stat = {} + +while line = ARGF.gets + if /\[start\] (\w+)/ =~ line + name = $1 + puts '--------------------------------------------------------------' + puts line + size = 0 + len = 0 + + while line = ARGF.gets + if /\[start\] (\w+)/ =~ line + puts "\t; # length: #{len}, size: #{size}" + puts "\t; # !!" + stat[name] = [len, size] + # + name = $1 + puts '--------------------------------------------------------------' + puts line + size = 0 + len = 0 + next + end + + unless /(\ALM)|(\ALB)|(\A\.)|(\A\/)/ =~ line + puts line + if /\[length = (\d+)\]/ =~ line + len += $1.to_i + size += 1 + end + end + + + if /__NEXT_INSN__/ !~ line && /\[end \] (\w+)/ =~ line + ename = $1 + if name != ename + puts "!! start with #{name}, but end with #{ename}" + end + stat[ename] = [len, size] + puts "\t; # length: #{len}, size: #{size}" + break + end + end + end +end + +stat.sort_by{|a, b| -b[0] * 1000 - a[0]}.each{|a, b| + puts "#{a}\t#{b.join("\t")}" +} +puts "total length :\t#{stat.inject(0){|r, e| r+e[1][0]}}" +puts "total size :\t#{stat.inject(0){|r, e| r+e[1][1]}}" diff --git a/tool/bisect.sh b/tool/bisect.sh new file mode 100755 index 0000000000..d486c9cfc6 --- /dev/null +++ b/tool/bisect.sh @@ -0,0 +1,45 @@ +#!/bin/sh +# usage: +# edit $(srcdir)/test.rb +# git bisect start `git svn find-rev <rBADREV>` `git svn find-rev <rGOODREV>` +# cd <builddir> +# make bisect (or bisect-ruby for full ruby) + +if [ "x" = "x$MAKE" ]; then + MAKE=make +fi + +case $1 in + miniruby | ruby ) # (miniruby|ruby) <srcdir> + srcdir="$2" + builddir=`pwd` # assume pwd is builddir + path="$builddir/_bisect.sh" + echo "path: $path" + cp "$0" "$path" + cd "$srcdir" + set -x + exec git bisect run "$path" "run-$1" + ;; + run-miniruby ) + $MAKE srcs || exit 125 + cd "${0%/*}" || exit 125 # assume a copy of this script is in builddir + $MAKE Makefile || exit 125 + $MAKE mini || exit 125 + $MAKE run || exit 1 + ;; + run-ruby ) + $MAKE srcs || exit 125 + cd "${0%/*}" || exit 125 # assume a copy of this script is in builddir + $MAKE Makefile || exit 125 + $MAKE program || exit 125 + $MAKE runruby || exit 1 + ;; + "" ) + echo foo bar + ;; + * ) + echo unknown command "'$1'" 1>&2 + exit 1 + ;; +esac +exit 0 diff --git a/tool/build-transcode b/tool/build-transcode new file mode 100755 index 0000000000..fa71155530 --- /dev/null +++ b/tool/build-transcode @@ -0,0 +1,16 @@ +#!/bin/sh + +[ "$1" -a -d "$1" ] && { cd "$1" || exit $?; } && shift +[ "$#" = 0 ] && set enc/trans/*.trans +for src; do + case "$src" in + *.trans) + c="`dirname $src`/`basename $src .trans`.c" + ${BASERUBY-ruby} tool/transcode-tblgen.rb -vo "$c" "$src" + ;; + *) + echo "$0: don't know how to deal with $src" + continue + ;; + esac +done diff --git a/tool/change_maker.rb b/tool/change_maker.rb new file mode 100755 index 0000000000..395bd34990 --- /dev/null +++ b/tool/change_maker.rb @@ -0,0 +1,47 @@ +#! ./miniruby + +# Used by "make change" to generate a list of files for a Changelog entry. +# Run it via "make change" in the Ruby root directory. + +$:.unshift(File.expand_path("../../lib", __FILE__)) +require File.expand_path("../vcs", __FILE__) + +def diff2index(cmd, *argv) + lines = [] + path = nil + output = `#{cmd} #{argv.join(" ")}` + if defined? Encoding::BINARY + output.force_encoding Encoding::BINARY + end + output.each_line do |line| + case line + when /^Index: (\S*)/, /^diff --git [a-z]\/(\S*) [a-z]\/\1/ + path = $1 + when /^@@\s*-[,\d]+ +\+(\d+)[,\d]*\s*@@(?: +([A-Za-z_][A-Za-z_0-9 ]*[A-Za-z_0-9]))?/ + line = $1.to_i + ent = "\t* #{path}" + ent << " (#{$2})" if $2 + lines << "#{ent}:" + end + end + lines.uniq! + lines.empty? ? nil : lines +end + +vcs = begin + VCS.detect(".") +rescue VCS::NotFoundError + nil +end + +case vcs +when VCS::SVN + cmd = "svn diff --diff-cmd=diff -x-pU0" + change = diff2index(cmd, ARGV) +when VCS::GIT + cmd = "git diff -U0" + change = diff2index(cmd, ARGV) || diff2index(cmd, "--cached", ARGV) +else + abort "does not seem to be under a vcs" +end +puts change if change diff --git a/tool/checksum.rb b/tool/checksum.rb new file mode 100755 index 0000000000..0de54a314d --- /dev/null +++ b/tool/checksum.rb @@ -0,0 +1,72 @@ +#!ruby + +require_relative 'vpath' + +class Checksum + def initialize(vpath) + @vpath = vpath + end + + attr_reader :source, :target + + def source=(source) + @source = source + @checksum = File.basename(source, ".*") + ".chksum" + end + + def target=(target) + @target = target + end + + def update? + src = @vpath.read(@source) + @len = src.length + @sum = src.sum + return false unless @vpath.search(File.method(:exist?), @target) + begin + data = @vpath.read(@checksum) + rescue + return false + else + return false unless data[/src="([0-9a-z_.-]+)",/, 1] == @source + return false unless @len == data[/\blen=(\d+)/, 1].to_i + return false unless @sum == data[/\bchecksum=(\d+)/, 1].to_i + return true + end + end + + def update! + open(@checksum, "wb") {|f| + f.puts("src=\"#{@source}\", len=#{@len}, checksum=#{@sum}") + } + end + + def update + return true if update? + update! if ret = yield(self) + ret + end + + def copy(name) + @vpath.open(name, "rb") {|f| + IO.copy_stream(f, name) + } + true + end + + def make(*args) + system(@make, *args) + end + + def def_options(opt = (require 'optparse'; OptionParser.new)) + @vpath.def_options(opt) + opt.on("--make=PATH") {|v| @make = v} + opt + end + + def self.update(argv) + k = new(VPath.new) + k.source, k.target, *argv = k.def_options.parse(*argv) + k.update {|k| yield(k, *argv)} + end +end diff --git a/tool/colorize.rb b/tool/colorize.rb new file mode 100644 index 0000000000..1cbbeef543 --- /dev/null +++ b/tool/colorize.rb @@ -0,0 +1,41 @@ +# frozen-string-literal: true + +class Colorize + def initialize(color = nil) + @colors = @reset = nil + if color or (color == nil && STDOUT.tty?) + if (/\A\e\[.*m\z/ =~ IO.popen("tput smso", "r", err: IO::NULL, &:read) rescue nil) + @beg = "\e[" + @colors = (colors = ENV['TEST_COLORS']) ? Hash[colors.scan(/(\w+)=([^:\n]*)/)] : {} + @reset = "#{@beg}m" + end + end + self + end + + DEFAULTS = { + "pass"=>"32", "fail"=>"31;1", "skip"=>"33;1", + "black"=>"30", "red"=>"31", "green"=>"32", "yellow"=>"33", + "blue"=>"34", "magenta"=>"35", "cyan"=>"36", "white"=>"37", + } + + def decorate(str, name) + if @colors and color = (@colors[name] || DEFAULTS[name]) + "#{@beg}#{color}m#{str}#{@reset}" + else + str + end + end + + DEFAULTS.each_key do |name| + define_method(name) {|str| + decorate(str, name) + } + end +end + +if $0 == __FILE__ + colorize = Colorize.new + col = ARGV.shift + ARGV.each {|str| puts colorize.decorate(str, col)} +end diff --git a/tool/downloader.rb b/tool/downloader.rb new file mode 100644 index 0000000000..335fe97762 --- /dev/null +++ b/tool/downloader.rb @@ -0,0 +1,331 @@ +# Used by configure and make to download or update mirrored Ruby and GCC +# files. This will use HTTPS if possible, falling back to HTTP. + +require 'fileutils' +require 'open-uri' +require 'pathname' +begin + require 'net/https' +rescue LoadError + https = 'http' +else + https = 'https' + + # open-uri of ruby 2.2.0 accept an array of PEMs as ssl_ca_cert, but old + # versions are not. so, patching OpenSSL::X509::Store#add_file instead. + class OpenSSL::X509::Store + alias orig_add_file add_file + def add_file(pems) + Array(pems).each do |pem| + if File.directory?(pem) + add_path pem + else + orig_add_file pem + end + end + end + end + # since open-uri internally checks ssl_ca_cert using File.directory?, + # allow to accept an array. + class <<File + alias orig_directory? directory? + def File.directory? files + files.is_a?(Array) ? false : orig_directory?(files) + end + end +end + +class Downloader + def self.https=(https) + @@https = https + end + + def self.https? + @@https == 'https' + end + + def self.https + @@https + end + + class GNU < self + def self.download(name, *rest) + if https? + super("https://raw.githubusercontent.com/gcc-mirror/gcc/master/#{name}", name, *rest) + else + super("https://repo.or.cz/official-gcc.git/blob_plain/HEAD:/#{name}", name, *rest) + end + end + end + + class RubyGems < self + def self.download(name, dir = nil, since = true, options = {}) + require 'rubygems' + options = options.dup + options[:ssl_ca_cert] = Dir.glob(File.expand_path("../lib/rubygems/ssl_certs/**/*.pem", File.dirname(__FILE__))) + super("https://rubygems.org/downloads/#{name}", name, dir, since, options) + end + end + + Gems = RubyGems + + class Unicode < self + def self.download(name, *rest) + super("http://www.unicode.org/Public/#{name}", name, *rest) + end + end + + def self.mode_for(data) + /\A#!/ =~ data ? 0755 : 0644 + end + + def self.http_options(file, since) + options = {} + if since + case since + when true + since = (File.mtime(file).httpdate rescue nil) + when Time + since = since.httpdate + end + if since + options['If-Modified-Since'] = since + end + end + options['Accept-Encoding'] = '*' # to disable Net::HTTP::GenericRequest#decode_content + options + end + + # Downloader.download(url, name, [dir, [since]]) + # + # Update a file from url if newer version is available. + # Creates the file if the file doesn't yet exist; however, the + # directory where the file is being created has to exist already. + # The +since+ parameter can take the following values, with associated meanings: + # true :: + # Take the last-modified time of the current file on disk, and only download + # if the server has a file that was modified later. Download unconditionally + # if we don't have the file yet. Default. + # +some time value+ :: + # Use this time value instead of the time of modification of the file on disk. + # nil :: + # Only download the file if it doesn't exist yet. + # false :: + # always download url regardless of whether we already have a file, + # and regardless of modification times. (This is essentially just a waste of + # network resources, except in the case that the file we have is somehow damaged. + # Please note that using this recurringly might create or be seen as a + # denial of service attack.) + # + # Example usage: + # download 'http://www.unicode.org/Public/UCD/latest/ucd/UnicodeData.txt', + # 'UnicodeData.txt', 'enc/unicode/data' + def self.download(url, name, dir = nil, since = true, options = {}) + options = options.dup + url = URI(url) + dryrun = options.delete(:dryrun) + if name + file = Pathname.new(under(dir, name)) + else + name = File.basename(url.path) + end + cache_save = options.delete(:cache_save) { + ENV["CACHE_SAVE"] != "no" + } + cache = cache_file(url, name, options.delete(:cache_dir)) + file ||= cache + if since.nil? and file.exist? + if $VERBOSE + $stdout.puts "#{file} already exists" + $stdout.flush + end + if cache_save + save_cache(cache, file, name) + end + return file.to_path + end + if dryrun + puts "Download #{url} into #{file}" + return + end + if link_cache(cache, file, name, $VERBOSE) + return file.to_path + end + if !https? and URI::HTTPS === url + warn "*** using http instead of https ***" + url.scheme = 'http' + url = URI(url.to_s) + end + if $VERBOSE + $stdout.print "downloading #{name} ... " + $stdout.flush + end + begin + data = url.read(options.merge(http_options(file, since.nil? ? true : since))) + rescue OpenURI::HTTPError => http_error + if http_error.message =~ /^304 / # 304 Not Modified + if $VERBOSE + $stdout.puts "#{name} not modified" + $stdout.flush + end + return file.to_path + end + raise + rescue Timeout::Error + if since.nil? and file.exist? + puts "Request for #{url} timed out, using old version." + return file.to_path + end + raise + rescue SocketError + if since.nil? and file.exist? + puts "No network connection, unable to download #{url}, using old version." + return file.to_path + end + raise + end + mtime = nil + dest = (cache_save && cache && !cache.exist? ? cache : file) + dest.parent.mkpath + dest.open("wb", 0600) do |f| + f.write(data) + f.chmod(mode_for(data)) + mtime = data.meta["last-modified"] + end + if mtime + mtime = Time.httpdate(mtime) + dest.utime(mtime, mtime) + end + if $VERBOSE + $stdout.puts "done" + $stdout.flush + end + if dest.eql?(cache) + link_cache(cache, file, name) + elsif cache_save + save_cache(cache, file, name) + end + return file.to_path + rescue => e + raise "failed to download #{name}\n#{e.message}: #{url}" + end + + def self.under(dir, name) + dir ? File.join(dir, File.basename(name)) : name + end + + def self.cache_file(url, name, cache_dir = nil) + case cache_dir + when false + return nil + when nil + cache_dir = ENV['CACHE_DIR'] + if !cache_dir or cache_dir.empty? + cache_dir = ".downloaded-cache" + end + end + Pathname.new(cache_dir) + (name || File.basename(URI(url).path)) + end + + def self.link_cache(cache, file, name, verbose = false) + return false unless cache and cache.exist? + return true if cache.eql?(file) + if /cygwin/ !~ RUBY_PLATFORM or /winsymlink:nativestrict/ =~ ENV['CYGWIN'] + begin + file.make_symlink(cache.relative_path_from(file.parent)) + rescue SystemCallError + else + if verbose + $stdout.puts "made symlink #{name} to #{cache}" + $stdout.flush + end + return true + end + end + begin + file.make_link(cache) + rescue SystemCallError + else + if verbose + $stdout.puts "made link #{name} to #{cache}" + $stdout.flush + end + return true + end + end + + def self.save_cache(cache, file, name) + if cache and !cache.eql?(file) and !cache.exist? + begin + file.rename(cache) + rescue + else + link_cache(cache, file, name) + end + end + end +end + +Downloader.https = https.freeze + +if $0 == __FILE__ + since = true + options = {} + until ARGV.empty? + case ARGV[0] + when '-d' + destdir = ARGV[1] + ARGV.shift + when '-p' + # strip directory names from the name to download, and add the + # prefix instead. + prefix = ARGV[1] + ARGV.shift + when '-e' + since = nil + when '-a' + since = false + when '-n', '--dryrun' + options[:dryrun] = true + when '--cache-dir' + options[:cache_dir] = ARGV[1] + ARGV.shift + when /\A--cache-dir=(.*)/m + options[:cache_dir] = $1 + when /\A-/ + abort "#{$0}: unknown option #{ARGV[0]}" + else + break + end + ARGV.shift + end + dl = Downloader.constants.find do |name| + ARGV[0].casecmp(name.to_s) == 0 + end unless ARGV.empty? + $VERBOSE = true + if dl + dl = Downloader.const_get(dl) + ARGV.shift + ARGV.each do |name| + dir = destdir + if prefix + name = name.sub(/\A\.\//, '') + if name.start_with?(destdir+"/") + name = name[(destdir.size+1)..-1] + if (dir = File.dirname(name)) == '.' + dir = destdir + else + dir = File.join(destdir, dir) + end + else + name = File.basename(name) + end + name = "#{prefix}/#{name}" + end + dl.download(name, dir, since, options) + end + else + abort "usage: #{$0} url name" unless ARGV.size == 2 + Downloader.download(ARGV[0], ARGV[1], destdir, since, options) + end +end diff --git a/tool/enc-emoji-citrus-gen.rb b/tool/enc-emoji-citrus-gen.rb new file mode 100644 index 0000000000..5037cbde1e --- /dev/null +++ b/tool/enc-emoji-citrus-gen.rb @@ -0,0 +1,131 @@ +require File.expand_path('../jisx0208', __FILE__) + +ENCODES = [ + { + :name => "SHIFT_JIS-DOCOMO", + :src_zone => [0xF8..0xFC, 0x40..0xFC, 8], + :dst_ilseq => 0xFFFE, + :map => [ + [0xE63E..0xE757, JISX0208::Char.from_sjis(0xF89F)], + ], + }, + { + :name => "ISO-2022-JP-KDDI", + :src_zone => [0x21..0x7E, 0x21..0x7E, 8], + :dst_ilseq => 0xFFFE, + :map => [ + [0xE468..0xE5B4, JISX0208::Char.new(0x7521)], + [0xE5B5..0xE5CC, JISX0208::Char.new(0x7867)], + [0xE5CD..0xE5DF, JISX0208::Char.new(0x7921)], + [0xEA80..0xEAFA, JISX0208::Char.new(0x7934)], + [0xEAFB..0xEB0D, JISX0208::Char.new(0x7854)], + [0xEB0E..0xEB8E, JISX0208::Char.new(0x7A51)], + ], + }, + { + :name => "SHIFT_JIS-KDDI", + :src_zone => [0xF3..0xFC, 0x40..0xFC, 8], + :dst_ilseq => 0xFFFE, + :map => [ + [0xE468..0xE5B4, JISX0208::Char.from_sjis(0xF640)], + [0xE5B5..0xE5CC, JISX0208::Char.from_sjis(0xF7E5)], + [0xE5CD..0xE5DF, JISX0208::Char.from_sjis(0xF340)], + [0xEA80..0xEAFA, JISX0208::Char.from_sjis(0xF353)], + [0xEAFB..0xEB0D, JISX0208::Char.from_sjis(0xF7D2)], + [0xEB0E..0xEB8E, JISX0208::Char.from_sjis(0xF3CF)], + ], + }, + { + :name => "SHIFT_JIS-SOFTBANK", + :src_zone => [0xF3..0xFC, 0x40..0xFC, 8], + :dst_ilseq => 0xFFFE, + :map => [ + [0xE001..0xE05A, JISX0208::Char.from_sjis(0xF941)], + [0xE101..0xE15A, JISX0208::Char.from_sjis(0xF741)], + [0xE201..0xE25A, JISX0208::Char.from_sjis(0xF7A1)], + [0xE301..0xE34D, JISX0208::Char.from_sjis(0xF9A1)], + [0xE401..0xE44C, JISX0208::Char.from_sjis(0xFB41)], + [0xE501..0xE53E, JISX0208::Char.from_sjis(0xFBA1)], + ], + }, +] + +def zone(*args) + bits = args.pop + [*args.map{|range| "0x%02X-0x%02X" % [range.begin, range.end] }, bits].join(' / ') +end + +def header(params) + (<<END_HEADER_TEMPLATE % [params[:name], zone(*params[:src_zone]), params[:dst_ilseq]]) +# DO NOT EDIT THIS FILE DIRECTLY + +TYPE ROWCOL +NAME %s +SRC_ZONE %s +OOB_MODE ILSEQ +DST_ILSEQ 0x%04X +DST_UNIT_BITS 16 +END_HEADER_TEMPLATE +end + +def generate_to_ucs(params, pairs) + pairs.sort_by! {|u, c| c } + name = "EMOJI_#{params[:name]}%UCS" + open("#{name}.src", "w") do |io| + io.print header(params.merge(name: name.tr('%', '/'))) + io.puts + io.puts "BEGIN_MAP" + io.print pairs.inject("") {|acc, uc| acc += "0x%04X = 0x%04X\n" % uc.reverse } + io.puts "END_MAP" + end +end + +def generate_from_ucs(params, pairs) + pairs.sort_by! {|u, c| u } + name = "UCS%EMOJI_#{params[:name]}" + open("#{name}.src", "w") do |io| + io.print header(params.merge(name: name.tr('%', '/'))) + io.puts + io.puts "BEGIN_MAP" + io.print pairs.inject("") {|acc, uc| acc += "0x%04X = 0x%04X\n" % uc } + io.puts "END_MAP" + end +end + +def make_pairs(code_map) + pairs = code_map.inject([]) {|acc, (range, ch)| + acc += range.map{|uni| pair = [uni, Integer(ch)]; ch = ch.succ; next pair } + } +end + +ENCODES.each do |params| + pairs = make_pairs(params[:map], ¶ms[:conv]) + generate_to_ucs(params, pairs) + generate_from_ucs(params, pairs) +end + +# generate KDDI-UNDOC for Shift_JIS-KDDI +kddi_sjis_map = ENCODES.select{|enc| enc[:name] == "SHIFT_JIS-KDDI"}.first[:map] +pairs = kddi_sjis_map.inject([]) {|acc, (range, ch)| + acc += range.map{|uni| pair = [ch.to_sjis - 0x700, Integer(ch)]; ch = ch.succ; next pair } +} +params = { + :name => "SHIFT_JIS-KDDI-UNDOC", + :src_zone => [0xF3..0xFC, 0x40..0xFC, 8], + :dst_ilseq => 0xFFFE, +} +generate_from_ucs(params, pairs) +generate_to_ucs(params, pairs) + +# generate KDDI-UNDOC for ISO-2022-JP-KDDI +kddi_2022_map = ENCODES.select{|enc| enc[:name] == "ISO-2022-JP-KDDI"}.first[:map] +pairs = kddi_2022_map.each_with_index.inject([]) {|acc, ((range, ch), i)| + sjis = kddi_sjis_map[i][1] + acc += range.map{|uni| pair = [sjis.to_sjis - 0x700, Integer(ch)]; ch = ch.succ; sjis = sjis.succ; next pair } +} +params = { + :name => "ISO-2022-JP-KDDI-UNDOC", + :src_zone => [0x21..0x7E, 0x21..0x7E, 8], + :dst_ilseq => 0xFFFE, +} +generate_from_ucs(params, pairs) diff --git a/tool/enc-emoji4unicode.rb b/tool/enc-emoji4unicode.rb new file mode 100644 index 0000000000..1e7d45901f --- /dev/null +++ b/tool/enc-emoji4unicode.rb @@ -0,0 +1,133 @@ +#!/usr/bin/env ruby + +# example: +# ./enc-emoji4unicode.rb emoji4unicode.xml > ../enc/trans/emoji-exchange-tbl.rb + +require 'rexml/document' +require File.expand_path("../transcode-tblgen", __FILE__) + +class EmojiTable + VERBOSE_MODE = false + + def initialize(xml_path) + @doc = REXML::Document.new File.open(xml_path) + @kddi_undoc = make_kddi_undoc_map() + end + + def conversion(from_carrier, to_carrier, &block) + REXML::XPath.each(@doc.root, '//e') do |e| + from = e.attribute(from_carrier.downcase).to_s + to = e.attribute(to_carrier.downcase).to_s + text_fallback = e.attribute('text_fallback').to_s + name = e.attribute('name').to_s + if from =~ /^(?:\*|\+)(.+)$/ # proposed or unified + from = $1 + end + if from.empty? || from !~ /^[0-9A-F]+$/ + # do nothing + else + from_utf8 = [from.hex].pack("U").unpack("H*").first + if to =~ /^(?:>|\*)?([0-9A-F\+]+)$/ + str_to = $1 + if str_to =~ /^\+/ # unicode "proposed" begins at "+" + proposal = true + str_to.sub!(/^\+/, '') + else + proposal = false + end + tos = str_to.split('+') + to_utf8 = tos.map(&:hex).pack("U*").unpack("H*").first + comment = "[%s] U+%X -> %s" % [name, from.hex, tos.map{|c| "U+%X"%c.hex}.join(' ')] + block.call(:from => from_utf8, + :to => to_utf8, + :comment => comment, + :fallback => false, + :proposal => proposal) + elsif to.empty? + if text_fallback.empty? + comment = "[%s] U+%X -> U+3013 (GETA)" % [name, from.hex] + block.call(:from => from_utf8, + :to => "\u{3013}".unpack("H*").first, + :comment => comment, # geta + :fallback => true, + :proposal => false) + else + to_utf8 = text_fallback.unpack("H*").first + comment = %([%s] U+%X -> "%s") % [name, from.hex, text_fallback] + block.call(:from => from_utf8, + :to => to_utf8, + :comment => comment, + :fallback => true, + :proposal => false) + end + else + raise "something wrong: %s -> %s" % [from, to] + end + end + end + end + + def generate(io, from_carrier, to_carrier) + from_encoding = (from_carrier == "Unicode") ? "UTF-8" : "UTF8-"+from_carrier + to_encoding = (to_carrier == "Unicode" ) ? "UTF-8" : "UTF8-"+to_carrier + io.puts "EMOJI_EXCHANGE_TBL['#{from_encoding}']['#{to_encoding}'] = [" + io.puts " # for documented codepoints" if from_carrier == "KDDI" + self.conversion(from_carrier, to_carrier) do |params| + from, to = params[:from], %Q{"#{params[:to]}"} + to = ":undef" if params[:fallback] || params[:proposal] + io.puts %{ ["#{from}", #{to}], # #{params[:comment]}} + end + if from_carrier == "KDDI" + io.puts " # for undocumented codepoints" + self.conversion(from_carrier, to_carrier) do |params| + from, to = params[:from], %Q{"#{params[:to]}"} + to = ":undef" if params[:fallback] || params[:proposal] + unicode = utf8_to_ucs(from) + undoc = ucs_to_utf8(@kddi_undoc[unicode]) + io.puts %{ ["#{undoc}", #{to}], # #{params[:comment]}} + end + end + io.puts "]" + io.puts + end + + private + + def utf8_to_ucs(cp) + return [cp].pack("H*").unpack("U*").first + end + + def ucs_to_utf8(cp) + return [cp].pack("U*").unpack("H*").first + end + + def make_kddi_undoc_map() + pub_to_sjis = citrus_decode_mapsrc( + "mskanji", 2, "UCS/EMOJI_SHIFT_JIS-KDDI").sort_by{|u, s| s} + sjis_to_undoc = citrus_decode_mapsrc( + "mskanji", 2, "EMOJI_SHIFT_JIS-KDDI-UNDOC/UCS").sort_by{|s, u| s} + return pub_to_sjis.zip(sjis_to_undoc).inject({}) {|h, rec| + raise "no match sjis codepoint" if rec[0][1] != rec[1][0] + h[rec[0][0]] = rec[1][1] + next h + } + end +end + +if ARGV.empty? + puts "usage: #$0 [emoji4unicode.xml]" + exit 1 +end +$srcdir = File.expand_path("../../enc/trans", __FILE__) +emoji_table = EmojiTable.new(ARGV[0]) + +companies = %w(DoCoMo KDDI SoftBank Unicode) + +io = STDOUT +io.puts "EMOJI_EXCHANGE_TBL = Hash.new{|h,k| h[k] = {}}" +companies.each do |from_company| + companies.each do |to_company| + next if from_company == to_company + emoji_table.generate(io, from_company, to_company) + end +end diff --git a/tool/enc-unicode.rb b/tool/enc-unicode.rb new file mode 100755 index 0000000000..2ba0b73606 --- /dev/null +++ b/tool/enc-unicode.rb @@ -0,0 +1,556 @@ +#!/usr/bin/env ruby + +# Creates the data structures needed by Oniguruma to map Unicode codepoints to +# property names and POSIX character classes +# +# To use this, get UnicodeData.txt, Scripts.txt, PropList.txt, +# PropertyAliases.txt, PropertyValueAliases.txt, DerivedCoreProperties.txt, +# DerivedAge.txt and Blocks.txt from unicode.org. +# (http://unicode.org/Public/UNIDATA/) And run following command. +# ruby1.9 tool/enc-unicode.rb data_dir > enc/unicode/name2ctype.kwd +# You can get source file for gperf. After this, simply make ruby. + +if ARGV[0] == "--header" + header = true + ARGV.shift +end +unless ARGV.size == 2 + abort "Usage: #{$0} data_directory emoji_data_directory" +end + +$unicode_version = File.basename(ARGV[0])[/\A[.\d]+\z/] + +POSIX_NAMES = %w[NEWLINE Alpha Blank Cntrl Digit Graph Lower Print XPosixPunct Space Upper XDigit Word Alnum ASCII Punct] + +def pair_codepoints(codepoints) + + # We have a sorted Array of codepoints that we wish to partition into + # ranges such that the start- and endpoints form an inclusive set of + # codepoints with property _property_. Note: It is intended that some ranges + # will begin with the value with which they end, e.g. 0x0020 -> 0x0020 + + codepoints.sort! + last_cp = codepoints.first + pairs = [[last_cp, nil]] + codepoints[1..-1].each do |codepoint| + next if last_cp == codepoint + + # If the current codepoint does not follow directly on from the last + # codepoint, the last codepoint represents the end of the current range, + # and the current codepoint represents the start of the next range. + if last_cp.next != codepoint + pairs[-1][-1] = last_cp + pairs << [codepoint, nil] + end + last_cp = codepoint + end + + # The final pair has as its endpoint the last codepoint for this property + pairs[-1][-1] = codepoints.last + pairs +end + +def parse_unicode_data(file) + last_cp = 0 + data = {'Any' => (0x0000..0x10ffff).to_a, 'Assigned' => [], + 'ASCII' => (0..0x007F).to_a, 'NEWLINE' => [0x0a], 'Cn' => []} + beg_cp = nil + IO.foreach(file) do |line| + fields = line.split(';') + cp = fields[0].to_i(16) + + case fields[1] + when /\A<(.*),\s*First>\z/ + beg_cp = cp + next + when /\A<(.*),\s*Last>\z/ + cps = (beg_cp..cp).to_a + else + beg_cp = cp + cps = [cp] + end + + # The Cn category represents unassigned characters. These are not listed in + # UnicodeData.txt so we must derive them by looking for 'holes' in the range + # of listed codepoints. We increment the last codepoint seen and compare it + # with the current codepoint. If the current codepoint is less than + # last_cp.next we have found a hole, so we add the missing codepoint to the + # Cn category. + data['Cn'].concat((last_cp.next...beg_cp).to_a) + + # Assigned - Defined in unicode.c; interpreted as every character in the + # Unicode range minus the unassigned characters + data['Assigned'].concat(cps) + + # The third field denotes the 'General' category, e.g. Lu + (data[fields[2]] ||= []).concat(cps) + + # The 'Major' category is the first letter of the 'General' category, e.g. + # 'Lu' -> 'L' + (data[fields[2][0,1]] ||= []).concat(cps) + last_cp = cp + end + + # The last Cn codepoint should be 0x10ffff. If it's not, append the missing + # codepoints to Cn and C + cn_remainder = (last_cp.next..0x10ffff).to_a + data['Cn'] += cn_remainder + data['C'] += data['Cn'] + + # Special case for LC (Cased_Letter). LC = Ll + Lt + Lu + data['LC'] = data['Ll'] + data['Lt'] + data['Lu'] + + # Define General Category properties + gcps = data.keys.sort - POSIX_NAMES + + # Returns General Category Property names and the data + [gcps, data] +end + +def define_posix_props(data) + # We now derive the character classes (POSIX brackets), e.g. [[:alpha:]] + # + + data['Alpha'] = data['Alphabetic'] + data['Upper'] = data['Uppercase'] + data['Lower'] = data['Lowercase'] + data['Punct'] = data['Punctuation'] + data['XPosixPunct'] = data['Punctuation'] + [0x24, 0x2b, 0x3c, 0x3d, 0x3e, 0x5e, 0x60, 0x7c, 0x7e] + data['Digit'] = data['Decimal_Number'] + data['XDigit'] = (0x0030..0x0039).to_a + (0x0041..0x0046).to_a + + (0x0061..0x0066).to_a + data['Alnum'] = data['Alpha'] + data['Digit'] + data['Space'] = data['White_Space'] + data['Blank'] = data['Space_Separator'] + [0x0009] + data['Cntrl'] = data['Cc'] + data['Word'] = data['Alpha'] + data['Mark'] + data['Digit'] + data['Connector_Punctuation'] + data['Graph'] = data['Any'] - data['Space'] - data['Cntrl'] - + data['Surrogate'] - data['Unassigned'] + data['Print'] = data['Graph'] + data['Space_Separator'] +end + +def parse_scripts(data, categories) + files = [ + {:fn => 'DerivedCoreProperties.txt', :title => 'Derived Property'}, + {:fn => 'Scripts.txt', :title => 'Script'}, + {:fn => 'PropList.txt', :title => 'Binary Property'}, + {:fn => 'emoji-data.txt', :title => 'Emoji'} + ] + current = nil + cps = [] + names = {} + files.each do |file| + data_foreach(file[:fn]) do |line| + if /^# Total (?:code points|elements): / =~ line + data[current] = cps + categories[current] = file[:title] + (names[file[:title]] ||= []) << current + cps = [] + elsif /^([0-9a-fA-F]+)(?:\.\.([0-9a-fA-F]+))?\s*;\s*(\w+)/ =~ line + current = $3 + $2 ? cps.concat(($1.to_i(16)..$2.to_i(16)).to_a) : cps.push($1.to_i(16)) + end + end + end + # All code points not explicitly listed for Script + # have the value Unknown (Zzzz). + data['Unknown'] = (0..0x10ffff).to_a - data.values_at(*names['Script']).flatten + categories['Unknown'] = 'Script' + names.values.flatten << 'Unknown' +end + +def parse_aliases(data) + kv = {} + data_foreach('PropertyAliases.txt') do |line| + next unless /^(\w+)\s*; (\w+)/ =~ line + data[$1] = data[$2] + kv[normalize_propname($1)] = normalize_propname($2) + end + data_foreach('PropertyValueAliases.txt') do |line| + next unless /^(sc|gc)\s*; (\w+)\s*; (\w+)(?:\s*; (\w+))?/ =~ line + if $1 == 'gc' + data[$3] = data[$2] + data[$4] = data[$2] + kv[normalize_propname($3)] = normalize_propname($2) + kv[normalize_propname($4)] = normalize_propname($2) if $4 + else + data[$2] = data[$3] + data[$4] = data[$3] + kv[normalize_propname($2)] = normalize_propname($3) + kv[normalize_propname($4)] = normalize_propname($3) if $4 + end + end + kv +end + +# According to Unicode6.0.0/ch03.pdf, Section 3.1, "An update version +# never involves any additions to the character repertoire." Versions +# in DerivedAge.txt should always be /\d+\.\d+/ +def parse_age(data) + current = nil + last_constname = nil + cps = [] + ages = [] + data_foreach('DerivedAge.txt') do |line| + if /^# Total code points: / =~ line + constname = constantize_agename(current) + # each version matches all previous versions + cps.concat(data[last_constname]) if last_constname + data[constname] = cps + make_const(constname, cps, "Derived Age #{current}") + ages << current + last_constname = constname + cps = [] + elsif /^([0-9a-fA-F]+)(?:\.\.([0-9a-fA-F]+))?\s*;\s*(\d+\.\d+)/ =~ line + current = $3 + $2 ? cps.concat(($1.to_i(16)..$2.to_i(16)).to_a) : cps.push($1.to_i(16)) + end + end + ages +end + +def parse_GraphemeBreakProperty(data) + current = nil + cps = [] + ages = [] + data_foreach('auxiliary/GraphemeBreakProperty.txt') do |line| + if /^# Total code points: / =~ line + constname = constantize_Grapheme_Cluster_Break(current) + data[constname] = cps + make_const(constname, cps, "Grapheme_Cluster_Break=#{current}") + ages << current + cps = [] + elsif /^([0-9a-fA-F]+)(?:\.\.([0-9a-fA-F]+))?\s*;\s*(\w+)/ =~ line + current = $3 + $2 ? cps.concat(($1.to_i(16)..$2.to_i(16)).to_a) : cps.push($1.to_i(16)) + end + end + ages +end + +def parse_block(data) + current = nil + cps = [] + blocks = [] + data_foreach('Blocks.txt') do |line| + if /^([0-9a-fA-F]+)\.\.([0-9a-fA-F]+);\s*(.*)/ =~ line + cps = ($1.to_i(16)..$2.to_i(16)).to_a + constname = constantize_blockname($3) + data[constname] = cps + make_const(constname, cps, "Block") + blocks << constname + end + end + + # All code points not belonging to any of the named blocks + # have the value No_Block. + no_block = (0..0x10ffff).to_a - data.values_at(*blocks).flatten + constname = constantize_blockname("No_Block") + make_const(constname, no_block, "Block") + blocks << constname +end + +# shim for Ruby 1.8 +unless {}.respond_to?(:key) + class Hash + alias key index + end +end + +$const_cache = {} +# make_const(property, pairs, name): Prints a 'static const' structure for a +# given property, group of paired codepoints, and a human-friendly name for +# the group +def make_const(prop, data, name) + if name.empty? + puts "\n/* '#{prop}' */" + else + puts "\n/* '#{prop}': #{name} */" + end + if origprop = $const_cache.key(data) + puts "#define CR_#{prop} CR_#{origprop}" + else + $const_cache[prop] = data + pairs = pair_codepoints(data) + puts "static const OnigCodePoint CR_#{prop}[] = {" + # The first element of the constant is the number of pairs of codepoints + puts "\t#{pairs.size}," + pairs.each do |pair| + pair.map! { |c| c == 0 ? '0x0000' : sprintf("%0#6x", c) } + puts "\t#{pair.first}, #{pair.last}," + end + puts "}; /* CR_#{prop} */" + end +end + +def normalize_propname(name) + name = name.downcase + name.delete!('- _') + name +end + +def constantize_agename(name) + "Age_#{name.sub(/\./, '_')}" +end + +def constantize_Grapheme_Cluster_Break(name) + "Grapheme_Cluster_Break_#{name}" +end + +def constantize_blockname(name) + "In_#{name.gsub(/\W/, '_')}" +end + +def get_file(name) + File.join(ARGV[name.start_with?("emoji-") ? 1 : 0], name) +end + +def data_foreach(name, &block) + fn = get_file(name) + warn "Reading #{name}" + pat = /^# #{File.basename(name).sub(/\./, '-([\\d.]+)\\.')}/ + File.open(fn, 'rb') do |f| + line = f.gets + unless /^emoji-/ =~ name + unless pat =~ line + raise ArgumentError, "#{name}: no Unicode version" + end + if !$unicode_version + $unicode_version = $1 + elsif $unicode_version != $1 + raise ArgumentError, "#{name}: Unicode version mismatch: #$1" + end + end + f.each(&block) + end +end + +# Write Data +class Unifdef + attr_accessor :output, :top, :stack, :stdout, :kwdonly + def initialize(out) + @top = @output = [] + @stack = [] + $stdout, @stdout = self, out + end + def restore + $stdout = @stdout + end + def ifdef(sym) + if @kwdonly + @stdout.puts "#ifdef #{sym}" + else + @stack << @top + @top << tmp = [sym] + @top = tmp + end + if block_given? + begin + return yield + ensure + endif(sym) + end + end + end + def endif(sym) + if @kwdonly + @stdout.puts "#endif /* #{sym} */" + else + unless sym == @top[0] + restore + raise ArgumentError, "#{sym} unmatch to #{@top[0]}" + end + @top = @stack.pop + end + end + def show(dest, *syms) + _show(dest, @output, syms) + end + def _show(dest, ary, syms) + if Symbol === (sym = ary[0]) + unless syms.include?(sym) + return + end + end + ary.each do |e| + case e + when Array + _show(dest, e, syms) + when String + dest.print e + end + end + end + def write(str) + if @kwdonly + @stdout.write(str) + else + @top << str + end + self + end + alias << write +end + +output = Unifdef.new($stdout) +output.kwdonly = !header + +puts '%{' +props, data = parse_unicode_data(get_file('UnicodeData.txt')) +categories = {} +props.concat parse_scripts(data, categories) +aliases = parse_aliases(data) +ages = blocks = graphemeBreaks = nil +define_posix_props(data) +POSIX_NAMES.each do |name| + if name == 'XPosixPunct' + make_const(name, data[name], "[[:Punct:]]") + elsif name == 'Punct' + make_const(name, data[name], "") + else + make_const(name, data[name], "[[:#{name}:]]") + end +end +output.ifdef :USE_UNICODE_PROPERTIES do + props.each do |name| + category = categories[name] || + case name.size + when 1 then 'Major Category' + when 2 then 'General Category' + else '-' + end + make_const(name, data[name], category) + end + output.ifdef :USE_UNICODE_AGE_PROPERTIES do + ages = parse_age(data) + end + graphemeBreaks = parse_GraphemeBreakProperty(data) + blocks = parse_block(data) +end +puts(<<'__HEREDOC') + +static const OnigCodePoint* const CodeRanges[] = { +__HEREDOC +POSIX_NAMES.each{|name|puts" CR_#{name},"} +output.ifdef :USE_UNICODE_PROPERTIES do + props.each{|name| puts" CR_#{name},"} + output.ifdef :USE_UNICODE_AGE_PROPERTIES do + ages.each{|name| puts" CR_#{constantize_agename(name)},"} + end + graphemeBreaks.each{|name| puts" CR_#{constantize_Grapheme_Cluster_Break(name)},"} + blocks.each{|name|puts" CR_#{name},"} +end + +puts(<<'__HEREDOC') +}; +struct uniname2ctype_struct { + short name; + unsigned short ctype; +}; +#define uniname2ctype_offset(str) offsetof(struct uniname2ctype_pool_t, uniname2ctype_pool_##str) + +#if !(/*ANSI*/+0) +static const struct uniname2ctype_struct *uniname2ctype_p(const char *, unsigned int); +#endif +%} +struct uniname2ctype_struct; +%% +__HEREDOC + +i = -1 +name_to_index = {} +POSIX_NAMES.each do |name| + i += 1 + next if name == 'NEWLINE' + name = normalize_propname(name) + name_to_index[name] = i + puts"%-40s %3d" % [name + ',', i] +end +output.ifdef :USE_UNICODE_PROPERTIES do + props.each do |name| + i += 1 + name = normalize_propname(name) + name_to_index[name] = i + puts "%-40s %3d" % [name + ',', i] + end + aliases.each_pair do |k, v| + next if name_to_index[k] + next unless v = name_to_index[v] + puts "%-40s %3d" % [k + ',', v] + end + output.ifdef :USE_UNICODE_AGE_PROPERTIES do + ages.each do |name| + i += 1 + name = "age=#{name}" + name_to_index[name] = i + puts "%-40s %3d" % [name + ',', i] + end + end + graphemeBreaks.each do |name| + i += 1 + name = "graphemeclusterbreak=#{name.delete('_').downcase}" + name_to_index[name] = i + puts "%-40s %3d" % [name + ',', i] + end + blocks.each do |name| + i += 1 + name = normalize_propname(name) + name_to_index[name] = i + puts "%-40s %3d" % [name + ',', i] + end +end +puts(<<'__HEREDOC') +%% +static int +uniname2ctype(const UChar *name, unsigned int len) +{ + const struct uniname2ctype_struct *p = uniname2ctype_p((const char *)name, len); + if (p) return p->ctype; + return -1; +} +__HEREDOC +versions = $unicode_version.scan(/\d+/) +print("#if defined ONIG_UNICODE_VERSION_STRING && !( \\\n") +%w[MAJOR MINOR TEENY].zip(versions) do |n, v| + print(" ONIG_UNICODE_VERSION_#{n} == #{v} && \\\n") +end +print(" 1)\n") +print("# error ONIG_UNICODE_VERSION_STRING mismatch\n") +print("#endif\n") +print("#define ONIG_UNICODE_VERSION_STRING #{$unicode_version.dump}\n") +%w[MAJOR MINOR TEENY].zip(versions) do |n, v| + print("#define ONIG_UNICODE_VERSION_#{n} #{v}\n") +end + +output.restore + +if header + require 'tempfile' + + NAME2CTYPE = %w[gperf -7 -c -j1 -i1 -t -C -P -T -H uniname2ctype_hash -Q uniname2ctype_pool -N uniname2ctype_p] + + fds = [] + syms = %i[USE_UNICODE_PROPERTIES USE_UNICODE_AGE_PROPERTIES] + begin + fds << (tmp = Tempfile.new(%w"name2ctype .h")) + IO.popen([*NAME2CTYPE, out: tmp], "w") {|f| output.show(f, *syms)} + end while syms.pop + fds.each(&:close) + IO.popen(%W[diff -DUSE_UNICODE_AGE_PROPERTIES #{fds[1].path} #{fds[0].path}], "r") {|age| + IO.popen(%W[diff -DUSE_UNICODE_PROPERTIES #{fds[2].path} -], "r", in: age) {|f| + ansi = false + f.each {|line| + if /ANSI-C code produced by gperf/ =~ line + ansi = true + end + line.sub!(/\/\*ANSI\*\//, '1') if ansi + line.gsub!(/\(int\)\((?:long|size_t)\)&\(\(struct uniname2ctype_pool_t \*\)0\)->uniname2ctype_pool_(str\d+),\s+/, + 'uniname2ctype_offset(\1), ') + if (/^(uniname2ctype_hash) /=~line)..(/^\}/=~line) + line.sub!(/^( *(?:register\s+)?(.*\S)\s+hval\s*=\s*)(?=len;)/, '\1(\2)') + end + puts line + } + } + } +end diff --git a/tool/eval.rb b/tool/eval.rb new file mode 100644 index 0000000000..981f72733b --- /dev/null +++ b/tool/eval.rb @@ -0,0 +1,160 @@ +# VM checking and benchmarking code + +require './rbconfig' +require 'fileutils' +require 'pp' + +Ruby = ENV['RUBY'] || RbConfig.ruby +# + +OPTIONS = %w{ + opt-direct-threaded-code + opt-basic-operations + opt-operands-unification + opt-instructions-unification + opt-inline-method-cache + opt-stack-caching +}.map{|opt| + '--disable-' + opt +} + +opts = OPTIONS.dup +Configs = OPTIONS.map{|opt| + o = opts.dup + opts.delete(opt) + o +} + [[]] + +pp Configs if $DEBUG + + +def exec_cmd(cmd) + puts cmd + unless system(cmd) + p cmd + raise "error" + end +end + +def dirname idx + "ev-#{idx}" +end + +def build + Configs.each_with_index{|config, idx| + dir = dirname(idx) + FileUtils.rm_rf(dir) if FileTest.exist?(dir) + Dir.mkdir(dir) + FileUtils.cd(dir){ + exec_cmd("#{Ruby} ../extconf.rb " + config.join(" ")) + exec_cmd("make clean test-all") + } + } +end + +def check + Configs.each_with_index{|c, idx| + puts "= #{idx}" + system("#{Ruby} -r ev-#{idx}/yarvcore -e 'puts YARVCore::OPTS'") + } +end + +def bench_each idx + puts "= #{idx}" + 5.times{|count| + print count + FileUtils.cd(dirname(idx)){ + exec_cmd("make benchmark OPT=-y ITEMS=#{ENV['ITEMS']} > ../b#{idx}-#{count}") + } + } + puts +end + +def bench + # return bench_each(6) + Configs.each_with_index{|c, idx| + bench_each idx + } +end + +def parse_result data + flag = false + stat = [] + data.each{|line| + if flag + if /(\w+)\t([\d\.]+)/ =~ line + stat << [$1, $2.to_f] + else + raise "not a data" + end + + end + if /benchmark summary/ =~ line + flag = true + end + } + stat +end + +def calc_each data + data.sort! + data.pop # remove max + data.shift # remove min + + data.inject(0.0){|res, e| + res += e + } / data.size +end + +def calc_stat stats + stat = [] + stats[0].each_with_index{|e, idx| + bm = e[0] + vals = stats.map{|st| + st[idx][1] + } + [bm, calc_each(vals)] + } +end + +def stat + total = [] + Configs.each_with_index{|c, idx| + stats = [] + 5.times{|count| + file = "b#{idx}-#{count}" + # p file + open(file){|f| + stats << parse_result(f.read) + } + } + # merge stats + total << calc_stat(stats) + total + } + # pp total + total[0].each_with_index{|e, idx| + bm = e[0] + # print "#{bm}\t" + total.each{|st| + print st[idx][1], "\t" + } + puts + } +end + +ARGV.each{|cmd| + case cmd + when 'build' + build + when 'check' + check + when 'bench' + bench + when 'stat' + stat + else + raise + end +} + diff --git a/tool/expand-config.rb b/tool/expand-config.rb new file mode 100755 index 0000000000..d34f29f586 --- /dev/null +++ b/tool/expand-config.rb @@ -0,0 +1,35 @@ +#!./miniruby -s + +# Used to expand Ruby config entries for Win32 Makefiles. + +config = File.read(conffile = $config) +config.sub!(/^(\s*)RUBY_VERSION\b.*(\sor\s*)$/, '\1true\2') +rbconfig = Module.new {module_eval(config, conffile)}::RbConfig +config = $expand ? rbconfig::CONFIG : rbconfig::MAKEFILE_CONFIG +config["RUBY_RELEASE_DATE"] ||= + File.read(File.expand_path("../../version.h", __FILE__))[/^\s*#\s*define\s+RUBY_RELEASE_DATE\s+"(.*)"/, 1] + +while /\A(\w+)=(.*)/ =~ ARGV[0] + config[$1] = $2 + config[$1].tr!(File::ALT_SEPARATOR, File::SEPARATOR) if File::ALT_SEPARATOR + ARGV.shift +end + +re = /@(#{config.keys.map {|k| Regexp.quote(k)}.join('|')})@/ + +if $output + output = open($output, "wb", $mode &&= $mode.oct) + output.chmod($mode) if $mode +else + output = STDOUT + output.binmode +end + +ARGF.each do |line| + line.gsub!(/@([a-z_]\w*)@/i) { + s = config.fetch($1, $expand ? $& : "") + s = s.gsub(/\$\((.+?)\)/, %Q[${\\1}]) unless $expand + s + } + output.puts line +end diff --git a/tool/extlibs.rb b/tool/extlibs.rb new file mode 100755 index 0000000000..a840724794 --- /dev/null +++ b/tool/extlibs.rb @@ -0,0 +1,186 @@ +#!/usr/bin/ruby + +# Used to download, extract and patch extension libraries (extlibs) +# for Ruby. See common.mk for Ruby's usage. + +require 'digest' +require_relative 'downloader' + +class ExtLibs + def cache_file(url, cache_dir) + Downloader.cache_file(url, nil, :cache_dir => cache_dir) + end + + def do_download(url, cache_dir) + Downloader.download(url, nil, nil, nil, :cache_dir => cache_dir) + end + + def do_checksum(cache, chksums) + chksums.each do |sum| + name, sum = sum.split(/:/) + if $VERBOSE + $stdout.print "checking #{name} of #{cache} ..." + $stdout.flush + end + hd = Digest(name.upcase).file(cache).hexdigest + if hd == sum + if $VERBOSE + $stdout.puts " OK" + $stdout.flush + end + else + if $VERBOSE + $stdout.puts " NG" + $stdout.flush + end + raise "checksum mismatch: #{cache}, #{name}:#{hd}, expected #{sum}" + end + end + end + + def do_extract(cache, dir) + if $VERBOSE + $stdout.puts "extracting #{cache} into #{dir}" + $stdout.flush + end + ext = File.extname(cache) + case ext + when '.gz', '.tgz' + f = IO.popen(["gzip", "-dc", cache]) + cache = cache.chomp('.gz') + when '.bz2', '.tbz' + f = IO.popen(["bzip2", "-dc", cache]) + cache = cache.chomp('.bz2') + when '.xz', '.txz' + f = IO.popen(["xz", "-dc", cache]) + cache = cache.chomp('.xz') + else + inp = cache + end + inp ||= f.binmode + ext = File.extname(cache) + case ext + when '.tar', /\A\.t[gbx]z\z/ + pid = Process.spawn("tar", "xpf", "-", in: inp, chdir: dir) + when '.zip' + pid = Process.spawn("unzip", inp, "-d", dir) + end + f.close if f + Process.wait(pid) + $?.success? or raise "failed to extract #{cache}" + end + + def do_patch(dest, patch, args) + if $VERBOSE + $stdout.puts "applying #{patch} under #{dest}" + $stdout.flush + end + Process.wait(Process.spawn("patch", "-d", dest, "-i", patch, *args)) + $?.success? or raise "failed to patch #{patch}" + end + + def do_command(mode, dest, url, cache_dir, chksums) + extracted = false + base = /.*(?=\.tar(?:\.\w+)?\z)/ + + case mode + when :download + cache = do_download(url, cache_dir) + do_checksum(cache, chksums) + when :extract + cache = cache_file(url, cache_dir) + target = File.join(dest, File.basename(cache)[base]) + unless File.directory?(target) + do_checksum(cache, chksums) + extracted = do_extract(cache, dest) + end + when :all + cache = do_download(url, cache_dir) + target = File.join(dest, File.basename(cache)[base]) + unless File.directory?(target) + do_checksum(cache, chksums) + extracted = do_extract(cache, dest) + end + end + extracted + end + + def run(argv) + cache_dir = nil + mode = :all + until argv.empty? + case argv[0] + when '--download' + mode = :download + when '--extract' + mode = :extract + when '--patch' + mode = :patch + when '--all' + mode = :all + when '--cache' + argv.shift + cache_dir = argv[0] + when /\A--cache=/ + cache_dir = $' + when '--' + argv.shift + break + when /\A-/ + warn "unknown option: #{argv[0]}" + return false + else + break + end + argv.shift + end + + success = true + argv.each do |dir| + Dir.glob("#{dir}/**/extlibs") do |list| + if $VERBOSE + $stdout.puts "downloading for #{list}" + $stdout.flush + end + extracted = false + dest = File.dirname(list) + url = chksums = nil + IO.foreach(list) do |line| + line.sub!(/\s*#.*/, '') + if chksums + chksums.concat(line.split) + elsif /^\t/ =~ line + if extracted and (mode == :all or mode == :patch) + patch, *args = line.split + do_patch(dest, patch, args) + end + next + else + url, *chksums = line.split(' ') + end + if chksums.last == '\\' + chksums.pop + next + end + next unless url + begin + extracted = do_command(mode, dest, url, cache_dir, chksums) + rescue => e + warn e.inspect + success = false + end + url = chksums = nil + end + end + end + success + end + + def self.run(argv) + self.new.run(argv) + end +end + +if $0 == __FILE__ + exit ExtLibs.run(ARGV) +end diff --git a/tool/fake.rb b/tool/fake.rb new file mode 100644 index 0000000000..99fc24e775 --- /dev/null +++ b/tool/fake.rb @@ -0,0 +1,70 @@ +# Used by Makefile and configure for building Ruby. +# See common.mk and Makefile.in for details. + +class File + sep = ("\\" if RUBY_PLATFORM =~ /mswin|bccwin|mingw/) + if sep != ALT_SEPARATOR + remove_const :ALT_SEPARATOR + ALT_SEPARATOR = sep + end +end + +static = !!(defined?($static) && $static) +$:.unshift(builddir) +posthook = proc do + config = RbConfig::CONFIG + mkconfig = RbConfig::MAKEFILE_CONFIG + extout = File.expand_path(mkconfig["EXTOUT"], builddir) + [ + ["top_srcdir", $top_srcdir], + ["topdir", $topdir], + ].each do |var, val| + next unless val + mkconfig[var] = config[var] = val + t = /\A#{Regexp.quote(val)}(?=\/)/ + $hdrdir.sub!(t) {"$(#{var})"} + mkconfig.keys.grep(/dir\z/) do |k| + mkconfig[k] = "$(#{var})#$'" if t =~ mkconfig[k] + end + end + if $extmk + $ruby = "$(topdir)/miniruby -I'$(topdir)' -I'$(top_srcdir)/lib' -I'$(extout)/$(arch)' -I'$(extout)/common'" + else + $ruby = baseruby + end + $static = static + untrace_var(:$ruby, posthook) +end +prehook = proc do |extmk| + pat = %r[(?:\A(?:\w:|//[^/]+)|\G)/[^/]*] + dir = builddir.scan(pat) + pwd = Dir.pwd.scan(pat) + if dir[0] == pwd[0] + while dir[0] and dir[0] == pwd[0] + dir.shift + pwd.shift + end + builddir = File.join((pwd.empty? ? ["."] : [".."]*pwd.size) + dir) + builddir = "." if builddir.empty? + end + join = proc {|*args| File.join(*args).sub!(/\A(?:\.\/)*/, '')} + $topdir ||= builddir + $top_srcdir ||= (File.identical?(top_srcdir, dir = join[$topdir, srcdir]) ? + dir : top_srcdir) + $extout = '$(topdir)/.ext' + $extout_prefix = '$(extout)$(target_prefix)/' + config = RbConfig::CONFIG + mkconfig = RbConfig::MAKEFILE_CONFIG + mkconfig["builddir"] = config["builddir"] = builddir + mkconfig["buildlibdir"] = config["buildlibdir"] = builddir + mkconfig["top_srcdir"] = $top_srcdir if $top_srcdir + mkconfig["extout"] ||= $extout + config["top_srcdir"] = File.expand_path($top_srcdir ||= top_srcdir) + config["rubyhdrdir"] = join[$top_srcdir, "include"] + config["rubyarchhdrdir"] = join[builddir, config["EXTOUT"], "include", config["arch"]] + config["extout"] ||= join[$topdir, ".ext"] + mkconfig["libdirname"] = "buildlibdir" + trace_var(:$ruby, posthook) + untrace_var(:$extmk, prehook) +end +trace_var(:$extmk, prehook) diff --git a/tool/fetch-bundled_gems.rb b/tool/fetch-bundled_gems.rb new file mode 100755 index 0000000000..ae3068d35c --- /dev/null +++ b/tool/fetch-bundled_gems.rb @@ -0,0 +1,27 @@ +#!ruby -an +BEGIN { + require 'fileutils' + + dir = ARGV.shift + ARGF.eof? + FileUtils.mkdir_p(dir) + Dir.chdir(dir) +} + +n, v, u = $F +case n +when "minitest" + v = "master" +when "test-unit" +else + v = "v" + v +end + +if File.directory?(n) + puts "updating #{n} ..." + system(*%W"git fetch", chdir: n) or abort +else + puts "retrieving #{n} ..." + system(*%W"git clone #{u} #{n}") or abort +end +system(*%W"git checkout #{v}", chdir: n) or abort diff --git a/tool/file2lastrev.rb b/tool/file2lastrev.rb new file mode 100755 index 0000000000..8d68da9f88 --- /dev/null +++ b/tool/file2lastrev.rb @@ -0,0 +1,98 @@ +#!/usr/bin/env ruby + +# Gets the most recent revision of a file in a VCS-agnostic way. +# Used by Doxygen, Makefiles and merger.rb. + +require 'optparse' + +# this file run with BASERUBY, which may be older than 1.9, so no +# require_relative +require File.expand_path('../vcs', __FILE__) + +Program = $0 + +@output = nil +def self.output=(output) + if @output and @output != output + raise "you can specify only one of --changed, --revision.h and --doxygen" + end + @output = output +end +@suppress_not_found = false + +format = '%Y-%m-%dT%H:%M:%S%z' +srcdir = nil +parser = OptionParser.new {|opts| + opts.on("--srcdir=PATH", "use PATH as source directory") do |path| + srcdir = path + end + opts.on("--changed", "changed rev") do + self.output = :changed + end + opts.on("--revision.h", "RUBY_REVISION macro") do + self.output = :revision_h + end + opts.on("--doxygen", "Doxygen format") do + self.output = :doxygen + end + opts.on("--modified[=FORMAT]", "modified time") do |fmt| + self.output = :modified + format = fmt if fmt + end + opts.on("-q", "--suppress_not_found") do + @suppress_not_found = true + end +} +parser.parse! rescue abort "#{File.basename(Program)}: #{$!}\n#{parser}" + +@output = + case @output + when :changed, nil + Proc.new {|last, changed| + changed + } + when :revision_h + Proc.new {|last, changed, modified, branch, title| + [ + "#define RUBY_REVISION #{changed || 0}", + if branch + e = '..' + limit = 16 + name = branch.sub(/\A(.{#{limit-e.size}}).{#{e.size+1},}/o) {$1+e} + "#define RUBY_BRANCH_NAME #{name.dump}" + end, + if title + "#define RUBY_LAST_COMMIT_TITLE #{title.dump}" + end, + ].compact + } + when :doxygen + Proc.new {|last, changed| + "r#{changed}/r#{last}" + } + when :modified + Proc.new {|last, changed, modified| + modified.strftime(format) + } + else + raise "unknown output format `#{@output}'" + end + +srcdir ||= File.dirname(File.dirname(Program)) +begin + vcs = VCS.detect(srcdir) +rescue VCS::NotFoundError => e + abort "#{File.basename(Program)}: #{e.message}" unless @suppress_not_found +else + ok = true + (ARGV.empty? ? [nil] : ARGV).each do |arg| + begin + puts @output[*vcs.get_revisions(arg)] + rescue => e + next if @suppress_not_found and VCS::NotFoundError === e + warn "#{File.basename(Program)}: #{e.message}" + ok = false + end + end + exit ok +end diff --git a/tool/gem-unpack.rb b/tool/gem-unpack.rb new file mode 100755 index 0000000000..0ddcea0704 --- /dev/null +++ b/tool/gem-unpack.rb @@ -0,0 +1,18 @@ +require 'rubygems' +require 'rubygems/package' + +# This library is used by "make extract-gems" to +# unpack bundled gem files. + +def Gem.unpack(file, dir = nil) + pkg = Gem::Package.new(file) + spec = pkg.spec + target = spec.full_name + target = File.join(dir, target) if dir + pkg.extract_files target + spec_file = File.join(target, "#{spec.name}.gemspec") + open(spec_file, 'wb') do |f| + f.print spec.to_ruby + end + puts "Unpacked #{file}" +end diff --git a/tool/gen_dummy_probes.rb b/tool/gen_dummy_probes.rb new file mode 100755 index 0000000000..45222830f3 --- /dev/null +++ b/tool/gen_dummy_probes.rb @@ -0,0 +1,32 @@ +#!/usr/bin/ruby +# -*- coding: us-ascii -*- + +# Used to create dummy probes (as for systemtap and DTrace) by Makefiles. +# See common.mk. + +text = ARGF.read + +# remove comments +text.gsub!(%r'(?:^ *)?/\*.*?\*/\n?'m, '') + +# remove the pragma declarations and ifdefs +text.gsub!(/^#(?:pragma|include|if|endif).*\n/, '') + +# replace the provider section with the start of the header file +text.gsub!(/provider ruby \{/, "#ifndef\t_PROBES_H\n#define\t_PROBES_H\n#define DTRACE_PROBES_DISABLED 1\n") + +# finish up the #ifndef sandwich +text.gsub!(/\};/, "\n#endif\t/* _PROBES_H */") + +# expand probes to DTRACE macros +text.gsub!(/^ *probe ([^\(]*)\(([^\)]*)\);/) { + name, args = $1, $2 + name.upcase! + name.gsub!(/__/, '_') + args.gsub!(/(\A|, *)[^,]*\b(?=\w+(?=,|\z))/, '\1') + "#define RUBY_DTRACE_#{name}_ENABLED() 0\n" \ + "#define RUBY_DTRACE_#{name}(#{args}) do {} while (0)" +} + +puts "/* -*- c -*- */" +print text diff --git a/tool/gen_ruby_tapset.rb b/tool/gen_ruby_tapset.rb new file mode 100755 index 0000000000..ec8650204f --- /dev/null +++ b/tool/gen_ruby_tapset.rb @@ -0,0 +1,106 @@ +#!/usr/bin/ruby +# -*- coding: us-ascii -*- +# Create a tapset for systemtap and DTrace +# usage: ./ruby gen_ruby_tapset.rb --ruby-path=/path/to/ruby probes.d > output + +require "optparse" + +def set_argument(argname, nth) + # remove C style type info + argname.gsub!(/.+ (.+)/, '\1') # e.g. char *hoge -> *hoge + argname.gsub!(/^\*/, '') # e.g. *filename -> filename + + "#{argname} = $arg#{nth}" +end + +ruby_path = "/usr/local/ruby" + +opts = OptionParser.new +opts.on("--ruby-path=PATH"){|v| ruby_path = v} +opts.parse!(ARGV) + +text = ARGF.read + +# remove preprocessor directives +text.gsub!(/^#.*$/, '') + +# remove provider name +text.gsub!(/^provider ruby \{/, "") +text.gsub!(/^\};/, "") + +# probename() +text.gsub!(/probe (.+)\( *\);/) { + probe_name = $1 + probe = <<-End + probe #{probe_name} = process("ruby").provider("ruby").mark("#{probe_name}") + { + } + End +} + +# probename(arg1) +text.gsub!(/ *probe (.+)\(([^,)]+)\);/) { + probe_name = $1 + arg1 = $2 + + probe = <<-End + probe #{probe_name} = process("ruby").provider("ruby").mark("#{probe_name}") + { + #{set_argument(arg1, 1)} + } + End +} + +# probename(arg1, arg2) +text.gsub!(/ *probe (.+)\(([^,)]+),([^,)]+)\);/) { + probe_name = $1 + arg1 = $2 + arg2 = $3 + + probe = <<-End + probe #{probe_name} = process("#{ruby_path}").provider("ruby").mark("#{probe_name}") + { + #{set_argument(arg1, 1)} + #{set_argument(arg2, 2)} + } + End +} + +# probename(arg1, arg2, arg3) +text.gsub!(/ *probe (.+)\(([^,)]+),([^,)]+),([^,)]+)\);/) { + probe_name = $1 + arg1 = $2 + arg2 = $3 + arg3 = $4 + + probe = <<-End + probe #{probe_name} = process("#{ruby_path}").provider("ruby").mark("#{probe_name}") + { + #{set_argument(arg1, 1)} + #{set_argument(arg2, 2)} + #{set_argument(arg3, 3)} + } + End +} + +# probename(arg1, arg2, arg3, arg4) +text.gsub!(/ *probe (.+)\(([^,)]+),([^,)]+),([^,)]+),([^,)]+)\);/) { + probe_name = $1 + arg1 = $2 + arg2 = $3 + arg3 = $4 + arg4 = $5 + + probe = <<-End + probe #{probe_name} = process("#{ruby_path}").provider("ruby").mark("#{probe_name}") + { + #{set_argument(arg1, 1)} + #{set_argument(arg2, 2)} + #{set_argument(arg3, 3)} + #{set_argument(arg4, 4)} + } + End +} + +print text + diff --git a/tool/generate-backport-changelog.rb b/tool/generate-backport-changelog.rb new file mode 100644 index 0000000000..0e8676a085 --- /dev/null +++ b/tool/generate-backport-changelog.rb @@ -0,0 +1,99 @@ +#!ruby +require "time" + +def usage! + STDERR.puts <<-EOS +Usage: #$0 [--trunk=<dir>] [--target=<dir>] <revision(s)> + +Generate ChangeLog entries for backporting. +The entries are output to STDOUT, and the messages of this tool are output to +STDERR. So you can simply redirect STDOUT to get the entries. + +You should specify the path of trunk by `--trunk`. If not, assumed cwd. +You also should specify the path of the target branch by `--target`. If not, +assumed cwd. +This means that you have to specify at least one of `--trunk` or `--target`. + +revision(s) can be below or their combinations: + 12345 # means r12345 + 12345,54321 # means r12345 and r54321 + 12345-12347 # means r12345, r12346 and r12347 (of course, if available) + +Note that the revisions is backported branch's ones, not trunk's. + +The target of this tool is *not* to generate ChangeLog automatically, but to +generate the draft of ChangeLog. +You have to check and modify the output. + EOS + exit +end + +Majors = { + "eregon" => "Benoit Daloze <eregontp@gmail.com>", + "kazu" => "Kazuhiro NISHIYAMA <zn@mbf.nifty.com>", + "ko1" => "Koichi Sasada <ko1@atdot.net>", + "marcandre" => "Marc-Andre Lafortune <ruby-core@marc-andre.ca>", + "naruse" => "NARUSE, Yui <naruse@ruby-lang.org>", + "nobu" => "Nobuyoshi Nakada <nobu@ruby-lang.org>", + "normal" => "Eric Wong <normalperson@yhbt.net>", + "rhe" => "Kazuki Yamaguchi <k@rhe.jp>", + "shugo" => "Shugo Maeda <shugo@ruby-lang.org>", + "stomar" => "Marcus Stollsteimer <sto.mar@web.de>", + "usa" => "NAKAMURA Usaku <usa@ruby-lang.org>", + "zzak" => "Zachary Scott <e@zzak.io>", +} + +trunk = "." +target = "." +ARGV.delete_if{|e| /^--trunk=(.*)/ =~ e && trunk = $1} +ARGV.delete_if{|e| /^--target=(.*)/ =~ e && target = $1} +usage! if ARGV.size == 0 || trunk == target + +revisions = [] +ARGV.each do |a| + a.split(/,/).each do |b| + if /-/ =~ b + revisions += Range.new(*b.split(/-/, 2).map{|e| Integer(e)}).to_a + else + revisions << Integer(b) + end + end +end +revisions.sort! +revisions.reverse! + +revisions.each do |rev| + if /^Index: ChangeLog$/ =~ `svn diff -c #{rev} #{target}` + STDERR.puts "#{rev} already has ChangeLog. Skip." + else + lines = `svn log -r #{rev} #{target}`.lines[1..-2] + if lines.empty? + STDERR.puts "#{rev} does not exist. Skip." + next + end + unless /^merge revision\(s\) (\d+)/ =~ lines[2] + STDERR.puts "#{rev} is not seems to be a merge commit. Skip." + next + end + original = $1 + committer = `svn log -r #{original} #{trunk}`.lines[1].split(/\|/)[1].strip + if Majors[committer] + committer = Majors[committer] + else + committer = "#{committer} <#{committer}@ruby-lang.org>" + end + time = Time.parse(lines.shift.split(/\|/)[2]).getlocal("+09:00") + puts "#{time.asctime} #{committer}" + puts + lines.shift(2) # skip "merge" line + lines.shift while lines.first == "\n" + lines.pop while lines.last == "\n" + lines.each do |line| + line.chomp! + line = "\t#{line}" if line[0] != "\t" && line != "" + puts line + end + puts + STDERR.puts "#{rev} is processed." + end +end diff --git a/tool/generic_erb.rb b/tool/generic_erb.rb new file mode 100644 index 0000000000..c9986c7284 --- /dev/null +++ b/tool/generic_erb.rb @@ -0,0 +1,58 @@ +# -*- coding: us-ascii -*- + +# Used to expand Ruby template files by common.mk, uncommon.mk and +# some Ruby extension libraries. + +require 'erb' +require 'optparse' +require 'fileutils' +$:.unshift(File.dirname(__FILE__)) +require 'vpath' +require 'colorize' + +vpath = VPath.new +timestamp = nil +output = nil +ifchange = nil +source = false +color = nil +templates = [] + +ARGV.options do |o| + o.on('-t', '--timestamp[=PATH]') {|v| timestamp = v || true} + o.on('-i', '--input=PATH') {|v| template << v} + o.on('-o', '--output=PATH') {|v| output = v} + o.on('-c', '--[no-]if-change') {|v| ifchange = v} + o.on('-x', '--source') {source = true} + o.on('--color') {color = true} + vpath.def_options(o) + o.order!(ARGV) + templates << (ARGV.shift or abort o.to_s) if templates.empty? +end +color = Colorize.new(color) +unchanged = color.pass("unchanged") +updated = color.fail("updated") + +result = templates.map do |template| + erb = ERB.new(File.read(template), nil, '%-') + erb.filename = template + source ? erb.src : proc{erb.result(binding)}.call +end +result = result.size == 1 ? result[0] : result.join("") +if output + if ifchange and (vpath.open(output, "rb") {|f| f.read} rescue nil) == result + puts "#{output} #{unchanged}" + else + open(output, "wb") {|f| f.print result} + puts "#{output} #{updated}" + end + if timestamp + if timestamp == true + dir, base = File.split(output) + timestamp = File.join(dir, ".time." + base) + end + FileUtils.touch(timestamp) + end +else + print result +end diff --git a/tool/git-refresh b/tool/git-refresh new file mode 100755 index 0000000000..d36c4d80bc --- /dev/null +++ b/tool/git-refresh @@ -0,0 +1,43 @@ +#!/bin/sh +set -e + +if (cd -P .) 2>/dev/null; then + CHDIR='cd -P' +else + CHDIR='cd' +fi + +quiet= +branch= + +until [ $# = 0 ]; do + case "$1" in + --) shift; break;; + -C|--directory) shift; $CHDIR "$1";; + -C*) $CHDIR `expr "$1" : '-C\(.*\)'`;; + --directory=*) $CHDIR `expr "$1" : '[^=]*=\(.*\)'`;; + -q) quiet=1;; + -b|--branch) shift; branch="$1";; + -b*) branch=`expr "$1" : '-b\(.*\)'`;; + --branch=*) branch=`expr "$1" : '[^=]*=\(.*\)'`;; + -*) echo "unknown option: $1" 1>&2; exit 1;; + *) break;; + esac + shift +done + +url="$1" +dir="$2" +shift 2 +[ x"$branch" = x ] && unset branch || : +if [ -d "$dir" ]; then + echo updating `expr "/$dir/" : '.*/\([^/][^/]*\)/'` ... + [ $quiet ] || set -x + $CHDIR "$dir" + ${branch+git} ${branch+fetch} ${branch+"$@"} + exec git ${branch+checkout} "${branch-pull}" "$@" +else + echo retrieving `expr "/$dir/" : '.*/\([^/][^/]*\)/'` ... + [ $quiet ] || set -x + exec git clone ${branch+--branch} ${branch+"$branch"} "$url" "$dir" "$@" +fi diff --git a/tool/gperf.sed b/tool/gperf.sed new file mode 100644 index 0000000000..6b3e1980be --- /dev/null +++ b/tool/gperf.sed @@ -0,0 +1,22 @@ +/ANSI-C code/{ + h + s/.*/ANSI:offset:/ + x +} +/\/\*!ANSI{\*\//{ + G + s/\/\*!ANSI{\*\/\(.*\)\/\*}!ANSI\*\/\(.*\)\nANSI:.*/\/\*\1\*\/\2/ +} +s/(int)([a-z_]*)&((struct \([a-zA-Z_0-9][a-zA-Z_0-9]*\)_t *\*)0)->\1_str\([1-9][0-9]*\),/gperf_offsetof(\1, \2),/g +/^#line/{ + G + x + s/:offset:/:/ + x + s/\(.*\)\(\n\).*:offset:.*/#define gperf_offsetof(s, n) (short)offsetof(struct s##_t, s##_str##n)\2\1/ + s/\n[^#].*// +} +/^[a-zA-Z_0-9]*hash/,/^}/{ + s/ hval = / hval = (unsigned int)/ + s/ return / return (unsigned int)/ +} diff --git a/tool/id2token.rb b/tool/id2token.rb new file mode 100755 index 0000000000..706154ecfb --- /dev/null +++ b/tool/id2token.rb @@ -0,0 +1,27 @@ +#! /usr/bin/ruby -p +# -*- coding: us-ascii -*- + +# Used to build the Ruby parsing code in common.mk and Ripper. + +BEGIN { + require 'optparse' + $:.unshift(File.dirname(__FILE__)) + require 'vpath' + vpath = VPath.new + header = nil + + opt = OptionParser.new do |o| + vpath.def_options(o) + header = o.order!(ARGV).shift + end or abort opt.opt_s + + TOKENS = {} + h = vpath.read(header) rescue abort("#{header} not found in #{vpath.inspect}") + h.scan(/^#define\s+RUBY_TOKEN_(\w+)\s+(\d+)/) do |token, id| + TOKENS[token] = id + end + + TOKENS_RE = /\bRUBY_TOKEN\((#{TOKENS.keys.join('|')})\)\s*(?=\s)/ +} + +$_.gsub!(TOKENS_RE) {TOKENS[$1]} if /^%token/ =~ $_ diff --git a/tool/ifchange b/tool/ifchange new file mode 100755 index 0000000000..396a7a0bcf --- /dev/null +++ b/tool/ifchange @@ -0,0 +1,87 @@ +#!/bin/sh +# usage: ifchange target temporary + +# Used in generating revision.h via Makefiles. + +set -e +timestamp= +keepsuffix= +empty= +color=auto +until [ $# -eq 0 ]; do + case "$1" in + --timestamp) + timestamp=. + ;; + --timestamp=*) + timestamp=`expr \( "$1" : '[^=]*=\(.*\)' \)` + ;; + --keep) + keepsuffix=.old + ;; + --keep=*) + keepsuffix=`expr \( "$1" : '[^=]*=\(.*\)' \)` + ;; + --empty) + empty=yes + ;; + --color) + color=always + ;; + --color=*) + color=`expr \( "$1" : '[^=]*=\(.*\)' \)` + ;; + *) + break + ;; + esac + shift +done + +target="$1" +temp="$2" +if [ "$temp" = - ]; then + temp="tmpdata$$.tmp~" + cat > "$temp" + trap 'rm -f "$temp"' 0 +fi + +msg_begin= msg_unchanged= msg_updated= msg_reset= +if [ "$color" = always -o \( "$color" = auto -a -t 1 \) ]; then + msg_begin="[" + case "`tput smso 2>/dev/null`" in + "$msg_begin"*m) + if [ ${TEST_COLORS:+set} ]; then + msg_unchanged=`expr ":$TEST_COLORS:" : ".*:pass=\([^:]*\):"` || : + msg_updated=`expr ":$TEST_COLORS:" : ".*:fail=\([^:]*\):"` || : + fi + msg_unchanged="${msg_begin}${msg_unchanged:-32}m" + msg_updated="${msg_begin}${msg_updated:-31;1}m" + msg_reset="${msg_begin}m" + ;; + esac + unset msg_begin +fi + +targetdir= +case "$target" in */*) targetdir=`dirname "$target"`;; esac +if [ -f "$target" -a ! -${empty:+f}${empty:-s} "$temp" ] || cmp "$target" "$temp" >/dev/null 2>&1; then + echo "$target ${msg_unchanged}unchanged${msg_reset}" + rm -f "$temp" +else + echo "$target ${msg_updated}updated${msg_reset}" + [ x"${targetdir}" = x -o -d "${targetdir}" ] || mkdir -p "${targetdir}" + [ x"${keepsuffix}" != x -a -f "$target" ] && mv -f "$target" "${target}${keepsuffix}" + mv -f "$temp" "$target" +fi + +if [ -n "${timestamp}" ]; then + if [ x"${timestamp}" = x. ]; then + if [ x"$targetdir" = x ]; then + timestamp=.time."$target" + else + timestamp="$targetdir"/.time.`basename "$target"` + fi + fi + : > "$timestamp" +fi diff --git a/tool/insns2vm.rb b/tool/insns2vm.rb new file mode 100755 index 0000000000..ecbbb52643 --- /dev/null +++ b/tool/insns2vm.rb @@ -0,0 +1,18 @@ +#!ruby + +# This is used by Makefile.in to generate .inc files. +# See Makefile.in for details. + +require 'optparse' + +Version = %w$Revision: 11626 $[1..-1] + +require "#{File.join(File.dirname(__FILE__), 'instruction')}" + +if $0 == __FILE__ + opts = ARGV.options + maker = RubyVM::SourceCodeGenerator.def_options(opts) + files = opts.parse! + generator = maker.call + generator.generate(files) +end diff --git a/tool/install-sh b/tool/install-sh new file mode 100644 index 0000000000..11e502f56d --- /dev/null +++ b/tool/install-sh @@ -0,0 +1,17 @@ +#!/bin/sh + +# Just only for using AC_PROG_INSTALL in configure.ac. +# See autoconf.info for more detail. + +cat <<EOF >&2 +Ruby uses a BSD-compatible install(1) if possible. If not, Ruby +provides its own install(1) alternative. + +This script is a place holder for AC_PROG_INSTALL in configure.ac. + +Please report a bug in Ruby to http://bugs.ruby-lang.org if you see +this message. + +Thank you. +EOF +exit 1 diff --git a/tool/instruction.rb b/tool/instruction.rb new file mode 100755 index 0000000000..447fed5948 --- /dev/null +++ b/tool/instruction.rb @@ -0,0 +1,1249 @@ +#!./miniruby +# -*- coding: us-ascii -*- +# +# This library is used by insns2vm.rb as part of autogenerating +# instruction files with .inc extensions like insns.inc and vm.inc. + +require 'erb' +$:.unshift(File.dirname(__FILE__)) +require 'vpath' + +class RubyVM + class Instruction + def initialize name, opes, pops, rets, comm, body, tvars, sp_inc, + orig = self, defopes = [], type = nil, + nsc = [], psc = [[], []] + + @name = name + @opes = opes # [[type, name], ...] + @pops = pops # [[type, name], ...] + @rets = rets # [[type, name], ...] + @comm = comm # {:c => category, :e => en desc, :j => ja desc} + @body = body # '...' + + @orig = orig + @defopes = defopes + @type = type + @tvars = tvars + + @nextsc = nsc + @pushsc = psc + @sc = [] + @unifs = [] + @optimized = [] + @is_sc = false + @sp_inc = sp_inc + @trace = trace + end + + def add_sc sci + @sc << sci + sci.set_sc + end + + attr_reader :name, :opes, :pops, :rets + attr_reader :body, :comm + attr_reader :nextsc, :pushsc + attr_reader :orig, :defopes, :type + attr_reader :sc + attr_reader :unifs, :optimized + attr_reader :is_sc + attr_reader :tvars + attr_reader :sp_inc + attr_accessor :trace + + def set_sc + @is_sc = true + end + + def add_unif insns + @unifs << insns + end + + def add_optimized insn + @optimized << insn + end + + def sp_increase_c_expr + if(pops.any?{|t, v| v == '...'} || + rets.any?{|t, v| v == '...'}) + # user definition + raise "no sp increase definition" if @sp_inc.nil? + ret = "int inc = 0;\n" + + @opes.each_with_index{|(t, v), i| + if (t == 'rb_num_t' && ((re = /\b#{v}\b/n) =~ @sp_inc)) || + (@defopes.any?{|t, val| re =~ val}) + ret << " int #{v} = FIX2INT(opes[#{i}]);\n" + elsif (t == 'CALL_INFO' && ((re = /\b#{v}\b/n) =~ @sp_inc)) + ret << " CALL_INFO #{v} = (CALL_INFO)(opes[#{i}]);\n" + end + } + + @defopes.each_with_index{|((t, var), val), i| + if t == 'rb_num_t' && val != '*' && /\b#{var}\b/ =~ @sp_inc + ret << " #{t} #{var} = #{val};\n" + end + } + + ret << " #{@sp_inc};\n" + ret << " return depth + inc;" + ret + else + "return depth + #{rets.size - pops.size};" + end + end + + def inspect + "#<Instruction:#{@name}>" + end + end + + class InstructionsLoader + def initialize opts = {} + @insns = [] + @insn_map = {} + + @vpath = opts[:VPATH] || File + @use_const = opts[:use_const] + @verbose = opts[:verbose] + @destdir = opts[:destdir] + + (@vm_opts = load_vm_opts).each {|k, v| + @vm_opts[k] = opts[k] if opts.key?(k) + } + + load_insns_def opts[:"insns.def"] || 'insns.def' + + load_opt_operand_def opts[:"opope.def"] || 'defs/opt_operand.def' + load_insn_unification_def opts[:"unif.def"] || 'defs/opt_insn_unif.def' + make_stackcaching_insns if vm_opt?('STACK_CACHING') + make_trace_insns + end + + attr_reader :vpath + attr_reader :destdir + + %w[use_const verbose].each do |attr| + attr_reader attr + alias_method "#{attr}?", attr + remove_method attr + end + + def [](s) + @insn_map[s.to_s] + end + + def each + @insns.each{|insn| + yield insn + } + end + + def size + @insns.size + end + + ### + private + + def vm_opt? name + @vm_opts[name] + end + + def load_vm_opts file = nil + file ||= 'vm_opts.h' + opts = {} + vpath.open(file) do |f| + f.grep(/^\#define\s+OPT_([A-Z_]+)\s+(\d+)/) do + opts[$1] = !$2.to_i.zero? + end + end + opts + end + + SKIP_COMMENT_PATTERN = Regexp.compile(Regexp.escape('/** ##skip')) + + include Enumerable + + def add_insn insn + @insns << insn + @insn_map[insn.name] = insn + end + + def make_insn name, opes, pops, rets, comm, body, sp_inc + add_insn Instruction.new(name, opes, pops, rets, comm, body, [], sp_inc) + end + + # str -> [[type, var], ...] + def parse_vars line + raise unless /\((.*?)\)/ =~ line + vars = $1.split(',') + vars.map!{|v| + if /\s*(\S+)\s+(\S+)\s*/ =~ v + type = $1 + var = $2 + elsif /\s*\.\.\.\s*/ =~ v + type = var = '...' + else + raise + end + [type, var] + } + vars + end + + def parse_comment comm + c = 'others' + j = '' + e = '' + comm.each_line{|line| + case line + when /@c (.+)/ + c = $1 + when /@e (.+)/ + e = $1 + when /@e\s*$/ + e = '' + when /@j (.+)$/ + j = $1 + when /@j\s*$/ + j = '' + end + } + { :c => c, + :e => e, + :j => j, + } + end + + def load_insns_def file + body = insn = opes = pops = rets = nil + comment = '' + + vpath.open(file) {|f| + f.instance_variable_set(:@line_no, 0) + class << f + def line_no + @line_no + end + def gets + @line_no += 1 + super + end + end + + while line = f.gets + line.chomp! + case line + + when SKIP_COMMENT_PATTERN + while line = f.gets.chomp + if /\s+\*\/$/ =~ line + break + end + end + + # collect instruction comment + when /^\/\*\*$/ + while line = f.gets + if /\s+\*\/\s*$/ =~ line + break + else + comment << line + end + end + + # start instruction body + when /^DEFINE_INSN$/ + insn = f.gets.chomp + opes = parse_vars(f.gets.chomp) + pops = parse_vars(f.gets.chomp).reverse + rets_str = f.gets.chomp + rets = parse_vars(rets_str).reverse + comment = parse_comment(comment) + insn_in = true + body = '' + + sp_inc = rets_str[%r"//\s*(.+)", 1] + + raise unless /^\{$/ =~ f.gets.chomp + line_no = f.line_no + + # end instruction body + when /^\}/ + if insn_in + body.instance_variable_set(:@line_no, line_no) + body.instance_variable_set(:@file, f.path) + insn = make_insn(insn, opes, pops, rets, comment, body, sp_inc) + insn_in = false + comment = '' + end + + else + if insn_in + body << line + "\n" + end + end + end + } + end + + ## opt op + def load_opt_operand_def file + vpath.foreach(file) {|line| + line = line.gsub(/\#.*/, '').strip + next if line.length == 0 + break if /__END__/ =~ line + /(\S+)\s+(.+)/ =~ line + insn = $1 + opts = $2 + add_opt_operand insn, opts.split(/,/).map{|e| e.strip} + } if file + end + + def label_escape label + label.gsub(/\(/, '_O_'). + gsub(/\)/, '_C_'). + gsub(/\*/, '_WC_') + end + + def add_opt_operand insn_name, opts + insn = @insn_map[insn_name] + opes = insn.opes + + if opes.size != opts.size + raise "operand size mismatch for #{insn.name} (opes: #{opes.size}, opts: #{opts.size})" + end + + ninsn = insn.name + '_OP_' + opts.map{|e| label_escape(e)}.join('_') + nopes = [] + defv = [] + + opts.each_with_index{|e, i| + if e == '*' + nopes << opes[i] + end + defv << [opes[i], e] + } + + make_insn_operand_optimized(insn, ninsn, nopes, defv) + end + + def make_insn_operand_optimized orig_insn, name, opes, defopes + comm = orig_insn.comm.dup + comm[:c] = 'optimize' + add_insn insn = Instruction.new( + name, opes, orig_insn.pops, orig_insn.rets, comm, + orig_insn.body, orig_insn.tvars, orig_insn.sp_inc, + orig_insn, defopes) + orig_insn.add_optimized insn + end + + ## insn unif + def load_insn_unification_def file + vpath.foreach(file) {|line| + line = line.gsub(/\#.*/, '').strip + next if line.length == 0 + break if /__END__/ =~ line + make_unified_insns line.split.map{|e| + raise "unknown insn: #{e}" unless @insn_map[e] + @insn_map[e] + } + } if file + end + + def all_combination sets + ret = sets.shift.map{|e| [e]} + + sets.each{|set| + prev = ret + ret = [] + prev.each{|ary| + set.each{|e| + eary = ary.dup + eary << e + ret << eary + } + } + } + ret + end + + def make_unified_insns insns + if vm_opt?('UNIFY_ALL_COMBINATION') + insn_sets = insns.map{|insn| + [insn] + insn.optimized + } + + all_combination(insn_sets).each{|insns_set| + make_unified_insn_each insns_set + } + else + make_unified_insn_each insns + end + end + + def mk_private_val vals, i, redef + vals.dup.map{|v| + # v[0] : type + # v[1] : var name + + v = v.dup + if v[0] != '...' + redef[v[1]] = v[0] + v[1] = "#{v[1]}_#{i}" + end + v + } + end + + def mk_private_val2 vals, i, redef + vals.dup.map{|v| + # v[0][0] : type + # v[0][1] : var name + # v[1] : default val + + pv = v.dup + v = pv[0] = pv[0].dup + if v[0] != '...' + redef[v[1]] = v[0] + v[1] = "#{v[1]}_#{i}" + end + pv + } + end + + def make_unified_insn_each insns + names = [] + opes = [] + pops = [] + rets = [] + comm = { + :c => 'optimize', + :e => 'unified insn', + :j => 'unified insn', + } + body = '' + passed = [] + tvars = [] + defopes = [] + sp_inc = '' + + insns.each_with_index{|insn, i| + names << insn.name + redef_vars = {} + + e_opes = mk_private_val(insn.opes, i, redef_vars) + e_pops = mk_private_val(insn.pops, i, redef_vars) + e_rets = mk_private_val(insn.rets, i, redef_vars) + # ToDo: fix it + e_defs = mk_private_val2(insn.defopes, i, redef_vars) + + passed_vars = [] + while pvar = e_pops.pop + rvar = rets.pop + if rvar + raise "unsupported unif insn: #{insns.inspect}" if rvar[0] == '...' + passed_vars << [pvar, rvar] + tvars << rvar + else + e_pops.push pvar + break + end + end + + opes.concat e_opes + pops.concat e_pops + rets.concat e_rets + defopes.concat e_defs + sp_inc << "#{insn.sp_inc}" + + body << "{ /* unif: #{i} */\n" + + passed_vars.map{|rpvars| + pv = rpvars[0] + rv = rpvars[1] + "#define #{pv[1]} #{rv[1]}" + }.join("\n") + + "\n" + + redef_vars.map{|v, type| + "#{type} #{v} = #{v}_#{i};" + }.join("\n") + "\n" + if line = insn.body.instance_variable_get(:@line_no) + file = insn.body.instance_variable_get(:@file) + body << "#line #{line+1} \"#{file}\"\n" + body << insn.body + body << "\n#line __CURRENT_LINE__ \"__CURRENT_FILE__\"\n" + else + body << insn.body + end + body << redef_vars.keys.map{|v| + "#{v}_#{i} = #{v};" + }.join("\n") + + "\n" + + passed_vars.map{|rpvars| + "#undef #{rpvars[0][1]}" + }.join("\n") + + "\n}\n" + } + + tvars_ary = [] + tvars.each{|tvar| + unless opes.any?{|var| + var[1] == tvar[1] + } || defopes.any?{|pvar| + pvar[0][1] == tvar[1] + } + tvars_ary << tvar + end + } + add_insn insn = Instruction.new("UNIFIED_" + names.join('_'), + opes, pops, rets.reverse, comm, body, + tvars_ary, sp_inc) + insn.defopes.replace defopes + insns[0].add_unif [insn, insns] + end + + ## sc + SPECIAL_INSN_FOR_SC_AFTER = { + /\Asend/ => [:a], + /\Aend/ => [:a], + /\Ayield/ => [:a], + /\Aclassdef/ => [:a], + /\Amoduledef/ => [:a], + } + FROM_SC = [[], [:a], [:b], [:a, :b], [:b, :a]] + + def make_stackcaching_insns + pops = rets = nil + + @insns.dup.each{|insn| + opops = insn.pops + orets = insn.rets + oopes = insn.opes + ocomm = insn.comm + oname = insn.name + + after = SPECIAL_INSN_FOR_SC_AFTER.find {|k, v| k =~ oname} + + insns = [] + FROM_SC.each{|from| + name, pops, rets, pushs1, pushs2, nextsc = + *calc_stack(insn, from, after, opops, orets) + + make_insn_sc(insn, name, oopes, pops, rets, [pushs1, pushs2], nextsc) + } + } + end + + def make_trace_insns + @insns.dup.each{|insn| + body = <<-EOS + vm_trace(ec, GET_CFP(), GET_PC()); + DISPATCH_ORIGINAL_INSN(#{insn.name}); + EOS + + trace_insn = Instruction.new(name = "trace_#{insn.name}", + insn.opes, insn.pops, insn.rets, insn.comm, + body, insn.tvars, insn.sp_inc) + trace_insn.trace = true + add_insn trace_insn + } + end + + def make_insn_sc orig_insn, name, opes, pops, rets, pushs, nextsc + comm = orig_insn.comm.dup + comm[:c] = 'optimize(sc)' + + scinsn = Instruction.new( + name, opes, pops, rets, comm, + orig_insn.body, orig_insn.tvars, orig_insn.sp_inc, + orig_insn, orig_insn.defopes, :sc, nextsc, pushs) + + add_insn scinsn + orig_insn.add_sc scinsn + end + + def self.complement_name st + "#{st[0] ? st[0] : 'x'}#{st[1] ? st[1] : 'x'}" + end + + def add_stack_value st + len = st.length + if len == 0 + st[0] = :a + [nil, :a] + elsif len == 1 + if st[0] == :a + st[1] = :b + else + st[1] = :a + end + [nil, st[1]] + else + st[0], st[1] = st[1], st[0] + [st[1], st[1]] + end + end + + def calc_stack insn, ofrom, oafter, opops, orets + from = ofrom.dup + pops = opops.dup + rets = orets.dup + rest_scr = ofrom.dup + + pushs_before = [] + pushs= [] + + pops.each_with_index{|e, i| + if e[0] == '...' + pushs_before = from + from = [] + end + r = from.pop + break unless r + pops[i] = pops[i].dup << r + } + + if oafter + from = oafter + from.each_with_index{|r, i| + rets[i] = rets[i].dup << r if rets[i] + } + else + rets = rets.reverse + rets.each_with_index{|e, i| + break if e[0] == '...' + pushed, r = add_stack_value from + rets[i] = rets[i].dup << r + if pushed + if rest_scr.pop + pushs << pushed + end + + if i - 2 >= 0 + rets[i-2].pop + end + end + } + end + + if false #|| insn.name =~ /test3/ + p ofrom + p pops + p rets + p pushs_before + p pushs + p from + exit + end + + ret = ["#{insn.name}_SC_#{InstructionsLoader.complement_name(ofrom)}_#{complement_name(from)}", + pops, rets, pushs_before, pushs, from] + end + end + + class SourceCodeGenerator + def initialize insns + @insns = insns + end + + attr_reader :insns + + def generate + raise "should not reach here" + end + + def vpath + @insns.vpath + end + + def verbose? + @insns.verbose? + end + + def use_const? + @insns.use_const? + end + + def template(name) + ERB.new(vpath.read("template/#{name}"), nil, '%-') + end + + def build_string + @lines = [] + yield + @lines.join("\n") + end + + EMPTY_STRING = ''.freeze + + def commit str = EMPTY_STRING + @lines << str + end + + def comment str + @lines << str if verbose? + end + + def output_path(fn) + d = @insns.destdir + fn = File.join(d, fn) if d + fn + end + end + + ################################################################### + # vm.inc + class VmBodyGenerator < SourceCodeGenerator + # vm.inc + def generate + template('vm.inc.tmpl').result(binding) + end + + def generate_from_insnname insnname + make_insn_def @insns[insnname.to_s] + end + + ####### + private + + def make_header_prepare_stack insn + comment " /* prepare stack status */" + + push_ba = insn.pushsc + raise "unsupport" if push_ba[0].size > 0 && push_ba[1].size > 0 + + n = 0 + push_ba.each {|pushs| n += pushs.length} + commit " CHECK_VM_STACK_OVERFLOW_FOR_INSN(VM_REG_CFP, #{n});" if n > 0 + push_ba.each{|pushs| + pushs.each{|r| + commit " PUSH(SCREG(#{r}));" + } + } + end + + def make_header_operands insn + comment " /* declare and get from iseq */" + + vars = insn.opes + n = 0 + ops = [] + + vars.each_with_index{|(type, var), i| + if type == '...' + break + end + + # skip make operands when body has no reference to this operand + # TODO: really needed? + re = /\b#{var}\b/n + if re =~ insn.body or re =~ insn.sp_inc or insn.rets.any?{|t, v| re =~ v} or re =~ 'ic' or re =~ 'ci' or re =~ 'cc' + ops << " #{type} #{var} = (#{type})GET_OPERAND(#{i+1});" + end + + n += 1 + } + @opn = n + + # reverse or not? + # ops.join + commit ops.reverse + end + + def make_header_default_operands insn + vars = insn.defopes + + vars.each{|e| + next if e[1] == '*' + if use_const? + commit " const #{e[0][0]} #{e[0][1]} = #{e[1]};" + else + commit " #define #{e[0][1]} #{e[1]}" + end + } + end + + def make_footer_default_operands insn + comment " /* declare and initialize default opes */" + if use_const? + commit + else + vars = insn.defopes + + vars.each{|e| + next if e[1] == '*' + commit "#undef #{e[0][1]}" + } + end + end + + def make_header_stack_pops insn + comment " /* declare and pop from stack */" + + n = 0 + pops = [] + vars = insn.pops + vars.each_with_index{|iter, i| + type, var, r = *iter + if type == '...' + break + end + if r + pops << " #{type} #{var} = SCREG(#{r});" + else + pops << " #{type} #{var} = TOPN(#{n});" + n += 1 + end + } + @popn = n + + # reverse or not? + commit pops.reverse + end + + def make_header_temporary_vars insn + comment " /* declare temporary vars */" + + insn.tvars.each{|var| + commit " #{var[0]} #{var[1]};" + } + end + + def make_header_stack_val insn + comment "/* declare stack push val */" + + vars = insn.opes + insn.pops + insn.defopes.map{|e| e[0]} + + insn.rets.each{|var| + if vars.all?{|e| e[1] != var[1]} && var[1] != '...' + commit " #{var[0]} #{var[1]};" + end + } + end + + def make_header_analysis insn + commit " COLLECT_USAGE_INSN(BIN(#{insn.name}));" + insn.opes.each_with_index{|op, i| + commit " COLLECT_USAGE_OPERAND(BIN(#{insn.name}), #{i}, #{op[1]});" + } + end + + def make_header_pc insn + commit " ADD_PC(1+#{@opn});" + commit " PREFETCH(GET_PC());" + end + + def make_header_popn insn + comment " /* management */" + commit " POPN(#{@popn});" if @popn > 0 + end + + def make_header_debug insn + comment " /* for debug */" + commit " DEBUG_ENTER_INSN(\"#{insn.name}\");" + end + + def make_header_defines insn + commit " #define CURRENT_INSN_#{insn.name} 1" + commit " #define INSN_IS_SC() #{insn.sc ? 0 : 1}" + commit " #define INSN_LABEL(lab) LABEL_#{insn.name}_##lab" + commit " #define LABEL_IS_SC(lab) LABEL_##lab##_###{insn.sc.size == 0 ? 't' : 'f'}" + end + + def each_footer_stack_val insn + insn.rets.reverse_each{|v| + break if v[1] == '...' + yield v + } + end + + def make_footer_stack_val insn + comment " /* push stack val */" + + n = 0 + each_footer_stack_val(insn){|v| + n += 1 unless v[2] + } + commit " CHECK_VM_STACK_OVERFLOW_FOR_INSN(VM_REG_CFP, #{n});" if n > 0 + each_footer_stack_val(insn){|v| + if v[2] + commit " SCREG(#{v[2]}) = #{v[1]};" + else + commit " PUSH(#{v[1]});" + end + } + end + + def make_footer_undefs insn + commit "#undef CURRENT_INSN_#{insn.name}" + commit "#undef INSN_IS_SC" + commit "#undef INSN_LABEL" + commit "#undef LABEL_IS_SC" + end + + def make_header insn + label = insn.trace ? '' : "START_OF_ORIGINAL_INSN(#{insn.name});" + commit "INSN_ENTRY(#{insn.name}){#{label}" + make_header_prepare_stack insn + commit "{" + unless insn.trace + make_header_stack_val insn + make_header_default_operands insn + make_header_operands insn + make_header_stack_pops insn + make_header_temporary_vars insn + # + make_header_debug insn + make_header_pc insn + make_header_popn insn + make_header_defines insn + make_header_analysis insn + end + commit "{" + end + + def make_footer insn + unless insn.trace + make_footer_stack_val insn + make_footer_default_operands insn + make_footer_undefs insn + end + commit " END_INSN(#{insn.name});}}}" + end + + def make_insn_def insn + build_string do + make_header insn + if line = insn.body.instance_variable_get(:@line_no) + file = insn.body.instance_variable_get(:@file) + commit "#line #{line+1} \"#{file}\"" + commit insn.body + commit '#line __CURRENT_LINE__ "__CURRENT_FILE__"' + else + commit insn.body + end + make_footer(insn) + end + end + end + + ################################################################### + # vmtc.inc + class VmTCIncGenerator < SourceCodeGenerator + def generate + template('vmtc.inc.tmpl').result(binding) + end + end + + ################################################################### + # insns_info.inc + class InsnsInfoIncGenerator < SourceCodeGenerator + def generate + template('insns_info.inc.tmpl').result(binding) + end + + ### + private + + def op2typesig op + case op + when /^OFFSET/ + "TS_OFFSET" + when /^rb_num_t/ + "TS_NUM" + when /^lindex_t/ + "TS_LINDEX" + when /^VALUE/ + "TS_VALUE" + when /^ID/ + "TS_ID" + when /GENTRY/ + "TS_GENTRY" + when /^IC/ + "TS_IC" + when /^CALL_INFO/ + "TS_CALLINFO" + when /^CALL_CACHE/ + "TS_CALLCACHE" + when /^\.\.\./ + "TS_VARIABLE" + when /^CDHASH/ + "TS_CDHASH" + when /^ISEQ/ + "TS_ISEQ" + when /rb_insn_func_t/ + "TS_FUNCPTR" + else + raise "unknown op type: #{op}" + end + end + + TYPE_CHARS = { + 'TS_OFFSET' => 'O', + 'TS_NUM' => 'N', + 'TS_LINDEX' => 'L', + 'TS_VALUE' => 'V', + 'TS_ID' => 'I', + 'TS_GENTRY' => 'G', + 'TS_IC' => 'K', + 'TS_CALLINFO' => 'C', + 'TS_CALLCACHE' => 'E', + 'TS_CDHASH' => 'H', + 'TS_ISEQ' => 'S', + 'TS_VARIABLE' => '.', + 'TS_FUNCPTR' => 'F', + } + + def max_length(array) + max = 0 + array.each do |i| + if (n = i.length) > max + max = n + end + end + max + end + end + + ################################################################### + # insns.inc + class InsnsIncGenerator < SourceCodeGenerator + def generate + template('insns.inc.tmpl').result(binding) + end + end + + ################################################################### + # minsns.inc + class MInsnsIncGenerator < SourceCodeGenerator + def generate + template('minsns.inc.tmpl').result(binding) + end + end + + ################################################################### + # optinsn.inc + class OptInsnIncGenerator < SourceCodeGenerator + def generate + optinsn_inc + end + + ### + private + + def val_as_type op + type = op[0][0] + val = op[1] + + case type + when /^long/, /^rb_num_t/, /^lindex_t/ + "INT2FIX(#{val})" + when /^VALUE/ + val + when /^ID/ + "INT2FIX(#{val})" + when /^ISEQ/, /^rb_insn_func_t/ + val + when /GENTRY/ + raise + when /^\.\.\./ + raise + else + raise "type: #{type}" + end + end + + # optinsn.inc + def optinsn_inc + opt_insns_map = Hash.new{[]} + + @insns.each{|insn| + next if insn.defopes.size == 0 + next if insn.type == :sc + next if /^UNIFIED/ =~ insn.name.to_s + + originsn = insn.orig + opt_insns_map[originsn] <<= insn + } + + opt_insns_map.each_value do |optinsns| + sorted = optinsns.sort_by {|opti| + opti.defopes.find_all{|e| e[1] == '*'}.size + } + optinsns.replace(sorted) + end + + template('optinsn.inc.tmpl').result(binding) + end + end + + ################################################################### + # optunifs.inc + class OptUnifsIncGenerator < SourceCodeGenerator + def generate + template('optunifs.inc.tmpl').result(binding) + end + end + + ################################################################### + # opt_sc.inc + class OptSCIncGenerator < SourceCodeGenerator + def generate + sc_insn_info = [] + @insns.each{|insn| + insns = insn.sc + if insns.size > 0 + insns = ['SC_ERROR'] + insns.map{|e| " BIN(#{e.name})"} + else + insns = Array.new(6){'SC_ERROR'} + end + sc_insn_info << " {\n#{insns.join(",\n")}}" + } + sc_insn_info = sc_insn_info.join(",\n") + + sc_insn_next = @insns.map{|insn| + " SCS_#{InstructionsLoader.complement_name(insn.nextsc).upcase}" + + (verbose? ? " /* #{insn.name} */" : '') + }.join(",\n") + template('opt_sc.inc.tmpl').result(binding) + end + end + + ################################################################### + # yasmdata.rb + class YASMDataRbGenerator < SourceCodeGenerator + def generate + insn_id2no = '' + @insns.each_with_index{|insn, i| + insn_id2no << " :#{insn.name} => #{i},\n" + } + template('yasmdata.rb').result(binding) + end + end + + ################################################################### + # yarvarch.* + class YARVDocGenerator < SourceCodeGenerator + def generate + + end + + def desc lang + d = '' + i = 0 + cat = nil + @insns.each{|insn| + seq = insn.opes.map{|t,v| v}.join(' ') + before = insn.pops.reverse.map{|t,v| v}.join(' ') + after = insn.rets.reverse.map{|t,v| v}.join(' ') + + if cat != insn.comm[:c] + d << "** #{insn.comm[:c]}\n\n" + cat = insn.comm[:c] + end + + d << "*** #{insn.name}\n" + d << "\n" + d << insn.comm[lang] << "\n\n" + d << ":instruction sequence: 0x%02x " % i << seq << "\n" + d << ":stack: #{before} => #{after}\n\n" + i+=1 + } + d + end + + def desc_ja + d = desc :j + template('yarvarch.ja').result(binding) + end + + def desc_en + d = desc :e + template('yarvarch.en').result(binding) + end + end + + class SourceCodeGenerator + Files = { # codes + 'vm.inc' => VmBodyGenerator, + 'vmtc.inc' => VmTCIncGenerator, + 'insns.inc' => InsnsIncGenerator, + 'insns_info.inc' => InsnsInfoIncGenerator, + # 'minsns.inc' => MInsnsIncGenerator, + 'optinsn.inc' => OptInsnIncGenerator, + 'optunifs.inc' => OptUnifsIncGenerator, + 'opt_sc.inc' => OptSCIncGenerator, + 'yasmdata.rb' => YASMDataRbGenerator, + } + + def generate args = [] + args = Files.keys if args.empty? + args.each{|fn| + s = Files[fn].new(@insns).generate + open(output_path(fn), 'w') {|f| f.puts(s)} + } + end + + def self.def_options(opt) + opts = { + :"insns.def" => 'insns.def', + :"opope.def" => 'defs/opt_operand.def', + :"unif.def" => 'defs/opt_insn_unif.def', + } + + opt.on("-Dname", /\AOPT_(\w+)\z/, "enable VM option") {|s, v| + opts[v] = true + } + opt.on("--enable=name[,name...]", Array, + "enable VM options (without OPT_ prefix)") {|*a| + a.each {|v| opts[v] = true} + } + opt.on("-Uname", /\AOPT_(\w+)\z/, "disable VM option") {|s, v| + opts[v] = false + } + opt.on("--disable=name[,name...]", Array, + "disable VM options (without OPT_ prefix)") {|*a| + a.each {|v| opts[v] = false} + } + opt.on("-i", "--insnsdef=FILE", "--instructions-def", + "instructions definition file") {|n| + opts[:insns_def] = n + } + opt.on("-o", "--opt-operanddef=FILE", "--opt-operand-def", + "vm option: operand definition file") {|n| + opts[:opope_def] = n + } + opt.on("-u", "--opt-insnunifdef=FILE", "--opt-insn-unif-def", + "vm option: instruction unification file") {|n| + opts[:unif_def] = n + } + opt.on("-C", "--[no-]use-const", + "use consts for default operands instead of macros") {|v| + opts[:use_const] = v + } + opt.on("-d", "--destdir", "--output-directory=DIR", + "make output file underneath DIR") {|v| + opts[:destdir] = v + } + opt.on("-V", "--[no-]verbose") {|v| + opts[:verbose] = v + } + + vpath = VPath.new + vpath.def_options(opt) + + proc { + opts[:VPATH] = vpath + build opts + } + end + + def self.build opts, vpath = ['./'] + opts[:VPATH] ||= VPath.new(*vpath) + self.new InstructionsLoader.new(opts) + end + end +end + diff --git a/tool/jisx0208.rb b/tool/jisx0208.rb new file mode 100644 index 0000000000..30185fb81b --- /dev/null +++ b/tool/jisx0208.rb @@ -0,0 +1,86 @@ +# Library used by tools/enc-emoji-citrus-gen.rb + +module JISX0208 + class Char + class << self + def from_sjis(sjis) + unless 0x8140 <= sjis && sjis <= 0xFCFC + raise ArgumentError, "out of the range of JIS X 0208: 0x#{sjis.to_s(16)}" + end + sjis_hi, sjis_lo = sjis >> 8, sjis & 0xFF + sjis_hi = (sjis_hi - ((sjis_hi <= 0x9F) ? 0x80 : 0xC0)) << 1 + if sjis_lo <= 0x9E + sjis_hi -= 1 + sjis_lo -= (sjis_lo <= 0x7E) ? 0x3F : 0x40 + else + sjis_lo -= 0x9E + end + return self.new(sjis_hi, sjis_lo) + end + end + + def initialize(row, cell=nil) + if cell + @code = row_cell_to_code(row, cell) + else + @code = row.to_int + end + end + + def ==(other) + if self.class === other + return Integer(self) == Integer(other) + end + return super(other) + end + + def to_int + return @code + end + + def hi + Integer(self) >> 8 + end + + def lo + Integer(self) & 0xFF + end + + def row + self.hi - 0x20 + end + + def cell + self.lo - 0x20 + end + + def succ + succ_hi, succ_lo = self.hi, self.lo + 1 + if succ_lo > 0x7E + succ_lo = 0x21 + succ_hi += 1 + end + return self.class.new(succ_hi << 8 | succ_lo) + end + + def to_sjis + h, l = self.hi, self.lo + h = (h + 1) / 2 + ((0x21..0x5E).include?(h) ? 0x70 : 0xB0) + l += self.hi.odd? ? 0x1F + ((l >= 0x60) ? 1 : 0) : 0x7E + return h << 8 | l + end + + def inspect + "#<JISX0208::Char:#{self.object_id.to_s(16)} sjis=#{self.to_sjis.to_s(16)} jis=#{self.to_int.to_s(16)}>" + end + + private + + def row_cell_to_code(row, cell) + unless 0 < row && (1..94).include?(cell) + raise ArgumentError, "out of row-cell range: #{row}-#{cell}" + end + return (row + 0x20) << 8 | (cell + 0x20) + end + end +end diff --git a/tool/make-snapshot b/tool/make-snapshot new file mode 100755 index 0000000000..b040aa73f8 --- /dev/null +++ b/tool/make-snapshot @@ -0,0 +1,467 @@ +#!/usr/bin/ruby -s +# -*- coding: us-ascii -*- +require 'uri' +require 'digest/sha1' +require 'digest/sha2' +require 'fileutils' +require 'shellwords' +require 'tmpdir' +require File.expand_path("../vcs", __FILE__) +require File.expand_path("../colorize", __FILE__) +STDOUT.sync = true + +$srcdir ||= nil +$exported = nil if ($exported ||= nil) == "" +$archname = nil if ($archname ||= nil) == "" +$keep_temp ||= nil +$patch_file ||= nil +$packages ||= nil +$digests ||= nil +$tooldir = File.expand_path("..", __FILE__) +$unicode_version = nil if ($unicode_version ||= nil) == "" +$colorize = Colorize.new + +def usage + <<USAGE +usage: #{File.basename $0} [option...] new-directory-to-save [version ...] +options: + -srcdir=PATH source directory path + -exported=PATH make snapshot from already exported working directory + -archname=NAME make the basename of snapshots NAME + -keep_temp keep temporary working directory + -patch_file=PATCH apply PATCH file after export + -packages=PKG[,...] make PKG packages (#{PACKAGES.keys.join(", ")}) + -digests=ALG[,...] show ALG digests (#{DIGESTS.join(", ")}) +version: + trunk, stable, branches/*, tags/*, X.Y, X.Y.Z, X.Y.Z-pL +each versions may be followed by optional @revision. +USAGE +end + +DIGESTS = %w[SHA1 SHA256 SHA512] +PACKAGES = { + "bzip" => %w".tar.bz2 bzip2 -c", + "gzip" => %w".tar.gz gzip -c", + "xz" => %w".tar.xz xz -c", + "zip" => %w".zip zip -qr", +} + +if system("7z", out: IO::NULL) + PACKAGES["gzip"] = %w".tar.gz 7z a dummy -tgzip -mx -so" + PACKAGES["zip"] = %w".zip 7z a -tzip -mx" << {out: IO::NULL} +end +if gzip = ENV.delete("GZIP") + PACKAGES["gzip"].concat(gzip.shellsplit) +end + +ENV["LC_ALL"] = ENV["LANG"] = "C" +SVNURL = URI.parse("http://svn.ruby-lang.org/repos/ruby/") +GITURL = URI.parse("git://github.com/ruby/ruby.git") +RUBY_VERSION_PATTERN = /^\#define\s+RUBY_VERSION\s+"([\d.]+)"/ + +ENV["VPATH"] ||= "include/ruby" +YACC = ENV["YACC"] ||= "bison" +ENV["BASERUBY"] ||= "ruby" +ENV["RUBY"] ||= "ruby" +ENV["MV"] ||= "mv" +ENV["RM"] ||= "rm -f" +ENV["MINIRUBY"] ||= "ruby" +ENV["PROGRAM"] ||= "ruby" +ENV["AUTOCONF"] ||= "autoconf" +ENV["BUILTIN_TRANSOBJS"] ||= "newline.o" + +class String + # for older ruby + alias bytesize size unless method_defined?(:bytesize) +end + +class Dir + def self.mktmpdir(path) + path = File.join(tmpdir, path+"-#{$$}-#{rand(100000)}") + begin + mkdir(path) + rescue Errno::EEXIST + path.succ! + retry + end + path + end unless respond_to?(:mktmpdir) +end + +$packages &&= $packages.split(/[, ]+/).tap {|pkg| + if all = pkg.index("all") + pkg[all, 1] = PACKAGES.keys - pkg + end + pkg -= PACKAGES.keys + pkg.empty? or abort "#{File.basename $0}: unknown packages - #{pkg.join(", ")}" +} +$packages ||= PACKAGES.keys + +$digests &&= $digests.split(/[, ]+/).tap {|dig| + dig -= DIGESTS + dig.empty? or abort "#{File.basename $0}: unknown digests - #{dig.join(", ")}" +} +$digests ||= DIGESTS + +$patch_file &&= File.expand_path($patch_file) +path = ENV["PATH"].split(File::PATH_SEPARATOR) +%w[YACC BASERUBY RUBY MV MINIRUBY].each do |var| + cmd, = ENV[var].shellsplit + unless path.any? {|dir| + file = File.expand_path(cmd, dir) + File.file?(file) and File.executable?(file) + } + abort "#{File.basename $0}: #{var} command not found - #{cmd}" + end +end + +%w[BASERUBY RUBY MINIRUBY].each do |var| + `#{ENV[var]} --disable-gem -e1 2>&1` + if $?.success? + ENV[var] += ' --disable-gem' + end +end + +if defined?($help) or defined?($_help) + puts usage + exit +end +unless destdir = ARGV.shift + abort usage +end +revisions = ARGV.empty? ? ["trunk"] : ARGV +unless tmp = $exported + FileUtils.mkpath(destdir) + destdir = File.expand_path(destdir) + tmp = Dir.mktmpdir("ruby-snapshot") + FileUtils.mkpath(tmp) + at_exit { + Dir.chdir "/" + FileUtils.rm_rf(tmp) + } unless $keep_temp +end + +def package(vcs, rev, destdir, tmp = nil) + patchlevel = false + prerelease = false + if revision = rev[/@(\d+)\z/, 1] + rev = $` + end + case rev + when /\Atrunk\z/ + url = vcs.trunk + when /\Abranches\// + url = vcs.branch($') + when /\Atags\// + url = vcs.tag($') + when /\Astable\z/ + vcs.branch_list("ruby_[0-9]*") {|n| url = n[/\Aruby_\d+_\d+\z/]} + url &&= vcs.branch(url) + when /\A(.*)\.(.*)\.(.*)-(preview|rc)(\d+)/ + prerelease = true + tag = "#{$4}#{$5}" + url = vcs.tag("v#{$1}_#{$2}_#{$3}_#{$4}#{$5}") + when /\A(.*)\.(.*)\.(.*)-p(\d+)/ + patchlevel = true + tag = "p#{$4}" + url = vcs.tag("v#{$1}_#{$2}_#{$3}_#{$4}") + when /\A(\d+)\.(\d+)(?:\.(\d+))?\z/ + if $3 && ($1 > "2" || $1 == "2" && $2 >= "1") + patchlevel = true + tag = "" + url = vcs.tag("v#{$1}_#{$2}_#{$3}") + else + url = vcs.branch("ruby_#{rev.tr('.', '_')}") + end + else + warn "#{$0}: unknown version - #{rev}" + return + end + revision ||= vcs.get_revisions(url)[1] + version = nil + unless revision + url = vcs.trunk + vcs.grep(RUBY_VERSION_PATTERN, url, "version.h") {version = $1} + unless rev == version + warn "#{$0}: #{rev} not found" + return + end + revision = vcs.get_revisions(url)[1] + end + v = nil + if $exported + if String === $exported + v = $exported + end + else + v = "ruby" + puts "Exporting #{rev}@#{revision}" + exported = tmp ? File.join(tmp, v) : v + unless vcs.export(revision, url, exported, true) {|line| print line} + warn("Export failed") + return + end + if $srcdir + Dir.glob($srcdir + "/{tool/config.{guess,sub},gems/*.gem,.downloaded-cache/*,enc/unicode/data/**/*.txt}") do |file| + puts "copying #{file}" + dest = exported + file[$srcdir.size..-1] + FileUtils.mkpath(File.dirname(dest)) + begin + FileUtils.ln(file, dest, force: true) + next unless File.symlink?(dest) + File.unlink(dest) + rescue SystemCallError + end + begin + FileUtils.cp_r(file, dest, preserve: true) + rescue SystemCallError + end + end + end + end + + status = IO.read(File.dirname(__FILE__) + "/prereq.status") + Dir.chdir(tmp) if tmp + + if !File.directory?(v) + v = Dir.glob("ruby-*").select(&File.method(:directory?)) + v.size == 1 or abort "#{File.basename $0}: not exported" + v = v[0] + end + + unless File.exist?("#{v}/ChangeLog") + # get last revision from previous ChangeLog archive + last_ChangeLog = Dir["#{v}/doc/ChangeLog-*"].grep(/-(\d+)\z/) {|n| [$1.to_i, n]}.max[1] + open(last_ChangeLog) do |f| + f.readline + unless /\Ar(\d+) / =~ f.readline + abort "#{File.basename $0}: Cannot find revision from '#{last_ChangeLog}'" + end + vcs.export_changelog(url, $1.to_i, revision.to_i, "#{v}/ChangeLog") + end + end + + open("#{v}/revision.h", "wb") {|f| f.puts "#define RUBY_REVISION #{revision}"} + version ||= (versionhdr = IO.read("#{v}/version.h"))[RUBY_VERSION_PATTERN, 1] + version or return + if patchlevel + unless tag.empty? + versionhdr ||= IO.read("#{v}/version.h") + patchlevel = versionhdr[/^\#define\s+RUBY_PATCHLEVEL\s+(\d+)/, 1] + tag = (patchlevel ? "p#{patchlevel}" : "r#{revision}") + end + elsif prerelease + versionhdr ||= IO.read("#{v}/version.h") + versionhdr.sub!(/^\#define\s+RUBY_PATCHLEVEL_STR\s+"\K.+?(?=")/, tag) + IO.write("#{v}/version.h", versionhdr) + else + tag ||= "r#{revision}" + end + unless v == $exported + if $archname + n = $archname + elsif tag.empty? + n = "ruby-#{version}" + else + n = "ruby-#{version}-#{tag}" + end + File.directory?(n) or File.rename v, n + v = n + end + system(*%W"patch -d #{v} -p0 -i #{$patch_file}") if $patch_file + if !$exported or $patch_file + colors = %w[red yellow green cyan blue magenta] + "take a breath, and go ahead".scan(/./) do |c| + if c == ' ' + print c + else + colors.push(color = colors.shift) + print $colorize.decorate(c, color) + end + sleep(c == "," ? 0.7 : 0.05) + end + puts + end + def (clean = []).add(n) push(n); n end + Dir.chdir(v) do + File.open(clean.add("cross.rb"), "w") do |f| + f.puts "Object.__send__(:remove_const, :CROSS_COMPILING) if defined?(CROSS_COMPILING)" + f.puts "CROSS_COMPILING=true" + f.puts "Object.__send__(:remove_const, :RUBY_PLATFORM)" + f.puts "RUBY_PLATFORM='none'" + f.puts "Object.__send__(:remove_const, :RUBY_VERSION)" + f.puts "RUBY_VERSION='#{version}'" + end + unless File.exist?("configure") + print "creating configure..." + unless system([ENV["AUTOCONF"]]*2) + puts $colorize.fail(" failed") + return + end + puts $colorize.pass(" done") + end + clean.add("autom4te.cache") + clean.add("enc/unicode/data") + print "creating prerequisites..." + if File.file?("common.mk") && /^prereq/ =~ commonmk = IO.read("common.mk") + puts + extout = clean.add('tmp') + begin + status = IO.read("tool/prereq.status") + rescue Errno::ENOENT + # use fallback file + end + File.open(clean.add("config.status"), "w") {|f| + f.print status + } + FileUtils.mkpath(hdrdir = "#{extout}/include/ruby") + File.open("#{hdrdir}/config.h", "w") {} + FileUtils.mkpath(defaults = "#{extout}/rubygems/defaults") + File.open("#{defaults}/operating_system.rb", "w") {} + File.open("#{defaults}/ruby.rb", "w") {} + miniruby = ENV['MINIRUBY'] + " -I. -I#{extout} -rcross" + baseruby = ENV["BASERUBY"] + mk = IO.read("Makefile.in").gsub(/^@.*\n/, '') + vars = { + "EXTOUT"=>extout, + "PATH_SEPARATOR"=>File::PATH_SEPARATOR, + "MINIRUBY"=>miniruby, + "RUBY"=>ENV["RUBY"], + "BASERUBY"=>baseruby, + "PWD"=>Dir.pwd, + "ruby_version"=>version, + } + status.scan(/^s([%,])@([A-Za-z_][A-Za-z_0-9]*)@\1(.*?)\1g$/) do + vars[$2] ||= $3 + end + vars["UNICODE_VERSION"] = $unicode_version if $unicode_version + args = vars.dup + mk.gsub!(/@([A-Za-z_]\w*)@/) {args.delete($1); vars[$1] || ENV[$1]} + mk << commonmk.gsub(/\{\$([^(){}]*)[^{}]*\}/, "") + mk << <<-'APPEND' + +prereq: clean-cache $(CLEAN_CACHE) +clean-cache $(CLEAN_CACHE): after-update +touch-unicode-files: _touch-unicode-files +update-download:: touch-unicode-files +update-download:: update-gems +after-update:: extract-gems +extract-gems: update-gems +update-gems: +_touch-unicode-files: + $(MAKEDIRS) $(UNICODE_SRC_DATA_DIR) + touch $(UNICODE_SRC_DATA_DIR)/.unicode-tables.time $(UNICODE_DATA_HEADERS) + APPEND + open(clean.add("Makefile"), "w") do |f| + f.puts "prereq: update-download" + f.puts mk + end + ENV["CACHE_SAVE"] = "no" + system(ENV["MAKE"] || ENV["make"] || "make", "prereq", *args.map {|arg| arg.join("=")}) + clean.push("rbconfig.rb", ".rbconfig.time", "enc.mk") + print "prerequisites" + else + system(*%W"#{YACC} -o parse.c parse.y") + end + vcs.after_export(".") if exported + FileUtils.rm_rf(clean) unless $keep_temp + FileUtils.rm_rf(".downloaded-cache") + if File.exist?("gems/bundled_gems") + gems = Dir.glob("gems/*.gem") + gems -= File.readlines("gems/bundled_gems").map {|line| + name, version, _ = line.split(' ') + "gems/#{name}-#{version}.gem" + } + FileUtils.rm_f(gems) + else + FileUtils.rm_rf("gems") + end + unless $?.success? + puts $colorize.fail(" failed") + return + end + puts $colorize.pass(" done") + end + + if v == "." + v = File.basename(Dir.pwd) + Dir.chdir ".." + else + Dir.chdir(File.dirname(v)) + v = File.basename(v) + end + + tarball = nil + return $packages.collect do |mesg| + (ext, *cmd) = PACKAGES[mesg] + File.directory?(destdir) or FileUtils.mkpath(destdir) + file = File.join(destdir, "#{$archname||v}#{ext}") + case ext + when /\.tar/ + if tarball + next if tarball.empty? + else + tarball = "#{$archname||v}.tar" + print "creating tarball... #{tarball}" + if system("tar", "cf", tarball, v) + puts $colorize.pass(" done") + else + puts $colorize.fail(" failed") + tarball = "" + next + end + end + print "creating #{mesg} tarball... #{file}" + done = system(*cmd, tarball, out: file) + else + print "creating #{mesg} archive... #{file}" + if Hash === cmd.last + *cmd, opt = *cmd + cmd << file << v << opt + else + (cmd = cmd.dup) << file << v + end + done = system(*cmd) + end + if done + puts $colorize.pass(" done") + file + else + puts $colorize.fail(" failed") + nil + end + end.compact +ensure + FileUtils.rm_rf(v) if v and !$exported and !$keep_temp +end + +if [$srcdir, ($svn||=nil), ($git||=nil)].compact.size > 1 + abort "#{File.basename $0}: -srcdir, -svn, and -git are exclusive" +end +if $srcdir + vcs = VCS.detect($srcdir) +elsif $svn + vcs = VCS::SVN.new($svn == true ? SVNURL : URI.parse($svn)) +elsif $git + vcs = VCS::GIT.new($git == true ? GITURL : URI.parse($git)) +else + vcs = VCS::SVN.new(SVNURL) +end + +success = true +revisions.collect {|rev| package(vcs, rev, destdir, tmp)}.flatten.each do |name| + if !name + success = false + next + end + str = open(name, "rb") {|f| f.read} + puts "* #{$colorize.pass(name)}" + puts " SIZE: #{str.bytesize} bytes" + $digests.each do |alg| + printf " %-8s%s\n", "#{alg}:", Digest.const_get(alg).hexdigest(str) + end +end + +exit false if !success + +# vim:fileencoding=US-ASCII sw=2 ts=4 noexpandtab ff=unix diff --git a/tool/make_hgraph.rb b/tool/make_hgraph.rb new file mode 100644 index 0000000000..0f388814dd --- /dev/null +++ b/tool/make_hgraph.rb @@ -0,0 +1,95 @@ +# +# Make dot file of internal class/module hierarchy graph. +# + +require 'objspace' + +module ObjectSpace + def self.object_id_of obj + if obj.kind_of?(ObjectSpace::InternalObjectWrapper) + obj.internal_object_id + else + obj.object_id + end + end + + T_ICLASS_NAME = {} + + def self.class_name_of klass + case klass + when Class, Module + # (singleton class).name returns nil + klass.name || klass.inspect + when InternalObjectWrapper # T_ICLASS + if klass.type == :T_ICLASS + "#<I:#{class_name_of(ObjectSpace.internal_class_of(klass))}>" + else + klass.inspect + end + else + klass.inspect + end + end + + def self.module_refenreces klass + h = {} # object_id -> [klass, class_of, super] + stack = [klass] + while klass = stack.pop + obj_id = ObjectSpace.object_id_of(klass) + next if h.has_key?(obj_id) + cls = ObjectSpace.internal_class_of(klass) + sup = ObjectSpace.internal_super_of(klass) + stack << cls if cls + stack << sup if sup + h[obj_id] = [klass, cls, sup].map{|e| ObjectSpace.class_name_of(e)} + end + h.values + end + + def self.module_refenreces_dot klass + result = [] + rank_set = {} + + result << "digraph mod_h {" + # result << " rankdir=LR;" + module_refenreces(klass).each{|(m, k, s)| + # next if /singleton/ =~ m + result << "#{m.dump} -> #{s.dump} [label=\"super\"];" + result << "#{m.dump} -> #{k.dump} [label=\"klass\"];" + + unless rank = rank_set[m] + rank = rank_set[m] = 0 + end + unless rank_set[s] + rank_set[s] = rank + 1 + end + unless rank_set[k] + rank_set[k] = rank + end + } + + rs = [] # [[mods...], ...] + rank_set.each{|m, r| + rs[r] = [] unless rs[r] + rs[r] << m + } + + rs.each{|ms| + result << "{rank = same; #{ms.map{|m| m.dump}.join(", ")}};" + } + result << "}" + result.join("\n") + end + + def self.module_refenreces_image klass, file + dot = module_refenreces_dot(klass) + img = nil + IO.popen("dot -Tpng", 'r+'){|io| + # + io.puts dot + io.close_write + img = io.read + } + open(File.expand_path(file), 'w+'){|f| f.puts img} + end +end diff --git a/tool/mdoc2man.rb b/tool/mdoc2man.rb new file mode 100755 index 0000000000..e005fcf19a --- /dev/null +++ b/tool/mdoc2man.rb @@ -0,0 +1,505 @@ +#!/usr/bin/env ruby +### +### mdoc2man - mdoc to man converter +### +### Quick usage: mdoc2man.rb < mdoc_manpage.8 > man_manpage.8 +### +### Ported from Perl by Akinori MUSHA. +### +### Copyright (c) 2001 University of Illinois Board of Trustees +### Copyright (c) 2001 Mark D. Roth +### Copyright (c) 2002, 2003 Akinori MUSHA +### All rights reserved. +### +### Redistribution and use in source and binary forms, with or without +### modification, are permitted provided that the following conditions +### are met: +### 1. Redistributions of source code must retain the above copyright +### notice, this list of conditions and the following disclaimer. +### 2. Redistributions in binary form must reproduce the above copyright +### notice, this list of conditions and the following disclaimer in the +### documentation and/or other materials provided with the distribution. +### 3. All advertising materials mentioning features or use of this software +### must display the following acknowledgement: +### This product includes software developed by the University of +### Illinois at Urbana, and their contributors. +### 4. The University nor the names of their +### contributors may be used to endorse or promote products derived from +### this software without specific prior written permission. +### +### THIS SOFTWARE IS PROVIDED BY THE TRUSTEES AND CONTRIBUTORS ``AS IS'' AND +### ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +### IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +### ARE DISCLAIMED. IN NO EVENT SHALL THE TRUSTEES OR CONTRIBUTORS BE LIABLE +### FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +### DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +### OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +### HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +### LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +### OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +### SUCH DAMAGE. +### +### $Id$ +### + +class Mdoc2Man + ANGLE = 1 + OPTION = 2 + PAREN = 3 + + RE_PUNCT = /^[!"'),\.\/:;>\?\]`]$/ + + def initialize + @name = @date = @id = nil + @refauthors = @reftitle = @refissue = @refdate = @refopt = nil + + @optlist = 0 ### 1 = bullet, 2 = enum, 3 = tag, 4 = item + @oldoptlist = 0 + @nospace = 0 ### 0, 1, 2 + @enum = 0 + @synopsis = true + @reference = false + @ext = false + @extopt = false + @literal = false + end + + def mdoc2man(i, o) + i.each { |line| + if /^\./ !~ line + o.print line + o.print ".br\n" if @literal + next + end + + line.slice!(0, 1) + + next if /\\"/ =~ line + + line = parse_macro(line) and o.print line + } + + initialize + end + + def shift_arg(words) + case words[0] + when nil, RE_PUNCT + nil + when /\A"(.+)/ + words.shift + word = $1 + loop { + break if word.chomp!('"') + token = words.shift or break + word << ' ' << token + } + word + else + words.shift + end + end + + def parse_macro(line) + words = line.split + retval = '' + + quote = [] + dl = false + + while word = words.shift + case word + when RE_PUNCT + next retval << word if word == ':' + while q = quote.pop + case q + when OPTION + retval << ']' + when PAREN + retval << ')' + when ANGLE + retval << '>' + end + end + retval << word + next + when 'Li', 'Pf' + @nospace = 1 + next + when 'Xo' + @ext = true + retval << ' ' unless retval.empty? || /[\n ]\z/ =~ retval + next + when 'Xc' + @ext = false + retval << "\n" unless @extopt + break + when 'Bd' + @literal = true if words[0] == '-literal' + retval << "\n" + break + when 'Ed' + @literal = false + break + when 'Ns' + @nospace = 1 if @nospace == 0 + retval.chomp!(' ') + next + when 'No' + retval.chomp!(' ') + retval << words.shift + next + when 'Dq' + retval << '``' + begin + retval << words.shift << ' ' + end until words.empty? || RE_PUNCT =~ words[0] + retval.chomp!(' ') + retval << '\'\'' + @nospace = 1 if @nospace == 0 && RE_PUNCT =~ words[0] + next + when 'Sq', 'Ql' + retval << '`' << words.shift << '\'' + @nospace = 1 if @nospace == 0 && RE_PUNCT =~ words[0] + next + # when 'Ic' + # retval << '\\fB' << words.shift << '\\fP' + # next + when 'Oo' + #retval << "[\\c\n" + @extopt = true + @nospace = 1 if @nospace == 0 + retval << '[' + next + when 'Oc' + @extopt = false + retval << ']' + next + when 'Ao' + @nospace = 1 if @nospace == 0 + retval << '<' + next + when 'Ac' + retval << '>' + next + end + + retval << ' ' if @nospace == 0 && !(retval.empty? || /[\n ]\z/ =~ retval) + @nospace = 0 if @nospace == 1 + + case word + when 'Dd' + @date = words.join(' ') + return nil + when 'Dt' + if words.size >= 2 && words[1] == '""' && + /^(.*)\(([0-9])\)$/ =~ words[0] + words[0] = $1 + words[1] = $2 + end + @id = words.join(' ') + return nil + when 'Os' + retval << '.TH ' << @id << ' "' << @date << '" "' << + words.join(' ') << '"' + break + when 'Sh' + retval << '.SH' + @synopsis = (words[0] == 'SYNOPSIS') + next + when 'Xr' + retval << '\\fB' << words.shift << + '\\fP(' << words.shift << ')' << (words.shift||'') + break + when 'Rs' + @refauthors = [] + @reftitle = '' + @refissue = '' + @refdate = '' + @refopt = '' + @reference = true + break + when 'Re' + retval << "\n" + + # authors + while @refauthors.size > 1 + retval << @refauthors.shift << ', ' + end + retval << 'and ' unless retval.empty? + retval << @refauthors.shift + + # title + retval << ', \\fI' << @reftitle << '\\fP' + + # issue + retval << ', ' << @refissue unless @refissue.empty? + + # date + retval << ', ' << @refdate unless @refdate.empty? + + # optional info + retval << ', ' << @refopt unless @refopt.empty? + + retval << ".\n" + + @reference = false + break + when 'An' + next + when 'Dl' + retval << ".nf\n" << '\\& ' + dl = true + next + when 'Ux' + retval << "UNIX" + next + when 'Bro' + retval << '{' + @nospace = 1 if @nospace == 0 + next + when 'Brc' + retval.sub!(/ *\z/, '}') + next + end + + if @reference + case word + when '%A' + @refauthors.unshift(words.join(' ')) + break + when '%T' + @reftitle = words.join(' ') + @reftitle.sub!(/^"/, '') + @reftitle.sub!(/"$/, '') + break + when '%N' + @refissue = words.join(' ') + break + when '%D' + @refdate = words.join(' ') + break + when '%O' + @refopt = words.join(' ') + break + end + end + + case word + when 'Nm' + name = words.empty? ? @name : words.shift + @name ||= name + retval << ".br\n" if @synopsis + retval << "\\fB" << name << "\\fP" + @nospace = 1 if @nospace == 0 && RE_PUNCT =~ words[0] + next + when 'Nd' + retval << '\\-' + next + when 'Fl' + retval << '\\fB\\-' << words.shift << '\\fP' + @nospace = 1 if @nospace == 0 && RE_PUNCT =~ words[0] + next + when 'Ar' + retval << '\\fI' + if words.empty? + retval << 'file ...\\fP' + else + retval << words.shift << '\\fP' + while words[0] == '|' + retval << ' ' << words.shift << ' \\fI' << words.shift << '\\fP' + end + @nospace = 1 if @nospace == 0 && RE_PUNCT =~ words[0] + next + end + when 'Cm' + retval << '\\fB' << words.shift << '\\fP' + while RE_PUNCT =~ words[0] + retval << words.shift + end + next + when 'Op' + quote << OPTION + @nospace = 1 if @nospace == 0 + retval << '[' + # words.push(words.pop + ']') + next + when 'Aq' + quote << ANGLE + @nospace = 1 if @nospace == 0 + retval << '<' + # words.push(words.pop + '>') + next + when 'Pp' + retval << "\n" + next + when 'Ss' + retval << '.SS' + next + end + + case word + when 'Pa' + if !quote.include?(OPTION) + retval << '\\fI' + retval << '\\&' if /^\./ =~ words[0] + retval << words.shift << '\\fP' + while RE_PUNCT =~ words[0] + retval << words.shift + end + # @nospace = 1 if @nospace == 0 && RE_PUNCT =~ words[0] + next + end + when 'Lk' + if !quote.include?(OPTION) + url = words.shift + if name = shift_arg(words) + retval << '\\fI' << name << ':\\fP ' + end + retval << '\\fB' + retval << '\\&' if /\A\./ =~ url + retval << url << '\\fP' + next + end + end + + case word + when 'Dv' + retval << '.BR' + next + when 'Em', 'Ev' + retval << '.IR' + next + when 'Pq' + retval << '(' + @nospace = 1 + quote << PAREN + next + when 'Sx', 'Sy' + retval << '.B ' << words.join(' ') + break + when 'Ic' + retval << '\\fB' + until words.empty? || RE_PUNCT =~ words[0] + case words[0] + when 'Op' + words.shift + retval << '[' + words.push(words.pop + ']') + next + when 'Aq' + words.shift + retval << '<' + words.push(words.pop + '>') + next + when 'Ar' + words.shift + retval << '\\fI' << words.shift << '\\fP' + else + retval << words.shift + end + + retval << ' ' if @nospace == 0 + end + + retval.chomp!(' ') + retval << '\\fP' + retval << words.shift unless words.empty? + break + when 'Bl' + @oldoptlist = @optlist + + case words[0] + when '-bullet' + @optlist = 1 + when '-enum' + @optlist = 2 + @enum = 0 + when '-tag' + @optlist = 3 + when '-item' + @optlist = 4 + end + + break + when 'El' + @optlist = @oldoptlist + next + end + + if @optlist != 0 && word == 'It' + case @optlist + when 1 + # bullets + retval << '.IP \\(bu' + when 2 + # enum + @enum += 1 + retval << '.IP ' << @enum << '.' + when 3 + # tags + retval << ".TP\n" + case words[0] + when 'Pa', 'Ev', 'Lk' + words.shift + retval << '.B' + end + when 4 + # item + retval << ".IP\n" + end + + next + end + + case word + when 'Sm' + case words[0] + when 'off' + @nospace = 2 + when 'on' + # retval << "\n" + @nospace = 0 + end + words.shift + next + end + + retval << word + end + + return nil if retval == '.' + + retval.sub!(/\A\.([^a-zA-Z])/, "\\1") + # retval.chomp!(' ') + + while q = quote.pop + case q + when OPTION + retval << ']' + when PAREN + retval << ')' + when ANGLE + retval << '>' + end + end + + # retval << ' ' unless @nospace == 0 || retval.empty? || /\n\z/ =~ retval + + retval << ' ' unless !@ext || @extopt || / $/ =~ retval + + retval << "\n" unless @ext || @extopt || retval.empty? || /\n\z/ =~ retval + + retval << ".fi\n" if dl + + return retval + end + + def self.mdoc2man(i, o) + new.mdoc2man(i, o) + end +end + +if $0 == __FILE__ + Mdoc2Man.mdoc2man(ARGF, STDOUT) +end diff --git a/tool/merger.rb b/tool/merger.rb new file mode 100755 index 0000000000..f7630cd02b --- /dev/null +++ b/tool/merger.rb @@ -0,0 +1,312 @@ +#!/bin/sh +# -*- ruby -*- +exec "${RUBY-ruby}" "-x" "$0" "$@" && [ ] if false +#!ruby +# This needs ruby 1.9 and subversion. +# As a Ruby committer, run this in an SVN repository +# to commit a change. + +require 'fileutils' +require 'tempfile' + +$repos = 'svn+ssh://svn@ci.ruby-lang.org/ruby/' +ENV['LC_ALL'] = 'C' + +def help + puts <<-end +\e[1msimple backport\e[0m + ruby #$0 1234 + +\e[1mrange backport\e[0m + ruby #$0 1234:5678 + +\e[1mbackport from other branch\e[0m + ruby #$0 17502 mvm + +\e[1mrevision increment\e[0m + ruby #$0 revisionup + +\e[1mteeny increment\e[0m + ruby #$0 teenyup + +\e[1mtagging major release\e[0m + ruby #$0 tag 2.2.0 + +\e[1mtagging patch release\e[0m (about 2.1.0 or later, it means X.Y.Z (Z > 0) release) + ruby #$0 tag + +\e[1mtagging preview/RC\e[0m + ruby #$0 tag 2.2.0-preview1 + +\e[1mremove tag\e[0m + ruby #$0 removetag 2.2.9 + +\e[33;1m* all operations shall be applied to the working directory.\e[0m +end +end + +# Prints the version of Ruby found in version.h + +def version + v = p = nil + open 'version.h', 'rb' do |f| + f.each_line do |l| + case l + when /^#define RUBY_VERSION "(\d+)\.(\d+)\.(\d+)"$/ + v = $~.captures + when /^#define RUBY_PATCHLEVEL (-?\d+)$/ + p = $1 + end + end + end + return v, p +end + +def interactive str, editfile = nil + loop do + yield + STDERR.puts "\e[1;33m#{str} ([y]es|[a]bort|[r]etry#{'|[e]dit' if editfile})\e[0m" + case STDIN.gets + when /\Aa/i then exit + when /\Ar/i then redo + when /\Ay/i then break + when /\Ae/i then system(ENV["EDITOR"], editfile) + else exit + end + end +end + +def version_up(inc=nil) + d = Time.now + d = d.localtime(9*60*60) # server is Japan Standard Time +09:00 + system(*%w'svn revert version.h') + v, pl = version + + if inc == :teeny + v[2].succ! + end + # patchlevel + if pl != "-1" + pl.succ! + end + + str = open 'version.h', 'rb' do |f| f.read end + [%W[RUBY_VERSION "#{v.join '.'}"], + %W[RUBY_VERSION_CODE #{v.join ''}], + %W[RUBY_VERSION_MAJOR #{v[0]}], + %W[RUBY_VERSION_MINOR #{v[1]}], + %W[RUBY_VERSION_TEENY #{v[2]}], + %W[RUBY_RELEASE_DATE "#{d.strftime '%Y-%m-%d'}"], + %W[RUBY_RELEASE_CODE #{d.strftime '%Y%m%d'}], + %W[RUBY_PATCHLEVEL #{pl}], + %W[RUBY_RELEASE_YEAR #{d.year}], + %W[RUBY_RELEASE_MONTH #{d.month}], + %W[RUBY_RELEASE_DAY #{d.day}], + ].each do |(k, i)| + str.sub!(/^(#define\s+#{k}\s+).*$/, "\\1#{i}") + end + str.sub!(/\s+\z/m, '') + fn = sprintf 'version.h.tmp.%032b', rand(1 << 31) + File.rename 'version.h', fn + open 'version.h', 'wb' do |f| + f.puts str + end + File.unlink fn +end + +def tag intv_p = false, relname=nil + # relname: + # * 2.2.0-preview1 + # * 2.2.0-rc1 + # * 2.2.0 + v, pl = version + x = v.join('_') + if relname + abort "patchlevel is not -1 but '#{pl}' for preview or rc" if pl != '-1' && /-(?:preview|rc)/ =~ relname + abort "patchlevel is not 0 but '#{pl}' for the first release" if pl != '0' && /-(?:preview|rc)/ !~ relname + pl = relname[/-(.*)\z/, 1] + curver = v.join('.') + (pl ? '-' + pl : '') + if relname != curver + abort "given relname '#{relname}' conflicts current version '#{curver}'" + end + branch_url = `svn info`[/URL: (.*)/, 1] + else + if pl == '-1' + abort "no relname is given and not in a release branch even if this is patch release" + end + branch_url = $repos + 'branches/ruby_' + if v[0] < "2" || (v[0] == "2" && v[1] < "1") + abort "patchlevel must be greater than 0 for patch release" if pl == "0" + branch_url << x + else + abort "teeny must be greater than 0 for patch release" if v[2] == "0" + branch_url << x.sub(/_\d+$/, '') + end + end + tagname = 'v' + x + (v[0] < "2" || (v[0] == "2" && v[1] < "1") || /^(?:preview|rc)/ =~ pl ? '_' + pl : '') + tag_url = $repos + 'tags/' + tagname + system(*%w'svn info', tag_url, out: IO::NULL, err: IO::NULL) + if $?.success? + abort "specfied tag already exists. check tag name and remove it if you want to force re-tagging" + end + if intv_p + interactive "OK? svn cp -m \"add tag #{tagname}\" #{branch_url} #{tag_url}" do + # nothing to do here + end + end + system(*%w'svn cp -m', "add tag #{tagname}", branch_url, tag_url) + puts "run following command in git-svn working directory to push the tag into GitHub:" + puts "git tag #{tagname} origin/tags/#{tagname} && git push ruby #{tagname}" +end + +def remove_tag intv_p = false, relname + # relname: + # * 2.2.0-preview1 + # * 2.2.0-rc1 + # * 2.2.0 + # * v2_2_0_preview1 + # * v2_2_0_rc1 + # * v2_2_0 + if !relname && !intv_p.is_a?(String) + raise ArgumentError, "relname is not specified" + end + intv_p, relname = false, intv_p if !relname && intv_p.is_a?(String) + + if /^v/ !~ relname + tagname = 'v' + relname.tr(".-", "_") + else + tagname = relname + end + tag_url = $repos + 'tags/' + tagname + if intv_p + interactive "OK? svn rm -m \"remove tag #{tagname}\" #{tag_url}" do + # nothing to do here + end + end + system(*%w'svn rm -m', "remove tag #{tagname}", tag_url) +end + +def default_merge_branch + %r{^URL: .*/branches/ruby_1_8_} =~ `svn info` ? 'branches/ruby_1_8' : 'trunk' +end + +case ARGV[0] +when "teenyup" + version_up(:teeny) + system 'svn diff version.h' +when "up", /\A(ver|version|rev|revision|lv|level|patch\s*level)\s*up/ + version_up + system 'svn diff version.h' +when "tag" + tag :interactive, ARGV[1] +when /\A(?:remove|rm|del)_?tag\z/ + remove_tag :interactive, ARGV[1] +when nil, "-h", "--help" + help + exit +else + system 'svn up' + system 'ruby tool/file2lastrev.rb --revision.h . > revision.tmp' + system 'tool/ifchange "--timestamp=.revision.time" "revision.h" "revision.tmp"' + FileUtils.rm_f('revision.tmp') + + case ARGV[0] + when /--ticket=(.*)/ + tickets = $1.split(/,/).map{|num| " [Backport ##{num}]"}.join + ARGV.shift + when /merge revision\(s\) ([\d,\-]+):( \[.*)/ + tickets = $2 + ARGV[0] = $1 + else + tickets = '' + end + + q = $repos + (ARGV[1] || default_merge_branch) + revstr = ARGV[0].delete('^, :\-0-9') + revs = revstr.split(/[,\s]+/) + log = '' + log_svn = '' + + revs.each do |rev| + case rev + when /\A\d+:\d+\z/ + r = ['-r', rev] + when /\A(\d+)-(\d+)\z/ + rev = "#{$1.to_i-1}:#$2" + r = ['-r', rev] + when /\A\d+\z/ + r = ['-c', rev] + when nil then + puts "#$0 revision" + exit + else + puts "invalid revision part '#{rev}' in '#{ARGV[0]}'" + exit + end + + l = IO.popen %w'svn diff' + r + %w'--diff-cmd=diff -x -pU0' + [File.join(q, 'ChangeLog')] do |f| + f.read + end + + log << l + l = l.lines.grep(/^\+\t/).join.gsub(/^\+/, '').gsub(/^\t\*/, "\n\t\*") + + if l.empty? + l = IO.popen %w'svn log ' + r + [q] do |f| + f.read + end.sub(/\A-+\nr.*/, '').sub(/\n-+\n\z/, '').gsub(/^./, "\t\\&") + end + log_svn << l + + a = %w'svn merge --accept=postpone' + r + [q] + STDERR.puts a.join(' ') + + system(*a) + system(*%w'svn revert ChangeLog') if /^\+/ =~ l + end + + if `svn diff --diff-cmd=diff -x -upw`.empty? + interactive 'Only ChangeLog is modified, right?' do + end + end + + if /^\+/ =~ log + system(*%w'svn revert ChangeLog') + IO.popen %w'patch -p0', 'wb' do |f| + f.write log.gsub(/\+(Mon|Tue|Wed|Thu|Fri|Sat|Sun) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) [ 123][0-9] [012][0-9]:[0-5][0-9]:[0-5][0-9] \d\d\d\d/, + # this format-time-string was from the file local variables of ChangeLog + '+'+Time.now.strftime('%a %b %e %H:%M:%S %Y')) + end + system(*%w'touch ChangeLog') # needed somehow, don't know why... + else + STDERR.puts '*** You should write ChangeLog NOW!!! ***' + end + + version_up + f = Tempfile.new 'merger.rb' + f.printf "merge revision(s) %s:%s", revstr, tickets + f.write log_svn + f.flush + f.close + + interactive 'conflicts resolved?', f.path do + IO.popen(ENV["PAGER"] || "less", "w") do |g| + g << `svn stat` + g << "\n\n" + f.open + g << f.read + f.close + g << "\n\n" + g << `svn diff --diff-cmd=diff -x -upw` + end + end + + if system(*%w'svn ci -F', f.path) + # tag :interactive # no longer needed. + system 'rm -f subversion.commitlog' + else + puts 'commit failed; try again.' + end + + f.close(true) +end diff --git a/tool/mk_call_iseq_optimized.rb b/tool/mk_call_iseq_optimized.rb new file mode 100644 index 0000000000..9b88c5eeb6 --- /dev/null +++ b/tool/mk_call_iseq_optimized.rb @@ -0,0 +1,73 @@ + +puts <<EOS +/* -*- c -*- */ +#if 1 /* enable or disable this optimization */ + +/* DO NOT EDIT THIS FILE DIRECTLY + * + * This file is generated by tool/mk_call_iseq_optimized.rb + */ + +EOS + +P = (0..3) +L = (0..5) + +def fname param, local + "vm_call_iseq_setup_normal_0start_#{param}params_#{local}locals" +end + +P.each{|param| + L.each{|local| + puts <<EOS +static VALUE +#{fname(param, local)}(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, const struct rb_call_info *ci, struct rb_call_cache *cc) +{ + return vm_call_iseq_setup_normal(ec, cfp, calling, ci, cc, 0, #{param}, #{local}); +} + +EOS + # + } +} + +puts <<EOS +/* vm_call_iseq_handlers[param][local] */ +static const vm_call_handler vm_call_iseq_handlers[][#{L.to_a.size}] = { +#{P.map{|param| '{' + L.map{|local| fname(param, local)}.join(",\n ") + '}'}.join(",\n")} +}; + +static inline vm_call_handler +vm_call_iseq_setup_func(const struct rb_call_info *ci, const int param_size, const int local_size) +{ + if (UNLIKELY(ci->flag & VM_CALL_TAILCALL)) { + return &vm_call_iseq_setup_tailcall_0start; + } + else if (0) { /* to disable optimize */ + return &vm_call_iseq_setup_normal_0start; + } + else { + if (param_size <= #{P.end} && + local_size <= #{L.end}) { + VM_ASSERT(local_size >= 0); + return vm_call_iseq_handlers[param_size][local_size]; + } + return &vm_call_iseq_setup_normal_0start; + } +} + +#else + + +static inline vm_call_handler +vm_call_iseq_setup_func(const struct rb_call_info *ci, struct rb_call_cache *cc) +{ + if (UNLIKELY(ci->flag & VM_CALL_TAILCALL)) { + return &vm_call_iseq_setup_tailcall_0start; + } + else { + return &vm_call_iseq_setup_normal_0start; + } +} +#endif +EOS diff --git a/tool/mkconfig.rb b/tool/mkconfig.rb new file mode 100755 index 0000000000..52ec6a85d7 --- /dev/null +++ b/tool/mkconfig.rb @@ -0,0 +1,325 @@ +#!./miniruby -s + +# This script, which is run when ruby is built, generates rbconfig.rb by +# parsing information from config.status. rbconfig.rb contains build +# information for ruby (compiler flags, paths, etc.) and is used e.g. by +# mkmf to build compatible native extensions. + +# avoid warnings with -d. +$install_name ||= nil +$so_name ||= nil +$unicode_version ||= nil +arch = $arch or raise "missing -arch" +version = $version or raise "missing -version" + +srcdir = File.expand_path('../..', __FILE__) +$:.unshift(".") + +require "fileutils" +mkconfig = File.basename($0) + +fast = {'prefix'=>true, 'ruby_install_name'=>true, 'INSTALL'=>true, 'EXEEXT'=>true} + +win32 = /mswin/ =~ arch +universal = /universal.*darwin/ =~ arch +v_fast = [] +v_others = [] +vars = {} +continued_name = nil +continued_line = nil +install_name = nil +so_name = nil +File.foreach "config.status" do |line| + next if /^#/ =~ line + name = nil + case line + when /^s([%,])@(\w+)@\1(?:\|\#_!!_\#\|)?(.*)\1/ + name = $2 + val = $3.gsub(/\\(?=,)/, '') + when /^S\["(\w+)"\]\s*=\s*"(.*)"\s*(\\)?$/ + name = $1 + val = $2 + if $3 + continued_line = [val] + continued_name = name + next + end + when /^"(.*)"\s*(\\)?$/ + next if !continued_line + continued_line << $1 + next if $2 + continued_line.each {|s| s.sub!(/\\n\z/, "\n")} + val = continued_line.join + name = continued_name + continued_line = nil + when /^(?:ac_given_)?INSTALL=(.*)/ + v_fast << " CONFIG[\"INSTALL\"] = " + $1 + "\n" + end + + if name + case name + when /^(?:ac_.*|configure_input|(?:top_)?srcdir|\w+OBJS)$/; next + when /^(?:X|(?:MINI|RUN|(?:HAVE_)?BASE|BOOTSTRAP|BTEST)RUBY(?:_COMMAND)?$)/; next + when /^INSTALLDOC|TARGET$/; next + when /^DTRACE/; next + when /^(?:MAJOR|MINOR|TEENY)$/; vars[name] = val; next + when /^LIBRUBY_D?LD/; next + when /^RUBY_INSTALL_NAME$/; next vars[name] = (install_name = val).dup if $install_name + when /^RUBY_SO_NAME$/; next vars[name] = (so_name = val).dup if $so_name + when /^arch$/; if val.empty? then val = arch else arch = val end + when /^sitearch$/; val = '$(arch)' if val.empty? + when /^DESTDIR$/; next + end + case val + when /^\$\(ac_\w+\)$/; next + when /^\$\{ac_\w+\}$/; next + when /^\$ac_\w+$/; next + end + if /^program_transform_name$/ =~ name + val.sub!(/\As(\\?\W)(?:\^|\${1,2})\1\1(;|\z)/, '') + if val.empty? + $install_name ||= "ruby" + next + end + unless $install_name + $install_name = "ruby" + val.gsub!(/\$\$/, '$') + val.scan(%r[\G[\s;]*(/(?:\\.|[^/])*/)?([sy])(\\?\W)((?:(?!\3)(?:\\.|.))*)\3((?:(?!\3)(?:\\.|.))*)\3([gi]*)]) do + |addr, cmd, sep, pat, rep, opt| + if addr + Regexp.new(addr[/\A\/(.*)\/\z/, 1]) =~ $install_name or next + end + case cmd + when 's' + pat = Regexp.new(pat, opt.include?('i')) + if opt.include?('g') + $install_name.gsub!(pat, rep) + else + $install_name.sub!(pat, rep) + end + when 'y' + $install_name.tr!(Regexp.quote(pat), rep) + end + end + end + end + eq = win32 && vars[name] ? '<< "\n"' : '=' + vars[name] = val + if name == "configure_args" + val.gsub!(/--with-out-ext/, "--without-ext") + end + val = val.gsub(/\$(?:\$|\{?(\w+)\}?)/) {$1 ? "$(#{$1})" : $&}.dump + case name + when /^prefix$/ + val = "(TOPDIR || DESTDIR + #{val})" + when /^ARCH_FLAG$/ + val = "arch_flag || #{val}" if universal + when /^UNIVERSAL_ARCHNAMES$/ + universal, val = val, 'universal' if universal + when /^arch$/ + if universal + val.sub!(/universal/, %q[#{arch && universal[/(?:\A|\s)#{Regexp.quote(arch)}=(\S+)/, 1] || '\&'}]) + end + when /^oldincludedir$/ + val = '"$(SDKROOT)"'+val if /darwin/ =~ arch + end + v = " CONFIG[\"#{name}\"] #{eq} #{val}\n" + if fast[name] + v_fast << v + else + v_others << v + end + case name + when "RUBY_PROGRAM_VERSION" + version = val[/\A"(.*)"\z/, 1] + end + end +# break if /^CEOF/ +end + +drive = File::PATH_SEPARATOR == ';' + +def vars.expand(val, config = self) + newval = val.gsub(/\$\$|\$\(([^()]+)\)|\$\{([^{}]+)\}/) { + var = $& + if !(v = $1 || $2) + '$' + elsif key = config[v = v[/\A[^:]+(?=(?::(.*?)=(.*))?\z)/]] + pat, sub = $1, $2 + config[v] = false + config[v] = expand(key, config) + key = key.gsub(/#{Regexp.quote(pat)}(?=\s|\z)/n) {sub} if pat + key + else + var + end + } + val.replace(newval) unless newval == val + val +end +prefix = vars.expand(vars["prefix"] ||= "") +rubyarchdir = vars.expand(vars["rubyarchdir"] ||= "") +relative_archdir = rubyarchdir.rindex(prefix, 0) ? rubyarchdir[prefix.size..-1] : rubyarchdir +puts %[\ +# encoding: ascii-8bit +# frozen-string-literal: false +# +# The module storing Ruby interpreter configurations on building. +# +# This file was created by #{mkconfig} when ruby was built. It contains +# build information for ruby which is used e.g. by mkmf to build +# compatible native extensions. Any changes made to this file will be +# lost the next time ruby is built. + +module RbConfig + RUBY_VERSION.start_with?("#{version[/^[0-9]+\.[0-9]+\./] || version}") or + raise "ruby lib version (#{version}) doesn't match executable version (\#{RUBY_VERSION})" + +] +print " # Ruby installed directory.\n" +print " TOPDIR = File.dirname(__FILE__).chomp!(#{relative_archdir.dump})\n" +print " # DESTDIR on make install.\n" +print " DESTDIR = ", (drive ? "TOPDIR && TOPDIR[/\\A[a-z]:/i] || " : ""), "'' unless defined? DESTDIR\n" +print <<'ARCH' if universal + arch_flag = ENV['ARCHFLAGS'] || ((e = ENV['RC_ARCHS']) && e.split.uniq.map {|a| "-arch #{a}"}.join(' ')) + arch = arch_flag && arch_flag[/\A\s*-arch\s+(\S+)\s*\z/, 1] +ARCH +print " universal = #{universal}\n" if universal +print " # The hash configurations stored.\n" +print " CONFIG = {}\n" +print " CONFIG[\"DESTDIR\"] = DESTDIR\n" + +versions = {} +IO.foreach(File.join(srcdir, "version.h")) do |l| + m = /^\s*#\s*define\s+RUBY_(PATCHLEVEL)\s+(-?\d+)/.match(l) + if m + versions[m[1]] = m[2] + break if versions.size == 4 + next + end + m = /^\s*#\s*define\s+RUBY_VERSION\s+\W?([.\d]+)/.match(l) + if m + versions['MAJOR'], versions['MINOR'], versions['TEENY'] = m[1].split('.') + break if versions.size == 4 + next + end +end +%w[MAJOR MINOR TEENY PATCHLEVEL].each do |v| + print " CONFIG[#{v.dump}] = #{versions[v].dump}\n" +end + +dest = drive ? %r'= "(?!\$[\(\{])(?i:[a-z]:)' : %r'= "(?!\$[\(\{])' +v_disabled = {} +v_others.collect! do |x| + if /^\s*CONFIG\["((?!abs_|old)[a-z]+(?:_prefix|dir))"\]/ === x + name = $1 + if /= "no"$/ =~ x + v_disabled[name] = true + v_others.delete(name) + next + end + x.sub(dest, '= "$(DESTDIR)') + else + x + end +end +v_others.compact! + +if $install_name + if install_name and vars.expand("$(RUBY_INSTALL_NAME)") == $install_name + $install_name = install_name + end + v_fast << " CONFIG[\"ruby_install_name\"] = \"" + $install_name + "\"\n" + v_fast << " CONFIG[\"RUBY_INSTALL_NAME\"] = \"" + $install_name + "\"\n" +end +if $so_name + if so_name and vars.expand("$(RUBY_SO_NAME)") == $so_name + $so_name = so_name + end + v_fast << " CONFIG[\"RUBY_SO_NAME\"] = \"" + $so_name + "\"\n" +end + +print(*v_fast) +print(*v_others) +print <<EOS if $unicode_version + CONFIG["UNICODE_VERSION"] = #{$unicode_version.dump} +EOS +print <<EOS if /darwin/ =~ arch + CONFIG["SDKROOT"] = "\#{ENV['SDKROOT']}" # don't run xcrun every time, usually useless. +EOS +print <<EOS + CONFIG["archdir"] = "$(rubyarchdir)" + CONFIG["topdir"] = File.dirname(__FILE__) + # Almost same with CONFIG. MAKEFILE_CONFIG has other variable + # reference like below. + # + # MAKEFILE_CONFIG["bindir"] = "$(exec_prefix)/bin" + # + # The values of this constant is used for creating Makefile. + # + # require 'rbconfig' + # + # print <<-END_OF_MAKEFILE + # prefix = \#{Config::MAKEFILE_CONFIG['prefix']} + # exec_prefix = \#{Config::MAKEFILE_CONFIG['exec_prefix']} + # bindir = \#{Config::MAKEFILE_CONFIG['bindir']} + # END_OF_MAKEFILE + # + # => prefix = /usr/local + # exec_prefix = $(prefix) + # bindir = $(exec_prefix)/bin MAKEFILE_CONFIG = {} + # + # RbConfig.expand is used for resolving references like above in rbconfig. + # + # require 'rbconfig' + # p Config.expand(Config::MAKEFILE_CONFIG["bindir"]) + # # => "/usr/local/bin" + MAKEFILE_CONFIG = {} + CONFIG.each{|k,v| MAKEFILE_CONFIG[k] = v.dup} + + # call-seq: + # + # RbConfig.expand(val) -> string + # RbConfig.expand(val, config) -> string + # + # expands variable with given +val+ value. + # + # RbConfig.expand("$(bindir)") # => /home/foobar/all-ruby/ruby19x/bin + def RbConfig::expand(val, config = CONFIG) + newval = val.gsub(/\\$\\$|\\$\\(([^()]+)\\)|\\$\\{([^{}]+)\\}/) { + var = $& + if !(v = $1 || $2) + '$' + elsif key = config[v = v[/\\A[^:]+(?=(?::(.*?)=(.*))?\\z)/]] + pat, sub = $1, $2 + config[v] = false + config[v] = RbConfig::expand(key, config) + key = key.gsub(/\#{Regexp.quote(pat)}(?=\\s|\\z)/n) {sub} if pat + key + else + var + end + } + val.replace(newval) unless newval == val + val + end + CONFIG.each_value do |val| + RbConfig::expand(val) + end + + # call-seq: + # + # RbConfig.ruby -> path + # + # returns the absolute pathname of the ruby command. + def RbConfig.ruby + File.join( + RbConfig::CONFIG["bindir"], + RbConfig::CONFIG["ruby_install_name"] + RbConfig::CONFIG["EXEEXT"] + ) + end +end +CROSS_COMPILING = nil unless defined? CROSS_COMPILING +EOS + +# vi:set sw=2: diff --git a/tool/mkrunnable.rb b/tool/mkrunnable.rb new file mode 100755 index 0000000000..e43c4329e7 --- /dev/null +++ b/tool/mkrunnable.rb @@ -0,0 +1,137 @@ +#!./miniruby +# -*- coding: us-ascii -*- + +# Used by "make runnable" target, to make symbolic links from a build +# directory. + +require './rbconfig' +require 'fileutils' + +case ARGV[0] +when "-n" + ARGV.shift + include FileUtils::DryRun +when "-v" + ARGV.shift + include FileUtils::Verbose +else + include FileUtils +end + +module Mswin + def ln_safe(src, dest, *opt) + cmd = ["mklink", dest.tr("/", "\\"), src.tr("/", "\\")] + cmd[1, 0] = opt + return if system("cmd", "/c", *cmd) + # TODO: use RUNAS or something + puts cmd.join(" ") + end + + def ln_dir_safe(src, dest) + ln_safe(src, dest, "/d") + end +end + +def clean_link(src, dest) + begin + link = File.readlink(dest) + rescue + else + return if link == src + File.unlink(dest) + end + yield src, dest +end + +def ln_safe(src, dest) + ln_sf(src, dest) +end + +alias ln_dir_safe ln_safe + +if !File.respond_to?(:symlink) && /mingw|mswin/ =~ (CROSS_COMPILING || RUBY_PLATFORM) + extend Mswin +end + +def clean_path(path) + path = "#{path}/".gsub(/(\A|\/)(?:\.\/)+/, '\1').tr_s('/', '/') + nil while path.sub!(/[^\/]+\/\.\.\//, '') + path +end + +def relative_path_from(path, base) + path = clean_path(path) + base = clean_path(base) + path, base = [path, base].map{|s|s.split("/")} + until path.empty? or base.empty? or path[0] != base[0] + path.shift + base.shift + end + path, base = [path, base].map{|s|s.join("/")} + if /(\A|\/)\.\.\// =~ base + File.expand_path(path) + else + base.gsub!(/[^\/]+/, '..') + File.join(base, path) + end +end + +def ln_relative(src, dest) + return if File.identical?(src, dest) + parent = File.dirname(dest) + File.directory?(parent) or mkdir_p(parent) + clean_link(relative_path_from(src, parent), dest) {|s, d| ln_safe(s, d)} +end + +def ln_dir_relative(src, dest) + return if File.identical?(src, dest) + parent = File.dirname(dest) + File.directory?(parent) or mkdir_p(parent) + clean_link(relative_path_from(src, parent), dest) {|s, d| ln_dir_safe(s, d)} +end + +config = RbConfig::MAKEFILE_CONFIG.merge("prefix" => ".", "exec_prefix" => ".") +config.each_value {|s| RbConfig.expand(s, config)} +srcdir = config["srcdir"] ||= File.dirname(__FILE__) +top_srcdir = config["top_srcdir"] ||= File.dirname(srcdir) +extout = ARGV[0] || config["EXTOUT"] +version = config["ruby_version"] +arch = config["arch"] +bindir = config["bindir"] +libdirname = config["libdirname"] +libdir = config[libdirname || "libdir"] +vendordir = config["vendordir"] +rubylibdir = config["rubylibdir"] +rubyarchdir = config["rubyarchdir"] +archdir = "#{extout}/#{arch}" +rubylibs = [vendordir, rubylibdir, rubyarchdir] +[bindir, libdir, archdir].uniq.each do |dir| + File.directory?(dir) or mkdir_p(dir) +end + +exeext = config["EXEEXT"] +ruby_install_name = config["ruby_install_name"] +rubyw_install_name = config["rubyw_install_name"] +goruby_install_name = "go" + ruby_install_name +[ruby_install_name, rubyw_install_name, goruby_install_name].map do |ruby| + ruby += exeext + if ruby and !ruby.empty? and !File.file?(target = "#{bindir}/#{ruby}") + ln_relative(ruby, target) + end +end +so = config["LIBRUBY_SO"] +libruby = [config["LIBRUBY_A"]] +if /\.dll\z/i =~ so + ln_relative(so, "#{bindir}/#{so}") +else + libruby << so +end +libruby.concat(config["LIBRUBY_ALIASES"].split) +libruby.each {|lib|ln_relative(lib, "#{libdir}/#{lib}")} +ln_dir_relative("#{extout}/common", rubylibdir) +rubyarchdir.sub!(rubylibdir, "#{extout}/common") +vendordir.sub!(rubylibdir, "#{extout}/common") +ln_dir_relative(archdir, rubyarchdir) +vendordir.sub!(rubyarchdir, archdir) +ln_dir_relative("#{top_srcdir}/lib", vendordir) +ln_relative("rbconfig.rb", "#{archdir}/rbconfig.rb") diff --git a/tool/node_name.rb b/tool/node_name.rb new file mode 100755 index 0000000000..5c67d7ccd5 --- /dev/null +++ b/tool/node_name.rb @@ -0,0 +1,10 @@ +#! ./miniruby + +# Used when making Ruby to generate node_name.inc. +# See common.mk for details. + +while gets + if ~/enum node_type \{/..~/^\};/ + ~/(NODE_.+),/ and puts(" case #{$1}:\n\treturn \"#{$1}\";") + end +end diff --git a/tool/parse.rb b/tool/parse.rb new file mode 100644 index 0000000000..93ae3e43cb --- /dev/null +++ b/tool/parse.rb @@ -0,0 +1,16 @@ +# Used as part of the "make parse" Makefile target. +# See common.mk for details. + +$file = ARGV[0] +$str = ARGF.read.sub(/^__END__.*\z/m, '') +puts '# ' + '-' * 70 +puts "# target program: " +puts '# ' + '-' * 70 +puts $str +puts '# ' + '-' * 70 + +$parsed = RubyVM::InstructionSequence.compile_file($file) +puts "# disasm result: " +puts '# ' + '-' * 70 +puts $parsed.disasm +puts '# ' + '-' * 70 diff --git a/tool/prereq.status b/tool/prereq.status new file mode 100644 index 0000000000..7509b8cbd7 --- /dev/null +++ b/tool/prereq.status @@ -0,0 +1,43 @@ +s,@EXTOUT@,tmp,g +s,@ruby_version@,0.0.0,g +s,@NULLCMD@,:,g +s,@ARCH_FLAG@,,g +s,@BASERUBY@,ruby,g +s,@BOOTSTRAPRUBY@,$(BASERUBY),g +s,@CC@,false,g +s,@CFLAGS@,,g +s,@CHDIR@,cd,g +s,@CONFIGURE@,configure,g +s,@CP@,cp,g +s,@CPPFLAGS@,,g +s,@CXXFLAGS@,,g +s,@DLDFLAGS@,,g +s,@EXEEXT@,,g +s,@HAVE_BASERUBY@,yes,g +s,@IFCHANGE@,tool/ifchange,g +s,@LDFLAGS@,,g +s,@LIBEXT@,a,g +s,@LIBRUBY@,libruby.a,g +s,@LIBRUBY_A@,libruby.a,g +s,@MINIRUBY@,$(BASERUBY),g +s,@MKDIR_P@,mkdir -p,g +s,@OBJEXT@,o,g +s,@PATH_SEPARATOR@,:,g +s,@PWD@,.,g +s,@RM@,rm -f,g +s,@RMALL@,rm -fr,g +s,@RMDIR@,rmdir,g +s,@RMDIRS@,$(RMDIR) -p,g +s,@RUBY@,$(BASERUBY),g +s,@RUNRUBY@,$(MINIRUBY),g +s,@UNICODE_FILES@,,g +s,@arch@,noarch,g +s,@bindir@,,g +s,@configure_args@,,g +s,@ruby_install_name@,,g +s,@rubyarchdir@,,g +s,@rubylibprefix@,,g +s,@srcdir@,.,g + +s/@[A-Za-z][A-Za-z0-9_]*@//g +s/{\$([A-Za-z]*)}//g diff --git a/tool/probes_to_wiki.rb b/tool/probes_to_wiki.rb new file mode 100644 index 0000000000..ba8204c188 --- /dev/null +++ b/tool/probes_to_wiki.rb @@ -0,0 +1,16 @@ +### +# Converts the probes.d file to redmine wiki format. Usage: +# +# ruby tool/probes_to_wiki.rb probes.d + +File.read(ARGV[0]).scan(/\/\*.*?\*\//m).grep(/ruby/) do |comment| + comment.gsub!(/^(\/\*|[ ]*)|\*\/$/, '').strip! + puts + comment.each_line.with_index do |line, i| + if i == 0 + puts "=== #{line.chomp}" + else + puts line.gsub(/`([^`]*)`/, '(({\1}))') + end + end +end diff --git a/tool/pull-latest-mspec-spec b/tool/pull-latest-mspec-spec new file mode 100755 index 0000000000..dfd80582b3 --- /dev/null +++ b/tool/pull-latest-mspec-spec @@ -0,0 +1,18 @@ +#!/bin/bash + +# Assumes all commits have been synchronized to https://github.com/ruby/spec +# See spec/mspec/tool/sync/sync-rubyspec.rb + +rm -rf spec/mspec +git clone --depth 1 https://github.com/ruby/mspec.git spec/mspec +commit=$(git -C spec/mspec log -n 1 --format='%h') +rm -rf spec/mspec/.git +git add spec/mspec +git commit -m "Update to ruby/mspec@${commit}" + +rm -rf spec/ruby +git clone --depth 1 https://github.com/ruby/spec.git spec/ruby +commit=$(git -C spec/ruby log -n 1 --format='%h') +rm -rf spec/ruby/.git +git add spec/ruby +git commit -m "Update to ruby/spec@${commit}" diff --git a/tool/rbinstall.rb b/tool/rbinstall.rb new file mode 100755 index 0000000000..d0bb30918e --- /dev/null +++ b/tool/rbinstall.rb @@ -0,0 +1,920 @@ +#!./miniruby + +# Used by the "make install" target to install Ruby. +# See common.mk for more details. + +begin + load "./rbconfig.rb" +rescue LoadError + CONFIG = Hash.new {""} +else + include RbConfig + $".unshift File.expand_path("./rbconfig.rb") +end + +srcdir = File.expand_path('../..', __FILE__) +unless defined?(CROSS_COMPILING) and CROSS_COMPILING + $:.replace([srcdir+"/lib", Dir.pwd]) +end +require 'fileutils' +require 'shellwords' +require 'optparse' +require 'optparse/shellwords' +require 'ostruct' +require 'rubygems' +begin + require "zlib" +rescue LoadError + $" << "zlib.rb" +end + +INDENT = " "*36 +STDOUT.sync = true +File.umask(0222) + +def parse_args(argv = ARGV) + $mantype = 'doc' + $destdir = nil + $extout = nil + $make = 'make' + $mflags = [] + $install = [] + $installed_list = nil + $dryrun = false + $rdocdir = nil + $data_mode = 0644 + $prog_mode = 0755 + $dir_mode = nil + $script_mode = nil + $strip = false + $cmdtype = (if File::ALT_SEPARATOR == '\\' + File.exist?("rubystub.exe") ? 'exe' : 'cmd' + end) + mflags = [] + opt = OptionParser.new + opt.on('-n', '--dry-run') {$dryrun = true} + opt.on('--dest-dir=DIR') {|dir| $destdir = dir} + opt.on('--extout=DIR') {|dir| $extout = (dir unless dir.empty?)} + opt.on('--make=COMMAND') {|make| $make = make} + opt.on('--mantype=MAN') {|man| $mantype = man} + opt.on('--make-flags=FLAGS', '--mflags', Shellwords) do |v| + if arg = v.first + arg.insert(0, '-') if /\A[^-][^=]*\Z/ =~ arg + end + $mflags.concat(v) + end + opt.on('-i', '--install=TYPE', $install_procs.keys) do |ins| + $install << ins + end + opt.on('--data-mode=OCTAL-MODE', OptionParser::OctalInteger) do |mode| + $data_mode = mode + end + opt.on('--prog-mode=OCTAL-MODE', OptionParser::OctalInteger) do |mode| + $prog_mode = mode + end + opt.on('--dir-mode=OCTAL-MODE', OptionParser::OctalInteger) do |mode| + $dir_mode = mode + end + opt.on('--script-mode=OCTAL-MODE', OptionParser::OctalInteger) do |mode| + $script_mode = mode + end + opt.on('--installed-list [FILENAME]') {|name| $installed_list = name} + opt.on('--rdoc-output [DIR]') {|dir| $rdocdir = dir} + opt.on('--cmd-type=TYPE', %w[cmd plain]) {|cmd| $cmdtype = (cmd unless cmd == 'plain')} + opt.on('--[no-]strip') {|strip| $strip = strip} + + opt.order!(argv) do |v| + case v + when /\AINSTALL[-_]([-\w]+)=(.*)/ + argv.unshift("--#{$1.tr('_', '-')}=#{$2}") + when /\A\w[-\w+]*=\z/ + mflags << v + when /\A\w[-\w+]*\z/ + $install << v.intern + else + raise OptionParser::InvalidArgument, v + end + end rescue abort "#{$!.message}\n#{opt.help}" + + unless defined?(RbConfig) + puts opt.help + exit + end + + $make, *rest = Shellwords.shellwords($make) + $mflags.unshift(*rest) unless rest.empty? + $mflags.unshift(*mflags) + + def $mflags.set?(flag) + grep(/\A-(?!-).*#{flag.chr}/i) { return true } + false + end + def $mflags.defined?(var) + grep(/\A#{var}=(.*)/) {return block_given? ? yield($1) : $1} + false + end + + if $mflags.set?(?n) + $dryrun = true + else + $mflags << '-n' if $dryrun + end + + $destdir ||= $mflags.defined?("DESTDIR") + if $extout ||= $mflags.defined?("EXTOUT") + RbConfig.expand($extout) + end + + $continue = $mflags.set?(?k) + + if $installed_list ||= $mflags.defined?('INSTALLED_LIST') + RbConfig.expand($installed_list, RbConfig::CONFIG) + $installed_list = open($installed_list, "ab") + $installed_list.sync = true + end + + $rdocdir ||= $mflags.defined?('RDOCOUT') + + $dir_mode ||= $prog_mode | 0700 + $script_mode ||= $prog_mode +end + +$install_procs = Hash.new {[]} +def install?(*types, &block) + $install_procs[:all] <<= block + types.each do |type| + $install_procs[type] <<= block + end +end + +def strip_file(files) + if !defined?($strip_command) and (cmd = CONFIG["STRIP"]) + case cmd + when "", "true", ":" then return + else $strip_command = Shellwords.shellwords(cmd) + end + elsif !$strip_command + return + end + system(*($strip_command + [files].flatten)) +end + +def install(src, dest, options = {}) + options = options.clone + strip = options.delete(:strip) + options[:preserve] = true + d = with_destdir(dest) + super(src, d, options) + srcs = Array(src) + if strip + d = srcs.map {|s| File.join(d, File.basename(s))} if $made_dirs[dest] + strip_file(d) + end + if $installed_list + dest = srcs.map {|s| File.join(dest, File.basename(s))} if $made_dirs[dest] + $installed_list.puts dest + end +end + +def ln_sf(src, dest) + super(src, with_destdir(dest)) + $installed_list.puts dest if $installed_list +end + +$made_dirs = {} +def makedirs(dirs) + dirs = fu_list(dirs) + dirs.collect! do |dir| + realdir = with_destdir(dir) + realdir unless $made_dirs.fetch(dir) do + $made_dirs[dir] = true + $installed_list.puts(File.join(dir, "")) if $installed_list + File.directory?(realdir) + end + end.compact! + super(dirs, :mode => $dir_mode) unless dirs.empty? +end + +FalseProc = proc {false} +def path_matcher(pat) + if pat and !pat.empty? + proc {|f| pat.any? {|n| File.fnmatch?(n, f)}} + else + FalseProc + end +end + +def install_recursive(srcdir, dest, options = {}) + opts = options.clone + noinst = opts.delete(:no_install) + glob = opts.delete(:glob) || "*" + maxdepth = opts.delete(:maxdepth) + subpath = (srcdir.size+1)..-1 + prune = [] + skip = [] + if noinst + if Array === noinst + prune = noinst.grep(/#{File::SEPARATOR}/o).map!{|f| f.chomp(File::SEPARATOR)} + skip = noinst.grep(/\A[^#{File::SEPARATOR}]*\z/o) + else + if noinst.index(File::SEPARATOR) + prune = [noinst] + else + skip = [noinst] + end + end + end + skip |= %w"#*# *~ *.old *.bak *.orig *.rej *.diff *.patch *.core" + prune = path_matcher(prune) + skip = path_matcher(skip) + File.directory?(srcdir) or return rescue return + paths = [[srcdir, dest, 0]] + found = [] + while file = paths.shift + found << file + file, d, dir = *file + if dir + depth = dir + 1 + next if maxdepth and maxdepth < depth + files = [] + Dir.foreach(file) do |f| + src = File.join(file, f) + d = File.join(dest, dir = src[subpath]) + stat = File.lstat(src) rescue next + if stat.directory? + files << [src, d, depth] if maxdepth != depth and /\A\./ !~ f and !prune[dir] + elsif stat.symlink? + # skip + else + files << [src, d, false] if File.fnmatch?(glob, f) and !skip[f] + end + end + paths.insert(0, *files) + end + end + for src, d, dir in found + if dir + makedirs(d) + else + makedirs(d[/.*(?=\/)/m]) + if block_given? + yield src, d, opts + else + install src, d, opts + end + end + end +end + +def open_for_install(path, mode) + data = open(realpath = with_destdir(path), "rb") {|f| f.read} rescue nil + newdata = yield + unless $dryrun + unless newdata == data + open(realpath, "wb", mode) {|f| f.write newdata} + end + File.chmod(mode, realpath) + end + $installed_list.puts path if $installed_list +end + +def with_destdir(dir) + return dir if !$destdir or $destdir.empty? + dir = dir.sub(/\A\w:/, '') if File::PATH_SEPARATOR == ';' + $destdir + dir +end + +def without_destdir(dir) + return dir if !$destdir or $destdir.empty? + dir.start_with?($destdir) ? dir[$destdir.size..-1] : dir +end + +def prepare(mesg, basedir, subdirs=nil) + return unless basedir + case + when !subdirs + dirs = basedir + when subdirs.size == 0 + subdirs = nil + when subdirs.size == 1 + dirs = [basedir = File.join(basedir, subdirs)] + subdirs = nil + else + dirs = [basedir, *subdirs.collect {|dir| File.join(basedir, dir)}] + end + printf("%-*s%s%s\n", INDENT.size, "installing #{mesg}:", basedir, + (subdirs ? " (#{subdirs.join(', ')})" : "")) + makedirs(dirs) +end + +def CONFIG.[](name, mandatory = false) + value = super(name) + if mandatory + raise "CONFIG['#{name}'] must be set" if !value or value.empty? + end + value +end + +exeext = CONFIG["EXEEXT"] + +ruby_install_name = CONFIG["ruby_install_name", true] +rubyw_install_name = CONFIG["rubyw_install_name"] +goruby_install_name = "go" + ruby_install_name + +bindir = CONFIG["bindir", true] +libdir = CONFIG[CONFIG.fetch("libdirname", "libdir"), true] +rubyhdrdir = CONFIG["rubyhdrdir", true] +archhdrdir = CONFIG["rubyarchhdrdir"] || (rubyhdrdir + "/" + CONFIG['arch']) +rubylibdir = CONFIG["rubylibdir", true] +archlibdir = CONFIG["rubyarchdir", true] +if CONFIG["sitedir"] + sitelibdir = CONFIG["sitelibdir"] + sitearchlibdir = CONFIG["sitearchdir"] +end +if CONFIG["vendordir"] + vendorlibdir = CONFIG["vendorlibdir"] + vendorarchlibdir = CONFIG["vendorarchdir"] +end +mandir = CONFIG["mandir", true] +docdir = CONFIG["docdir", true] +configure_args = Shellwords.shellwords(CONFIG["configure_args"]) +enable_shared = CONFIG["ENABLE_SHARED"] == 'yes' +dll = CONFIG["LIBRUBY_SO", enable_shared] +lib = CONFIG["LIBRUBY", true] +arc = CONFIG["LIBRUBY_A", true] +load_relative = CONFIG["LIBRUBY_RELATIVE"] == 'yes' + +install?(:local, :arch, :bin, :'bin-arch') do + prepare "binary commands", bindir + + install ruby_install_name+exeext, bindir, :mode => $prog_mode, :strip => $strip + if rubyw_install_name and !rubyw_install_name.empty? + install rubyw_install_name+exeext, bindir, :mode => $prog_mode, :strip => $strip + end + if File.exist? goruby_install_name+exeext + install goruby_install_name+exeext, bindir, :mode => $prog_mode, :strip => $strip + end + if enable_shared and dll != lib + install dll, bindir, :mode => $prog_mode, :strip => $strip + end +end + +install?(:local, :arch, :lib, :'lib-arch') do + prepare "base libraries", libdir + + install lib, libdir, :mode => $prog_mode, :strip => $strip unless lib == arc + install arc, libdir, :mode => $data_mode unless CONFIG["INSTALL_STATIC_LIBRARY"] == "no" + if dll == lib and dll != arc + for link in CONFIG["LIBRUBY_ALIASES"].split + ln_sf(dll, File.join(libdir, link)) + end + end + + prepare "arch files", archlibdir + install "rbconfig.rb", archlibdir, :mode => $data_mode + if CONFIG["ARCHFILE"] + for file in CONFIG["ARCHFILE"].split + install file, archlibdir, :mode => $data_mode + end + end +end + +install?(:local, :arch, :data) do + pc = CONFIG["ruby_pc"] + if pc and File.file?(pc) and File.size?(pc) + prepare "pkgconfig data", pkgconfigdir = File.join(libdir, "pkgconfig") + install pc, pkgconfigdir, :mode => $data_mode + end +end + +install?(:ext, :arch, :'ext-arch') do + prepare "extension objects", archlibdir + noinst = %w[-* -*/] | (CONFIG["no_install_files"] || "").split + install_recursive("#{$extout}/#{CONFIG['arch']}", archlibdir, :no_install => noinst, :mode => $prog_mode, :strip => $strip) + prepare "extension objects", sitearchlibdir + prepare "extension objects", vendorarchlibdir + if extso = File.read("exts.mk")[/^EXTSO[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] and + !(extso = extso.gsub(/\\\n/, '').split).empty? + libpathenv = CONFIG["LIBPATHENV"] + dest = CONFIG[!libpathenv || libpathenv == "PATH" ? "bindir" : "libdir"] + prepare "external libraries", dest + for file in extso + install file, dest, :mode => $prog_mode + end + end +end +install?(:ext, :arch, :hdr, :'arch-hdr', :'hdr-arch') do + prepare "extension headers", archhdrdir + install_recursive("#{$extout}/include/#{CONFIG['arch']}", archhdrdir, :glob => "*.h", :mode => $data_mode) +end +install?(:ext, :comm, :'ext-comm') do + prepare "extension scripts", rubylibdir + install_recursive("#{$extout}/common", rubylibdir, :mode => $data_mode) + prepare "extension scripts", sitelibdir + prepare "extension scripts", vendorlibdir +end +install?(:ext, :comm, :hdr, :'comm-hdr', :'hdr-comm') do + hdrdir = rubyhdrdir + "/ruby" + prepare "extension headers", hdrdir + install_recursive("#{$extout}/include/ruby", hdrdir, :glob => "*.h", :mode => $data_mode) +end + +install?(:doc, :rdoc) do + if $rdocdir + ridatadir = File.join(CONFIG['ridir'], CONFIG['ruby_version'], "system") + prepare "rdoc", ridatadir + install_recursive($rdocdir, ridatadir, :mode => $data_mode) + end +end +install?(:doc, :capi) do + prepare "capi-docs", docdir + install_recursive "doc/capi", docdir+"/capi", :mode => $data_mode +end + +prolog_script = <<EOS +bindir="#{load_relative ? '${0%/*}' : bindir.gsub(/\"/, '\\\\"')}" +EOS +if CONFIG["LIBRUBY_RELATIVE"] != 'yes' and libpathenv = CONFIG["LIBPATHENV"] + pathsep = File::PATH_SEPARATOR + prolog_script << <<EOS +libdir="#{load_relative ? '$\{bindir%/bin\}/lib' : libdir.gsub(/\"/, '\\\\"')}" +export #{libpathenv}="$libdir${#{libpathenv}:+#{pathsep}$#{libpathenv}}" +EOS +end +prolog_script << %Q[exec "$bindir/#{ruby_install_name}" "-x" "$0" "$@"\n] +PROLOG_SCRIPT = {} +PROLOG_SCRIPT["exe"] = "#!#{bindir}/#{ruby_install_name}" +PROLOG_SCRIPT["cmd"] = <<EOS +:""||{ ""=> %q<-*- ruby -*- +@"%~dp0#{ruby_install_name}" -x "%~f0" %* +@exit /b %ERRORLEVEL% +};{#\n#{prolog_script.gsub(/(?=\n)/, ' #')}>,\n} +EOS +PROLOG_SCRIPT.default = (load_relative || /\s/ =~ bindir) ? + <<EOS : PROLOG_SCRIPT["exe"] +#!/bin/sh +# -*- ruby -*- +_=_\\ +=begin +#{prolog_script}=end +EOS + +$script_installer = Struct.new(:ruby_shebang, :ruby_bin, :ruby_install_name, + :stub, :trans) do + ruby_shebang = File.join(bindir, ruby_install_name) + if File::ALT_SEPARATOR + ruby_bin = ruby_shebang.tr(File::SEPARATOR, File::ALT_SEPARATOR) + if $cmdtype == 'exe' + stub = File.open("rubystub.exe", "rb") {|f| f.read} << "\n" rescue nil + end + end + if trans = CONFIG["program_transform_name"] + exp = [] + trans.gsub!(/\$\$/, '$') + trans.scan(%r[\G[\s;]*(/(?:\\.|[^/])*/)?([sy])(\\?\W)((?:(?!\3)(?:\\.|.))*)\3((?:(?!\3)(?:\\.|.))*)\3([gi]*)]) do + |addr, cmd, sep, pat, rep, opt| + addr &&= Regexp.new(addr[/\A\/(.*)\/\z/, 1]) + case cmd + when 's' + next if pat == '^' and rep.empty? + exp << [addr, (opt.include?('g') ? :gsub! : :sub!), + Regexp.new(pat, opt.include?('i')), rep.gsub(/&/){'\&'}] + when 'y' + exp << [addr, :tr!, Regexp.quote(pat), rep] + end + end + trans = proc do |base| + exp.each {|addr, opt, pat, rep| base.__send__(opt, pat, rep) if !addr or addr =~ base} + base + end + elsif /ruby/ =~ ruby_install_name + trans = proc {|base| ruby_install_name.sub(/ruby/, base)} + else + trans = proc {|base| base} + end + + def prolog(shebang) + shebang.sub!(/\r$/, '') + script = PROLOG_SCRIPT[$cmdtype] + shebang.sub!(/\A(\#!.*?ruby\b)?/) do + if script.end_with?("\n") + script + ($1 || "#!ruby\n") + else + $1 ? script : "#{script}\n" + end + end + shebang + end + + def install(src, cmd) + cmd = cmd.sub(/[^\/]*\z/m) {|n| RbConfig.expand(trans[n])} + + shebang, body = open(src, "rb") do |f| + next f.gets, f.read + end + shebang or raise "empty file - #{src}" + shebang = prolog(shebang) + body.gsub!(/\r$/, '') + + cmd << ".#{$cmdtype}" if $cmdtype + open_for_install(cmd, $script_mode) do + case $cmdtype + when "exe" + stub + shebang + body + else + shebang + body + end + end + end + + break new(ruby_shebang, ruby_bin, ruby_install_name, stub, trans) +end + +install?(:local, :comm, :bin, :'bin-comm') do + prepare "command scripts", bindir + + install_recursive(File.join(srcdir, "bin"), bindir, :maxdepth => 1) do |src, cmd| + $script_installer.install(src, cmd) + end +end + +install?(:local, :comm, :lib) do + prepare "library scripts", rubylibdir + noinst = %w[*.txt *.rdoc *.gemspec] + install_recursive(File.join(srcdir, "lib"), rubylibdir, :no_install => noinst, :mode => $data_mode) +end + +install?(:local, :comm, :hdr, :'comm-hdr') do + prepare "common headers", rubyhdrdir + + noinst = [] + unless RUBY_PLATFORM =~ /mswin|mingw|bccwin/ + noinst << "win32.h" + end + noinst = nil if noinst.empty? + install_recursive(File.join(srcdir, "include"), rubyhdrdir, :no_install => noinst, :glob => "*.h", :mode => $data_mode) +end + +install?(:local, :comm, :man) do + mdocs = Dir["#{srcdir}/man/*.[1-9]"] + prepare "manpages", mandir, ([] | mdocs.collect {|mdoc| mdoc[/\d+$/]}).sort.collect {|sec| "man#{sec}"} + + case $mantype + when /\.(?:(gz)|bz2)\z/ + compress = $1 ? "gzip" : "bzip2" + suffix = $& + end + mandir = File.join(mandir, "man") + has_goruby = File.exist?(goruby_install_name+exeext) + require File.join(srcdir, "tool/mdoc2man.rb") if /\Adoc\b/ !~ $mantype + mdocs.each do |mdoc| + next unless File.file?(mdoc) and open(mdoc){|fh| fh.read(1) == '.'} + base = File.basename(mdoc) + if base == "goruby.1" + next unless has_goruby + end + + destdir = mandir + (section = mdoc[/\d+$/]) + destname = ruby_install_name.sub(/ruby/, base.chomp(".#{section}")) + destfile = File.join(destdir, "#{destname}.#{section}") + + if /\Adoc\b/ =~ $mantype + if compress + w = open(mdoc) {|f| + stdin = STDIN.dup + STDIN.reopen(f) + begin + destfile << suffix + IO.popen(compress) {|f| f.read} + ensure + STDIN.reopen(stdin) + stdin.close + end + } + open_for_install(destfile, $data_mode) {w} + else + install mdoc, destfile, :mode => $data_mode + end + else + class << (w = []) + alias print push + end + open(mdoc) {|r| Mdoc2Man.mdoc2man(r, w)} + w = w.join("") + if compress + require 'tmpdir' + Dir.mktmpdir("man") {|d| + dest = File.join(d, File.basename(destfile)) + File.open(dest, "wb") {|f| f.write w} + if system(compress, dest) + w = File.open(dest+suffix, "rb") {|f| f.read} + destfile << suffix + end + } + end + open_for_install(destfile, $data_mode) {w} + end + end +end + +module RbInstall + module Specs + class FileCollector + def initialize(gemspec) + @gemspec = gemspec + @base_dir = File.dirname(gemspec) + end + + def collect + (ruby_libraries + built_libraries).sort + end + + private + def type + /\/(ext|lib)?\/.*?\z/ =~ @base_dir + $1 + end + + def ruby_libraries + case type + when "ext" + prefix = "#{$extout}/common/" + base = "#{prefix}#{relative_base}" + when "lib" + base = @base_dir + prefix = base.sub(/lib\/.*?\z/, "") + "lib/" + end + + if base + Dir.glob("#{base}{.rb,/**/*.rb}").collect do |ruby_source| + remove_prefix(prefix, ruby_source) + end + else + [remove_prefix(File.dirname(@gemspec) + '/', @gemspec.gsub(/gemspec/, 'rb'))] + end + end + + def built_libraries + case type + when "ext" + prefix = "#{$extout}/#{CONFIG['arch']}/" + base = "#{prefix}#{relative_base}" + dlext = CONFIG['DLEXT'] + Dir.glob("#{base}{.#{dlext},/**/*.#{dlext}}").collect do |built_library| + remove_prefix(prefix, built_library) + end + when "lib" + [] + else + [] + end + end + + def relative_base + /\/#{Regexp.escape(type)}\/(.*?)\z/ =~ @base_dir + $1 + end + + def remove_prefix(prefix, string) + string.sub(/\A#{Regexp.escape(prefix)}/, "") + end + end + end + + class UnpackedInstaller < Gem::Installer + module DirPackage + def extract_files(destination_dir, pattern = "*") + path = File.dirname(@gem.path) + return if path == destination_dir + File.chmod(0700, destination_dir) + mode = pattern == "bin/*" ? $script_mode : $data_mode + spec.files.each do |f| + src = File.join(path, f) + dest = File.join(without_destdir(destination_dir), f) + makedirs(dest[/.*(?=\/)/m]) + install src, dest, :mode => mode + end + File.chmod($dir_mode, destination_dir) + end + end + + def initialize(spec, *options) + super(spec.loaded_from, *options) + @package.extend(DirPackage).spec = spec + end + + def write_cache_file + end + + def build_extensions + end + + def shebang(bin_file_name) + path = File.join(gem_dir, spec.bindir, bin_file_name) + first_line = File.open(path, "rb") {|file| file.gets} + $script_installer.prolog(first_line).chomp + end + + def app_script_text(bin_file_name) + # move shell script part after comments generated by RubyGems. + super.sub(/\A + (\#!\/bin\/sh\n\#.*-\*-\s*ruby\s*-\*-.*\n) + ((?:.*\n)*?\#!.*ruby.*\n) + \#\n + ((?:\#.*\n)+)/x, '\1\3\2') + end + + def check_executable_overwrite(filename) + return if @wrappers and same_bin_script?(filename, @bin_dir) + super + end + + def generate_bin_script(filename, bindir) + return if same_bin_script?(filename, bindir) + super + end + + def same_bin_script?(filename, bindir) + path = File.join(bindir, formatted_program_filename(filename)) + begin + return true if File.binread(path) == app_script_text(filename) + rescue + end + false + end + end +end + +class Gem::Installer + install = instance_method(:install) + define_method(:install) do + spec.post_install_message = nil + begin + u = File.umask(0022) + install.bind(self).call + ensure + File.umask(u) + end + end + + generate_bin_script = instance_method(:generate_bin_script) + define_method(:generate_bin_script) do |filename, bindir| + generate_bin_script.bind(self).call(filename, bindir) + File.chmod($script_mode, File.join(bindir, formatted_program_filename(filename))) + end +end + +# :startdoc: + +install?(:ext, :comm, :gem, :'default-gems', :'default-gems-comm') do + install_default_gem('lib', srcdir) +end +install?(:ext, :arch, :gem, :'default-gems', :'default-gems-arch') do + install_default_gem('ext', srcdir) +end + +def load_gemspec(file) + code = File.read(file, encoding: "utf-8:-") + code.gsub!(/`git.*?`/m, '""') + begin + spec = eval(code, binding, file) + rescue SignalException, SystemExit + raise + rescue SyntaxError, Exception + end + raise("invalid spec in #{file}") unless spec + spec.loaded_from = file + spec +end + +def install_default_gem(dir, srcdir) + gem_dir = Gem.default_dir + directories = Gem.ensure_gem_subdirectories(gem_dir, :mode => $dir_mode) + prepare "default gems from #{dir}", gem_dir, directories + + spec_dir = File.join(gem_dir, directories.grep(/^spec/)[0]) + default_spec_dir = "#{spec_dir}/default" + makedirs(default_spec_dir) + + gems = Dir.glob("#{srcdir}/#{dir}/**/*.gemspec").map {|src| + spec = load_gemspec(src) + file_collector = RbInstall::Specs::FileCollector.new(src) + files = file_collector.collect + next if files.empty? + spec.files = files + spec + } + gems.compact.sort_by(&:name).each do |gemspec| + full_name = "#{gemspec.name}-#{gemspec.version}" + + puts "#{INDENT}#{gemspec.name} #{gemspec.version}" + gemspec_path = File.join(default_spec_dir, "#{full_name}.gemspec") + open_for_install(gemspec_path, $data_mode) do + gemspec.to_ruby + end + + unless gemspec.executables.empty? then + bin_dir = File.join(gem_dir, 'gems', full_name, gemspec.bindir) + makedirs(bin_dir) + + gemspec.executables.map {|exec| + $script_installer.install(File.join(srcdir, 'bin', exec), + File.join(bin_dir, exec)) + } + end + end +end + +install?(:ext, :comm, :gem, :'bundled-gems') do + gem_dir = Gem.default_dir + directories = Gem.ensure_gem_subdirectories(gem_dir, :mode => $dir_mode) + prepare "bundled gems", gem_dir, directories + install_dir = with_destdir(gem_dir) + installed_gems = {} + options = { + :install_dir => install_dir, + :bin_dir => with_destdir(bindir), + :domain => :local, + :ignore_dependencies => true, + :dir_mode => $dir_mode, + :data_mode => $data_mode, + :prog_mode => $prog_mode, + :wrappers => true, + :format_executable => true, + } + gem_ext_dir = "#$extout/gems/#{CONFIG['arch']}" + extensions_dir = Gem::StubSpecification.gemspec_stub("", gem_dir, gem_dir).extensions_dir + Gem::Specification.each_gemspec([srcdir+'/gems/*']) do |path| + spec = load_gemspec(path) + next unless spec.platform == Gem::Platform::RUBY + next unless spec.full_name == path[srcdir.size..-1][/\A\/gems\/([^\/]+)/, 1] + spec.extension_dir = "#{extensions_dir}/#{spec.full_name}" + if File.directory?(ext = "#{gem_ext_dir}/#{spec.full_name}") + spec.extensions[0] ||= "-" + end + ins = RbInstall::UnpackedInstaller.new(spec, options) + puts "#{INDENT}#{spec.name} #{spec.version}" + ins.install + File.chmod($data_mode, File.join(install_dir, "specifications", "#{spec.full_name}.gemspec")) + unless spec.extensions.empty? + install_recursive(ext, spec.extension_dir) + end + installed_gems[spec.full_name] = true + end + installed_gems, gems = Dir.glob(srcdir+'/gems/*.gem').partition {|gem| installed_gems.key?(File.basename(gem, '.gem'))} + unless installed_gems.empty? + install installed_gems, gem_dir+"/cache" + end + next if gems.empty? + if defined?(Zlib) + Gem.instance_variable_set(:@ruby, with_destdir(File.join(bindir, ruby_install_name))) + silent = Gem::SilentUI.new + gems.each do |gem| + inst = Gem::Installer.new(gem, options) + inst.spec.extension_dir = with_destdir(inst.spec.extension_dir) + begin + Gem::DefaultUserInteraction.use_ui(silent) {inst.install} + rescue Gem::InstallError => e + next + end + gemname = File.basename(gem) + puts "#{INDENT}#{gemname}" + end + # fix directory permissions + # TODO: Gem.install should accept :dir_mode option or something + File.chmod($dir_mode, *Dir.glob(install_dir+"/**/")) + # fix .gemspec permissions + File.chmod($data_mode, *Dir.glob(install_dir+"/specifications/*.gemspec")) + else + puts "skip installing bundled gems because of lacking zlib" + end +end + +parse_args() + +include FileUtils +include FileUtils::NoWrite if $dryrun +@fileutils_output = STDOUT +@fileutils_label = '' + +all = $install.delete(:all) +$install << :local << :ext if $install.empty? +installs = $install.map do |inst| + if !(procs = $install_procs[inst]) || procs.empty? + next warn("unknown install target - #{inst}") + end + procs +end +installs.flatten! +installs.uniq! +installs |= $install_procs[:all] if all +installs.each do |block| + dir = Dir.pwd + begin + block.call + ensure + Dir.chdir(dir) + end +end + +# vi:set sw=2: diff --git a/tool/rbuninstall.rb b/tool/rbuninstall.rb new file mode 100755 index 0000000000..eb324c966e --- /dev/null +++ b/tool/rbuninstall.rb @@ -0,0 +1,71 @@ +#! /usr/bin/ruby -nl + +# Used by the "make uninstall" target to uninstall Ruby. +# See common.mk for more details. + +BEGIN { + $dryrun = false + $tty = STDOUT.tty? + until ARGV.empty? + case ARGV[0] + when /\A--destdir=(.*)/ + $destdir = $1 + when /\A-n\z/ + $dryrun = true + when /\A--(?:no-)?tty\z/ + $tty = !$1 + else + break + end + ARGV.shift + end + $dirs = [] + $files = [] +} +list = ($_.chomp!('/') ? $dirs : $files) +$_ = File.join($destdir, $_) if $destdir +list << $_ +END { + status = true + $\ = ors = (!$dryrun and $tty) ? "\e[K\r" : "\n" + $files.each do |file| + print "rm #{file}" + unless $dryrun + begin + File.unlink(file) + rescue Errno::ENOENT + rescue + status = false + puts $! + end + end + end + unlink = {} + $dirs.each do |dir| + unlink[dir] = true + end + while dir = $dirs.pop + print "rmdir #{dir}" + unless $dryrun + begin + begin + unlink.delete(dir) + Dir.rmdir(dir) + rescue Errno::ENOTDIR + raise unless File.symlink?(dir) + File.unlink(dir) + end + rescue Errno::ENOENT, Errno::ENOTEMPTY + rescue + status = false + puts $! + else + parent = File.dirname(dir) + $dirs.push(parent) unless parent == dir or unlink[parent] + end + end + end + $\ = nil + print ors.chomp + exit(status) +} diff --git a/tool/redmine-backporter.rb b/tool/redmine-backporter.rb new file mode 100755 index 0000000000..38ebf55fed --- /dev/null +++ b/tool/redmine-backporter.rb @@ -0,0 +1,608 @@ +#!/usr/bin/env ruby +require 'open-uri' +require 'openssl' +require 'net/http' +require 'json' +require 'io/console' +require 'stringio' +require 'strscan' +require 'optparse' +require 'abbrev' +require 'pp' +begin + require 'readline' +rescue LoadError + module Readline; end +end + +VERSION = '0.0.1' + +opts = OptionParser.new +target_version = nil +repo_path = nil +api_key = nil +ssl_verify = true +opts.on('-k REDMINE_API_KEY', '--key=REDMINE_API_KEY', 'specify your REDMINE_API_KEY') {|v| api_key = v} +opts.on('-t TARGET_VERSION', '--target=TARGET_VARSION', /\A\d(?:\.\d)+\z/, 'specify target version (ex: 2.1)') {|v| target_version = v} +opts.on('-r RUBY_REPO_PATH', '--repository=RUBY_REPO_PATH', 'specify repository path') {|v| repo_path = v} +opts.on('--[no-]ssl-verify', TrueClass, 'use / not use SSL verify') {|v| ssl_verify = v} +opts.version = VERSION +opts.parse!(ARGV) + +http_options = {use_ssl: true} +http_options[:verify_mode] = OpenSSL::SSL::VERIFY_NONE unless ssl_verify +$openuri_options = {} +$openuri_options[:ssl_verify_mode] = OpenSSL::SSL::VERIFY_NONE unless ssl_verify + +TARGET_VERSION = target_version || ENV['TARGET_VERSION'] || (raise 'need to specify TARGET_VERSION') +RUBY_REPO_PATH = repo_path || ENV['RUBY_REPO_PATH'] +BACKPORT_CF_KEY = 'cf_5' +STATUS_CLOSE = 5 +REDMINE_API_KEY = api_key || ENV['REDMINE_API_KEY'] || (raise 'need to specify REDMINE_API_KEY') +REDMINE_BASE = 'https://bugs.ruby-lang.org' + +@query = { + 'f[]' => BACKPORT_CF_KEY, + "op[#{BACKPORT_CF_KEY}]" => '~', + "v[#{BACKPORT_CF_KEY}][]" => "#{TARGET_VERSION}: REQUIRED", + 'limit' => 40, + 'status_id' => STATUS_CLOSE, + 'sort' => 'updated_on' +} + +PRIORITIES = { + 'Low' => [:white, :blue], + 'Normal' => [], + 'High' => [:red], + 'Urgent' => [:red, :white], + 'Immediate' => [:red, :white, {underscore: true}], +} +COLORS = { + black: 30, + red: 31, + green: 32, + yellow: 33, + blue: 34, + magenta: 35, + cyan: 36, + white: 37, +} + +class String + def color(fore=nil, back=nil, bold: false, underscore: false) + seq = "" + if bold + seq << "\e[1m" + end + if underscore + seq << "\e[2m" + end + if fore + c = COLORS[fore] + raise "unknown foreground color #{fore}" unless c + seq << "\e[#{c}m" + end + if back + c = COLORS[back] + raise "unknown background color #{back}" unless c + seq << "\e[#{c + 10}m" + end + if seq.empty? + self + else + seq << self << "\e[0m" + end + end +end + +def wcwidth(wc) + return 8 if wc == "\t" + n = wc.ord + if n < 0x20 + 0 + elsif n < 0x80 + 1 + else + 2 + end +end + +def fold(str, col) + i = 0 + size = str.size + len = 0 + while i < size + case c = str[i] + when "\r", "\n" + len = 0 + else + d = wcwidth(c) + len += d + if len == col + str.insert(i+1, "\n") + len = 0 + i += 2 + next + elsif len > col + str.insert(i, "\n") + len = d + i += 2 + next + end + end + i += 1 + end + str +end + +class StringScanner + # lx: limit of x (colmns of screen) + # ly: limit of y (rows of screen) + def getrows(lx, ly) + cp1 = charpos + x = 0 + y = 0 + until eos? + case c = getch + when "\r" + x = 0 + when "\n" + x = 0 + y += 1 + when "\t" + x += 8 + when /[\x00-\x7f]/ + # halfwidth + x += 1 + else + # fullwidth + x += 2 + end + + if x > lx + x = 0 + y += 1 + unscan + end + if y >= ly + return string[cp1...charpos] + end + end + string[cp1..-1] + end +end + +def more(sio) + console = IO.console + ly, lx = console.winsize + ly -= 1 + str = sio.string + cls = "\r" + (" " * lx) + "\r" + + ss = StringScanner.new(str) + + rows = ss.getrows(lx, ly) + puts rows + until ss.eos? + print ":" + case c = console.getch + when ' ' + rows = ss.getrows(lx, ly) + puts cls + rows + when 'j', "\r" + rows = ss.getrows(lx, 1) + puts cls + rows + when "q" + print cls + break + else + print "\b" + end + end +end + +class << Readline + def readline(prompt = '') + console = IO.console + console.binmode + ly, lx = console.winsize + if /mswin|mingw/ =~ RUBY_PLATFORM or /^(?:vt\d\d\d|xterm)/i =~ ENV["TERM"] + cls = "\r\e[2K" + else + cls = "\r" << (" " * lx) + end + cls << "\r" << prompt + console.print prompt + console.flush + line = '' + while 1 + case c = console.getch + when "\r", "\n" + puts + HISTORY << line + return line + when "\C-?", "\b" # DEL/BS + print "\b \b" if line.chop! + when "\C-u" + print cls + line.clear + when "\C-d" + return nil if line.empty? + line << c + when "\C-p" + HISTORY.pos -= 1 + line = HISTORY.current + print cls + print line + when "\C-n" + HISTORY.pos += 1 + line = HISTORY.current + print cls + print line + else + if c >= " " + print c + line << c + end + end + end + end + + HISTORY = [] + def HISTORY.<<(val) + HISTORY.push(val) + @pos = self.size + self + end + def HISTORY.pos + @pos ||= 0 + end + def HISTORY.pos=(val) + @pos = val + if @pos < 0 + @pos = -1 + elsif @pos >= self.size + @pos = self.size + end + end + def HISTORY.current + @pos ||= 0 + if @pos < 0 || @pos >= self.size + '' + else + self[@pos] + end + end +end unless defined?(Readline.readline) + +def mergeinfo + `svn propget svn:mergeinfo #{RUBY_REPO_PATH}` +end + +def find_svn_log(pattern) + `svn log --xml --stop-on-copy --search="#{pattern}" #{RUBY_REPO_PATH}` +end + +def show_last_journal(http, uri) + res = http.get("#{uri.path}?include=journals") + res.value + h = JSON(res.body) + x = h["issue"] + raise "no issue" unless x + x = x["journals"] + raise "no journals" unless x + x = x.last + puts "== #{x["user"]["name"]} (#{x["created_on"]})" + x["details"].each do |y| + puts JSON(y) + end + puts x["notes"] +end + +def merger_path + RUBY_PLATFORM =~ /mswin|mingw/ ? 'merger' : File.expand_path('../merger.rb', __FILE__) +end + +def backport_command_string + unless @changesets.respond_to?(:validated) + @changesets = @changesets.select do |c| + begin + uri = URI("#{REDMINE_BASE}/projects/ruby-trunk/repository/revisions/#{c}") + uri.read($openuri_options) + true + rescue + false + end + end + @changesets.define_singleton_method(:validated){true} + end + " #{merger_path} --ticket=#{@issue} #{@changesets.sort.join(',')}" +end + +def status_char(obj) + case obj["name"] + when "Closed" + "C".color(bold: true) + else + obj["name"][0] + end +end + +console = IO.console +row, col = console.winsize +@query['limit'] = row - 2 +puts "Backporter #{VERSION}".color(bold: true) + " for #{TARGET_VERSION}" + +class CommandSyntaxError < RuntimeError; end +commands = { + "ls" => proc{|args| + raise CommandSyntaxError unless /\A(\d+)?\z/ =~ args + uri = URI(REDMINE_BASE+'/projects/ruby-trunk/issues.json?'+URI.encode_www_form(@query.dup.merge('page' => ($1 ? $1.to_i : 1)))) + # puts uri + res = JSON(uri.read($openuri_options)) + @issues = issues = res["issues"] + from = res["offset"] + 1 + total = res["total_count"] + to = from + issues.size - 1 + puts "#{from}-#{to} / #{total}" + issues.each_with_index do |x, i| + id = "##{x["id"]}".color(*PRIORITIES[x["priority"]["name"]]) + puts "#{'%2d' % i} #{id} #{x["priority"]["name"][0]} #{status_char(x["status"])} #{x["subject"][0,80]}" + end + }, + + "show" => proc{|args| + if /\A(\d+)\z/ =~ args + id = $1.to_i + id = @issues[id]["id"] if @issues && id < @issues.size + @issue = id + elsif @issue + id = @issue + else + raise CommandSyntaxError + end + uri = "#{REDMINE_BASE}/issues/#{id}" + uri = URI(uri+".json?include=children,attachments,relations,changesets,journals") + res = JSON(uri.read($openuri_options)) + i = res["issue"] + unless i["changesets"] + abort "You don't have view_changesets permission" + end + unless i["custom_fields"] + puts "The specified ticket \##{@issue} seems to be a feature ticket" + @issue = nil + next + end + id = "##{i["id"]}".color(*PRIORITIES[i["priority"]["name"]]) + sio = StringIO.new + sio.puts <<eom +#{i["subject"].color(bold: true, underscore: true)} +#{i["project"]["name"]} [#{i["tracker"]["name"]} #{id}] #{i["status"]["name"]} (#{i["created_on"]}) +author: #{i["author"]["name"]} +assigned: #{i["assigned_to"].to_h["name"]} +eom + i["custom_fields"].each do |x| + sio.puts "%-10s: %s" % [x["name"], x["value"]] + end + #res["attachements"].each do |x| + #end + sio.puts i["description"] + sio.puts + sio.puts "= changesets".color(bold: true, underscore: true) + @changesets = [] + i["changesets"].each do |x| + @changesets << x["revision"] + sio.puts "== #{x["revision"]} #{x["committed_on"]} #{x["user"]["name"] rescue nil}".color(bold: true, underscore: true) + sio.puts x["comments"] + end + @changesets = @changesets.sort.uniq + if i["journals"] && !i["journals"].empty? + sio.puts "= journals".color(bold: true, underscore: true) + i["journals"].each do |x| + sio.puts "== #{x["user"]["name"]} (#{x["created_on"]})".color(bold: true, underscore: true) + x["details"].each do |y| + sio.puts JSON(y) + end + sio.puts x["notes"] + end + end + more(sio) + }, + + "rel" => proc{|args| + # this feature requires custom redmine which allows add_related_issue API + raise CommandSyntaxError unless /\Ar?(\d+)\z/ =~ args + unless @issue + puts "ticket not selected" + next + end + rev = $1 + uri = URI("#{REDMINE_BASE}/projects/ruby-trunk/repository/revisions/#{rev}/issues.json") + Net::HTTP.start(uri.host, uri.port, http_options) do |http| + res = http.post(uri.path, "issue_id=#@issue", + 'X-Redmine-API-Key' => REDMINE_API_KEY) + begin + res.value + rescue + if $!.respond_to?(:response) && $!.response.is_a?(Net::HTTPConflict) + $stderr.puts "the revision has already related to the ticket" + else + $stderr.puts "deployed redmine doesn't have https://github.com/ruby/bugs.ruby-lang.org/commit/01fbba60d68cb916ddbccc8a8710e68c5217171d\nask naruse or hsbt" + end + next + end + puts res.body + @changesets << rev + class << @changesets + remove_method(:validated) rescue nil + end + end + }, + + "backport" => proc{|args| + # this feature implies backport command which wraps tool/merger.rb + raise CommandSyntaxError unless args.empty? + unless @issue + puts "ticket not selected" + next + end + puts backport_command_string + }, + + "done" => proc{|args| + raise CommandSyntaxError unless /\A(\d+)?(?:\s*-- +(.*))?\z/ =~ args + notes = $2 + notes.strip! if notes + if $1 + i = $1.to_i + i = @issues[i]["id"] if @issues && i < @issues.size + @issue = i + end + unless @issue + puts "ticket not selected" + next + end + + log = find_svn_log("##@issue]") + if log && /revision="(?<rev>\d+)/ =~ log + str = log[/merge revision\(s\) ([^:]+)(?=:)/] + str.insert(5, "d") + str = "ruby_#{TARGET_VERSION.tr('.','_')} r#{rev} #{str}." + if notes + str << "\n" + str << notes + end + notes = str + else + puts "no commit is found whose log include ##@issue" + next + end + puts notes + + uri = URI("#{REDMINE_BASE}/issues/#{@issue}.json") + Net::HTTP.start(uri.host, uri.port, http_options) do |http| + res = http.get(uri.path) + data = JSON(res.body) + h = data["issue"]["custom_fields"].find{|x|x["id"]==5} + if h and val = h["value"] and val != "" + case val[/(?:\A|, )#{Regexp.quote TARGET_VERSION}: ([^,]+)/, 1] + when 'REQUIRED', 'UNKNOWN', 'DONTNEED', 'WONTFIX' + val[$~.offset(1)[0]...$~.offset(1)[1]] = 'DONE' + when 'DONE' # , /\A\d+\z/ + puts 'already backport is done' + next # already done + when nil + val << ", #{TARGET_VERSION}: DONE" + else + raise "unknown status '#$1'" + end + else + val = "#{TARGET_VERSION}: DONE" + end + + data = { "issue" => { "custom_fields" => [ {"id"=>5, "value" => val} ] } } + data['issue']['notes'] = notes if notes + res = http.put(uri.path, JSON(data), + 'X-Redmine-API-Key' => REDMINE_API_KEY, + 'Content-Type' => 'application/json') + res.value + + show_last_journal(http, uri) + end + }, + + "close" => proc{|args| + raise CommandSyntaxError unless /\A(\d+)?\z/ =~ args + if $1 + i = $1.to_i + i = @issues[i]["id"] if @issues && i < @issues.size + @issue = i + end + unless @issue + puts "ticket not selected" + next + end + + uri = URI("#{REDMINE_BASE}/issues/#{@issue}.json") + Net::HTTP.start(uri.host, uri.port, http_options) do |http| + data = { "issue" => { "status_id" => STATUS_CLOSE } } + res = http.put(uri.path, JSON(data), + 'X-Redmine-API-Key' => REDMINE_API_KEY, + 'Content-Type' => 'application/json') + res.value + + show_last_journal(http, uri) + end + }, + + "last" => proc{|args| + raise CommandSyntaxError unless /\A(\d+)?\z/ =~ args + if $1 + i = $1.to_i + i = @issues[i]["id"] if @issues && i < @issues.size + @issue = i + end + unless @issue + puts "ticket not selected" + next + end + + uri = URI("#{REDMINE_BASE}/issues/#{@issue}.json") + Net::HTTP.start(uri.host, uri.port, http_options) do |http| + show_last_journal(http, uri) + end + }, + + "!" => proc{|args| + system(args.strip) + }, + + "quit" => proc{|args| + raise CommandSyntaxError unless args.empty? + exit + }, + "exit" => "quit", + + "help" => proc{|args| + puts 'ls [PAGE] '.color(bold: true) + ' show all required tickets' + puts '[show] TICKET '.color(bold: true) + ' show the detail of the TICKET, and select it' + puts 'backport '.color(bold: true) + ' show the option of selected ticket for merger.rb' + puts 'rel REVISION '.color(bold: true) + ' add the selected ticket as related to the REVISION' + puts 'done [TICKET] [-- NOTE]'.color(bold: true) + ' set Backport field of the TICKET to DONE' + puts 'close [TICKET] '.color(bold: true) + ' close the TICKET' + puts 'last [TICKET] '.color(bold: true) + ' show the last journal of the TICKET' + puts '! COMMAND '.color(bold: true) + ' execute COMMAND' + } +} +list = Abbrev.abbrev(commands.keys) + +@issues = nil +@issue = nil +@changesets = nil +while true + begin + l = Readline.readline "#{('#' + @issue.to_s).color(bold: true) if @issue}> " + rescue Interrupt + break + end + break unless l + cmd, args = l.strip.split(/\s+|\b/, 2) + next unless cmd + if (!args || args.empty?) && /\A\d+\z/ =~ cmd + args = cmd + cmd = "show" + end + cmd = list[cmd] + if commands[cmd].is_a? String + cmd = list[commands[cmd]] + end + begin + if cmd + commands[cmd].call(args) + else + raise CommandSyntaxError + end + rescue CommandSyntaxError + puts "error #{l.inspect}" + end +end diff --git a/tool/release.sh b/tool/release.sh new file mode 100755 index 0000000000..04c361a48a --- /dev/null +++ b/tool/release.sh @@ -0,0 +1,38 @@ +#!/bin/sh + +RUBYDIR=/home/ftp/pub/ruby +EXTS='.tar.gz .tar.bz2 .tar.xz .zip' + +releases=`ls ruby-*|grep -o 'ruby-[0-9]\{1,\}\.[0-9]\{1,\}\.[0-9]\{1,\}\(-\(preview\|rc\|p\)[0-9]\{1,4\}\)\?'|uniq` + +# check files +for r in $releases +do + echo "checking files for $r..." + for ext in $EXTS + do + if ! [ -f $r$ext ];then + echo "ERROR: $r$ext not found" + exit 1 + fi + done + echo "files are ok" +done + +# version directory +for r in $releases +do + xy=`echo $r|grep -o '[0-9]\.[0-9]'` + preview=`echo $r|grep -o -- '-\(preview\|rc\)'` + dir="${RUBYDIR}/$xy" + echo "$dir" + mkdir -p $dir + for ext in $EXTS + do + cp $r$ext $dir/$r$ext + ln -sf $xy/$r$ext ${RUBYDIR}/$r$ext + if [ x$preview = x ];then + ln -sf $xy/$r$ext ${RUBYDIR}/ruby-$xy-stable$ext + fi + done +done diff --git a/tool/rmdirs b/tool/rmdirs new file mode 100755 index 0000000000..76c4a39cb1 --- /dev/null +++ b/tool/rmdirs @@ -0,0 +1,14 @@ +#!/bin/sh + +# Script used by configure to delete directories recursively. + +for dir do + while rmdir "$dir" >/dev/null 2>&1 && + parent=`expr "$dir" : '\(.*\)/[^/][^/]*'`; do + case "$parent" in + . | .. | "$dir") break;; + *) dir="$parent";; + esac + done +done +true diff --git a/tool/run-gcov.rb b/tool/run-gcov.rb new file mode 100644 index 0000000000..5df7622aa3 --- /dev/null +++ b/tool/run-gcov.rb @@ -0,0 +1,54 @@ +#!ruby +require "pathname" +require "open3" + +Pathname.glob("**/*.gcda").sort.each do |gcda| + if gcda.fnmatch("ext/*") + cwd, gcda = gcda.split.map {|s| s.to_s } + objdir = "." + elsif gcda.fnmatch("rubyspec_temp/*") + next + else + cwd, objdir, gcda = ".", gcda.dirname.to_s, gcda.to_s + end + puts "$ gcov -lpbc -o #{ objdir } #{ gcda }" + out, err, _status = Open3.capture3("gcov", "-lpbc", "-o", objdir, gcda, chdir: cwd) + puts out + puts err + + # a black list of source files that contains wrong #line directives + if err !~ %r( + \A( + Cannot\ open\ source\ file\ ( + defs/keywords + |zonetab\.list + |enc/jis/props\.kwd + |parser\.c + |parser\.rl + )\n + )*\z + )x + raise "Unexpected gcov output" + end + + if out !~ %r( + \A( + File\ .*\nLines\ executed:.*\n + ( + Branches\ executed:.*\n + Taken\ at\ least\ once:.*\n + | + No\ branches\n + )? + ( + Calls\ executed:.*\n + | + No\ calls\n + )? + Creating\ .*\n + \n + )+\z + )x + raise "Unexpected gcov output" + end +end diff --git a/tool/run-lcov.rb b/tool/run-lcov.rb new file mode 100644 index 0000000000..f27578200a --- /dev/null +++ b/tool/run-lcov.rb @@ -0,0 +1,164 @@ +#!ruby +require "pathname" +require "open3" +require "tmpdir" + +def backup_gcda_files(gcda_files) + gcda_files = gcda_files.map do |gcda| + [gcda, gcda.sub_ext(".bak")] + end + begin + gcda_files.each do |before, after| + before.rename(after) + end + yield + ensure + gcda_files.each do |before, after| + after.rename(before) + end + end +end + +def run_lcov(*args) + system("lcov", "--rc", "lcov_branch_coverage=1", *args) +end + +$info_files = [] +def run_lcov_capture(dir, info) + $info_files << info + run_lcov("--capture", "-d", dir, "-o", info) +end + +def run_lcov_merge(files, info) + run_lcov(*files.flat_map {|f| ["--add-tracefile", f] }, "-o", info) +end + +def run_lcov_remove(info_src, info_out) + dirs = %w(/usr/*) + dirs << File.join(Dir.tmpdir, "*") + %w( + test/* + ext/-test-/* + ext/nkf/nkf-utf8/nkf.c + ).each {|f| dirs << File.join(File.dirname(__dir__), f) } + run_lcov("--remove", info_src, *dirs, "-o", info_out) +end + +def run_genhtml(info, out) + system("genhtml", "--branch-coverage", "--ignore-errors", "source", info, "-o", out) +end + +def gen_rb_lcov(file) + res = Marshal.load(File.binread(file)) + + open("lcov-rb-all.info", "w") do |f| + f.puts "TN:" # no test name + base_dir = File.dirname(__dir__) + res.each do |path, cov| + next unless path.start_with?(base_dir) + next if path.start_with?(File.join(base_dir, "test")) + f.puts "SF:#{ path }" + + total = covered = 0 + cov.each_with_index do |count, lineno| + next unless count + f.puts "DA:#{ lineno + 1 },#{ count }" + total += 1 + covered += 1 if count > 0 + end + f.puts "LF:#{ total }" + f.puts "LH:#{ covered }" + + f.puts "end_of_record" + end + end +end + +def gen_rb_lcov(file) + res = Marshal.load(File.binread(file)) + + open("lcov-rb-all.info", "w") do |f| + f.puts "TN:" # no test name + base_dir = File.dirname(File.dirname(__dir__)) + res.each do |path, cov| + next unless path.start_with?(base_dir) + next if path.start_with?(File.join(base_dir, "test")) + f.puts "SF:#{ path }" + + # function coverage + total = covered = 0 + cov[:methods].each do |(klass, name, lineno), count| + f.puts "FN:#{ lineno },#{ klass }##{ name }" + total += 1 + covered += 1 if count > 0 + end + f.puts "FNF:#{ total }" + f.puts "FNF:#{ covered }" + cov[:methods].each do |(klass, name, _), count| + f.puts "FNDA:#{ count },#{ klass }##{ name }" + end + + # line coverage + total = covered = 0 + cov[:lines].each_with_index do |count, lineno| + next unless count + f.puts "DA:#{ lineno + 1 },#{ count }" + total += 1 + covered += 1 if count > 0 + end + f.puts "LF:#{ total }" + f.puts "LH:#{ covered }" + + # branch coverage + total = covered = 0 + id = 0 + cov[:branches].each do |(_base_type, _, base_lineno), targets| + i = 0 + targets.each do |(_target_type, _target_lineno), count| + f.puts "BRDA:#{ base_lineno },#{ id },#{ i },#{ count }" + total += 1 + covered += 1 if count > 0 + i += 1 + end + id += 1 + end + f.puts "BRF:#{ total }" + f.puts "BRH:#{ covered }" + f.puts "end_of_record" + end + end +end + +gcda_files = Pathname.glob("**/*.gcda") +ext_gcda_files = gcda_files.select {|f| f.fnmatch("ext/*") } +rubyspec_temp_gcda_files = gcda_files.select {|f| f.fnmatch("rubyspec_temp/*") } + +backup_gcda_files(rubyspec_temp_gcda_files) do + if ext_gcda_files != [] + backup_gcda_files(ext_gcda_files) do + info = "lcov-root.info" + run_lcov_capture(".", info) + end + end + ext_gcda_files.group_by {|f| f.descend.to_a[1] }.each do |key, files| + info = "lcov-#{ key.to_s.gsub(File::Separator, "-") }.info" + run_lcov_capture(key.to_s, info) + end +end +if $info_files != [] + run_lcov_merge($info_files, "lcov-c-all.info") + run_lcov_remove("lcov-c-all.info", "lcov-c-all-filtered.info") + run_genhtml("lcov-c-all-filtered.info", "lcov-c-out") +end + +if File.readable?("test-coverage.dat") + gen_rb_lcov("test-coverage.dat") + run_lcov_remove("lcov-rb-all.info", "lcov-rb-all-filtered.info") + run_genhtml("lcov-rb-all-filtered.info", "lcov-rb-out") +end + +if File.readable?("lcov-c-all.info") && File.readable?("lcov-rb-all.info") + run_lcov_merge(%w(lcov-c-all.info lcov-rb-all.info), "lcov-all.info") + run_lcov_remove("lcov-all.info", "lcov-all-filtered.info") + run_genhtml("lcov-all-filtered.info", "lcov-out") +end diff --git a/tool/runruby.rb b/tool/runruby.rb new file mode 100755 index 0000000000..a5aa3e7d8d --- /dev/null +++ b/tool/runruby.rb @@ -0,0 +1,159 @@ +#!./miniruby + +# Used by "make runruby", configure, and by hand to run a locally-built Ruby +# with correct environment variables and arguments. + +show = false +precommand = [] +srcdir = File.realpath('..', File.dirname(__FILE__)) +while arg = ARGV[0] + break ARGV.shift if arg == '--' + case arg + when '-C', /\A-C(.+)/m + ARGV.shift + Dir.chdir($1 || ARGV.shift) + next + end + /\A--([-\w]+)(?:=(.*))?\z/ =~ arg or break + arg, value = $1, $2 + re = Regexp.new('\A'+arg.gsub(/\w+\b/, '\&\\w*')+'\z', "i") + case + when re =~ "srcdir" + srcdir = value + when re =~ "archdir" + archdir = value + when re =~ "cpu" + precommand << "arch" << "-arch" << value + when re =~ "extout" + extout = value + when re =~ "pure" + # obsolete switch do nothing + when re =~ "debugger" + require 'shellwords' + case value + when nil + debugger = :gdb + when "no" + else + debugger = Shellwords.shellwords(value) + end and precommand |= [:debugger] + when re =~ "precommand" + require 'shellwords' + precommand.concat(Shellwords.shellwords(value)) + when re =~ "show" + show = true + when re =~ "chdir" + Dir.chdir(value) + else + break + end + ARGV.shift +end + +unless defined?(File.realpath) + def File.realpath(*args) + path = expand_path(*args) + if File.stat(path).directory? + Dir.chdir(path) {Dir.pwd} + else + dir, base = File.split(path) + File.join(Dir.chdir(dir) {Dir.pwd}, base) + end + end +end + +begin + conffile = File.realpath('rbconfig.rb', archdir) +rescue Errno::ENOENT => e + # retry if !archdir and ARGV[0] and File.directory?(archdir = ARGV.shift) + abort "#$0: rbconfig.rb not found, use --archdir option" +end + +abs_archdir = File.dirname(conffile) +archdir ||= abs_archdir +$:.unshift(abs_archdir) + +config = File.read(conffile) +config.sub!(/^(\s*)RUBY_VERSION\b.*(\sor\s*)\n.*\n/, '') +config = Module.new {module_eval(config, conffile)}::RbConfig::CONFIG + +ruby = File.join(archdir, config["RUBY_INSTALL_NAME"]+config['EXEEXT']) +unless File.exist?(ruby) + abort "#{ruby} is not found.\nTry `make' first, then `make test', please.\n" +end + +libs = [abs_archdir] +extout ||= config["EXTOUT"] +if extout + abs_extout = File.expand_path(extout, abs_archdir) + libs << File.expand_path("common", abs_extout) << File.expand_path(config['arch'], abs_extout) +end +libs << File.expand_path("lib", srcdir) +config["bindir"] = abs_archdir + +env = { + # Test with the smallest possible machine stack sizes. + # These values are clamped to machine-dependent minimum values in vm_core.h + 'RUBY_THREAD_MACHINE_STACK_SIZE' => '1', + 'RUBY_FIBER_MACHINE_STACK_SIZE' => '1', +} + +runner = File.join(abs_archdir, "exe/ruby#{config['EXEEXT']}") +runner = nil unless File.exist?(runner) +abs_ruby = runner || File.expand_path(ruby) +env["RUBY"] = abs_ruby +env["GEM_PATH"] = env["GEM_HOME"] = File.expand_path(".bundle", srcdir) +env["BUNDLE_RUBY"] = abs_ruby +env["BUNDLE_GEM"] = "#{abs_ruby} -rrubygems #{srcdir}/bin/gem --backtrace" +env["PATH"] = [File.dirname(abs_ruby), abs_archdir, ENV["PATH"]].compact.join(File::PATH_SEPARATOR) + +if e = ENV["RUBYLIB"] + libs |= e.split(File::PATH_SEPARATOR) +end +env["RUBYLIB"] = $:.replace(libs).join(File::PATH_SEPARATOR) + +libruby_so = File.join(abs_archdir, config['LIBRUBY_SO']) +if File.file?(libruby_so) + if e = config['LIBPATHENV'] and !e.empty? + env[e] = [abs_archdir, ENV[e]].compact.join(File::PATH_SEPARATOR) + end + unless runner + if e = config['PRELOADENV'] + e = nil if e.empty? + e ||= "LD_PRELOAD" if /linux/ =~ RUBY_PLATFORM + end + if e + env[e] = [libruby_so, ENV[e]].compact.join(File::PATH_SEPARATOR) + end + end +end + +ENV.update env + +if debugger or ENV['RUNRUBY_USE_GDB'] == 'true' + if debugger == :gdb or !debugger + debugger = %w'gdb' + if File.exist?(gdb = 'run.gdb') or + File.exist?(gdb = File.join(abs_archdir, 'run.gdb')) + debugger.push('-x', gdb) + end + debugger << '--args' + end + if idx = precommand.index(:debugger) + precommand[idx, 1] = debugger + else + precommand.concat(debugger) + end +end + +cmd = [runner || ruby] +cmd.concat(ARGV) +cmd.unshift(*precommand) unless precommand.empty? + +if show + require 'shellwords' + env.each {|k,v| puts "#{k}=#{v}"} + puts Shellwords.join(cmd) +end + +exec(*cmd, close_others: false) diff --git a/tool/strip-rdoc.rb b/tool/strip-rdoc.rb new file mode 100755 index 0000000000..0ac9c39323 --- /dev/null +++ b/tool/strip-rdoc.rb @@ -0,0 +1,26 @@ +#!ruby + +# Filter for preventing Doxygen from processing RDoc comments. +# Used by the Doxygen template. + +ARGF.binmode +source = ARGF.read +source = source.gsub(%r{/\*([!*])((?!\*/).+?)\*/}m) do |comment| + marker, comment = $1, $2 + next "/**#{comment}*/" unless /^\s*\*\s?\-\-\s*$/ =~ comment + doxybody = nil + comment.each_line do |line| + if doxybody + if /^\s*\*\s?\+\+\s*$/ =~ line + break + end + doxybody << line + else + if /^\s*\*\s?--\s*$/ =~ line + doxybody = "\n" + end + end + end + "/*#{marker}#{doxybody}*/" +end +print source diff --git a/tool/sync_default_gems.rb b/tool/sync_default_gems.rb new file mode 100644 index 0000000000..b12b7a792a --- /dev/null +++ b/tool/sync_default_gems.rb @@ -0,0 +1,198 @@ +# sync following repositories to ruby repository +# +# * https://github.com/rubygems/rubygems +# * https://github.com/ruby/rdoc +# * https://github.com/flori/json +# * https://github.com/ruby/psych +# * https://github.com/ruby/fileutils +# * https://github.com/ruby/fiddle +# * https://github.com/ruby/stringio +# * https://github.com/ruby/io-console +# * https://github.com/ruby/csv +# * https://github.com/ruby/webrick +# * https://github.com/ruby/dbm +# * https://github.com/ruby/gdbm +# * https://github.com/ruby/sdbm +# * https://github.com/ruby/etc +# * https://github.com/ruby/date +# * https://github.com/ruby/zlib +# * https://github.com/ruby/fcntl +# * https://github.com/ruby/scanf +# * https://github.com/ruby/cmath +# * https://github.com/ruby/strscan +# * https://github.com/ruby/ipaddr +# + +$repositories = { + rubygems: 'rubygems/rubygems', + rdoc: 'ruby/rdoc', + json: 'flori/json', + psych: 'ruby/psych', + fileutils: 'ruby/fileutils', + fiddle: 'ruby/fiddle', + stringio: 'ruby/stringio', + ioconsole: 'ruby/io-console', + csv: 'ruby/csv', + webrick: 'ruby/webrick', + dbm: 'ruby/dbm', + gdbm: 'ruby/gdbm', + sdbm: 'ruby/sdbm', + etc: 'ruby/etc', + date: 'ruby/date', + zlib: 'ruby/zlib', + fcntl: 'ruby/fcntl', + scanf: 'ruby/scanf', + cmath: 'ruby/cmath', + strscan: 'ruby/strscan', + ipaddr: 'ruby/ipaddr', +} + +def sync_default_gems(gem) + author, repository = $repositories[gem.to_sym].split('/') + unless File.exist?("../../#{author}/#{repository}") + `mkdir -p ../../#{author}` + `git clone git@github.com:#{author}/#{repository}.git ../../#{author}/#{repository}` + end + + puts "Sync #{$repositories[gem.to_sym]}" + + case gem + when "rubygems" + `rm -rf lib/rubygems* test/rubygems` + `cp -r ../../rubygems/rubygems/lib/rubygems* ./lib` + `cp -r ../../rubygems/rubygems/test/rubygems ./test` + when "rdoc" + `rm -rf lib/rdoc* test/rdoc` + `cp -rf ../rdoc/lib/rdoc* ./lib` + `cp -rf ../rdoc/test test/rdoc` + `cp ../rdoc/rdoc.gemspec ./lib/rdoc` + `rm -f lib/rdoc/markdown.kpeg lib/rdoc/markdown/literals.kpeg lib/rdoc/rd/block_parser.ry lib/rdoc/rd/inline_parser.ry` + `git checkout lib/rdoc/.document` + when "json" + `rm -rf ext/json test/json` + `cp -rf ../../flori/json/ext/json/ext ext/json` + `cp -rf ../../flori/json/tests test/json` + `cp -rf ../../flori/json/lib ext/json` + `rm -rf ext/json/lib/json/pure*` + `cp ../../flori/json/json.gemspec ext/json` + `rm -r ext/json/lib/json/ext` + `git checkout ext/json/extconf.rb ext/json/parser/prereq.mk ext/json/generator/depend ext/json/parser/depend` + when "psych" + `rm -rf ext/psych test/psych` + `cp -rf ../psych/ext/psych ./ext` + `cp -rf ../psych/lib ./ext/psych` + `cp -rf ../psych/test/psych ./test` + `rm -rf ext/psych/lib/org ext/psych/lib/psych.jar ext/psych/lib/psych_jars.rb` + `rm -rf ext/psych/lib/psych.{bundle,so} ext/psych/lib/{2.0,2.1,2.2,2.3,2.4}` + `rm -f ext/psych/yaml/LICENSE` + `cp ../psych/psych.gemspec ext/psych/` + `git checkout ext/psych/depend` + when "fileutils" + `rm -rf lib/fileutils.rb test/fileutils lib/fileutils.gemspec` + `cp -rf ../fileutils/lib/* lib` + `cp -rf ../fileutils/test/fileutils test` + `cp -f ../fileutils/fileutils.gemspec lib` + when "fiddle" + `rm -rf ext/fiddle test/fiddle` + `cp -rf ../fiddle/ext/fiddle ext` + `cp -rf ../fiddle/lib ext/fiddle` + `cp -rf ../fiddle/test/fiddle test` + `cp -f ../fiddle/fiddle.gemspec ext/fiddle` + `git checkout ext/fiddle/depend` + when "stringio" + `rm -rf ext/stringio test/stringio` + `cp -rf ../stringio/ext/stringio ext` + `cp -rf ../stringio/test/stringio test` + `cp -f ../stringio/stringio.gemspec ext/stringio` + `git checkout ext/stringio/depend ext/stringio/README.md` + when "ioconsole" + `rm -rf ext/io/console test/io/console` + `cp -rf ../io-console/ext/io/console ext/io` + `cp -rf ../io-console/test/io/console test/io` + `mkdir -p ext/io/console/lib` + `cp -rf ../io-console/lib/console ext/io/console/lib` + `cp -f ../io-console/io-console.gemspec ext/io/console` + `git checkout ext/io/console/depend` + when "csv" + `rm -rf lib/csv.rb test/csv lib/csv.gemspec` + `cp -rf ../csv/lib/* lib` + `cp -rf ../csv/test/csv test` + `cp -f ../csv/csv.gemspec lib` + when "webrick" + `rm -rf lib/webrick test/webrick` + `cp -rf ../webrick/lib/webrick lib` + `cp -rf ../webrick/test/webrick test` + `cp -f ../webrick/webrick.gemspec lib/webrick` + when "dbm" + `rm -rf ext/dbm test/dbm` + `cp -rf ../dbm/ext/dbm ext` + `cp -rf ../dbm/test/dbm test` + `cp -f ../dbm/dbm.gemspec ext/dbm` + `git checkout ext/dbm/depend` + when "gdbm" + `rm -rf ext/gdbm test/gdbm` + `cp -rf ../gdbm/ext/gdbm ext` + `cp -rf ../gdbm/test/gdbm test` + `cp -f ../gdbm/gdbm.gemspec ext/gdbm` + `git checkout ext/gdbm/depend ext/gdbm/README` + when "sdbm" + `rm -rf ext/sdbm test/sdbm` + `cp -rf ../sdbm/ext/sdbm ext` + `cp -rf ../sdbm/test/sdbm test` + `cp -f ../sdbm/sdbm.gemspec ext/sdbm` + `git checkout ext/sdbm/depend` + when "etc" + `rm -rf ext/etc test/etc` + `cp -rf ../etc/ext/etc ext` + `cp -rf ../etc/test/etc test` + `cp -f ../etc/etc.gemspec ext/etc` + `git checkout ext/etc/depend` + when "date" + `rm -rf ext/date test/date` + `cp -rf ../date/ext/date ext` + `cp -rf ../date/lib ext/date` + `cp -rf ../date/test/date test` + `cp -f ../date/date.gemspec ext/date` + `git checkout ext/date/depend` + when "zlib" + `rm -rf ext/zlib test/zlib` + `cp -rf ../zlib/ext/zlib ext` + `cp -rf ../zlib/test/zlib test` + `cp -f ../zlib/zlib.gemspec ext/zlib` + `git checkout ext/zlib/depend` + when "fcntl" + `rm -rf ext/fcntl` + `cp -rf ../fcntl/ext/fcntl ext` + `cp -f ../fcntl/fcntl.gemspec ext/fcntl` + `git checkout ext/fcntl/depend` + when "scanf" + `rm -rf lib/scanf.rb test/scanf` + `cp -rf ../scanf/lib/* lib` + `cp -rf ../scanf/test/scanf test` + `cp -f ../scanf/scanf.gemspec lib` + when "cmath" + `rm -rf lib/cmath.rb test/test_cmath.rb` + `cp -rf ../cmath/lib/* lib` + `cp -rf ../cmath/test/test_cmath.rb test` + `cp -f ../cmath/cmath.gemspec lib` + when "strscan" + `rm -rf ext/strscan test/strscan` + `cp -rf ../strscan/ext/strscan ext` + `cp -rf ../strscan/test/strscan test` + `cp -f ../strscan/strscan.gemspec ext/strscan` + `rm -f ext/strscan/regenc.h ext/strscan/regint.h` + `git checkout ext/strscan/depend` + when "ipaddr" + `rm -rf lib/ipaddr.rb test/test_ipaddr.rb` + `cp -rf ../ipaddr/lib/* lib` + `cp -rf ../ipaddr/test/test_ipaddr.rb test` + `cp -f ../ipaddr/ipaddr.gemspec lib` + else + end +end + +if ARGV[0] + sync_default_gems(ARGV[0]) +else + $repositories.keys.each{|gem| sync_default_gems(gem.to_s)} +end diff --git a/tool/test-coverage.rb b/tool/test-coverage.rb new file mode 100644 index 0000000000..7a27c5cfb4 --- /dev/null +++ b/tool/test-coverage.rb @@ -0,0 +1,103 @@ +require "coverage" + +Coverage.start(lines: true, branches: true, methods: true) + +TEST_COVERAGE_DATA_FILE = "test-coverage.dat" + +def merge_coverage_data(res1, res2) + res1.each do |path, cov1| + cov2 = res2[path] + if cov2 + cov1[:lines].each_with_index do |count1, i| + next unless count1 + add_count(cov2[:lines], i, count1) + end + cov1[:branches].each do |base_key, targets1| + if cov2[:branches][base_key] + targets1.each do |target_key, count1| + add_count(cov2[:branches][base_key], target_key, count1) + end + else + cov2[:branches][base_key] = targets1 + end + end + cov1[:methods].each do |key, count1| + add_count(cov2[:methods], key, count1) + end + else + res2[path] = cov1 + end + end + res2 +end + +def add_count(h, key, count) + if h[key] + h[key] += count + else + h[key] = count + end +end + +def save_coverage_data(res1) + res1.each do |_path, cov| + if cov[:methods] + h = {} + cov[:methods].each do |(klass, *key), count| + h[[klass.inspect, *key]] = count + end + cov[:methods].replace h + end + end + File.open(TEST_COVERAGE_DATA_FILE, File::RDWR | File::CREAT | File::BINARY) do |f| + f.flock(File::LOCK_EX) + s = f.read + res2 = s.size > 0 ? Marshal.load(s) : {} + res1 = merge_coverage_data(res1, res2) + f.rewind + f << Marshal.dump(res2) + f.flush + f.truncate(f.pos) + end +end + +def invoke_simplecov_formatter + %w[doclie simplecov-html simplecov].each do |f| + $LOAD_PATH.unshift "#{__dir__}/../coverage/#{f}/lib" + end + + require "simplecov" + res = Marshal.load(File.binread(TEST_COVERAGE_DATA_FILE)) + simplecov_result = {} + base_dir = File.dirname(__dir__) + + res.each do |path, cov| + next unless path.start_with?(base_dir) + next if path.start_with?(File.join(base_dir, "test")) + simplecov_result[path] = cov[:lines] + end + + res = SimpleCov::Result.new(simplecov_result) + res.command_name = "Ruby's `make test-all`" + SimpleCov::Formatter::HTMLFormatter.new.format(res) +end + +pid = $$ +pwd = Dir.pwd + +at_exit do + exit_exc = $! + + Dir.chdir(pwd) do + save_coverage_data(Coverage.result) + if pid == $$ + begin + nil while Process.waitpid(-1) + rescue Errno::ECHILD + invoke_simplecov_formatter + end + end + end + + raise exit_exc if exit_exc +end diff --git a/tool/test/test_jisx0208.rb b/tool/test/test_jisx0208.rb new file mode 100644 index 0000000000..d323c84745 --- /dev/null +++ b/tool/test/test_jisx0208.rb @@ -0,0 +1,40 @@ +require 'test/unit' + +require '../tool/jisx0208' + +class Test_JISX0208_Char < Test::Unit::TestCase + def test_create_with_row_cell + assert_equal JISX0208::Char.new(0x2121), JISX0208::Char.new(1, 1) + end + + def test_succ + assert_equal JISX0208::Char.new(0x2221), JISX0208::Char.new(0x217e).succ + assert_equal JISX0208::Char.new(2, 1), JISX0208::Char.new(1, 94).succ + assert_equal JISX0208::Char.new(0x7f21), JISX0208::Char.new(0x7e7e).succ + end + + def test_to_shift_jis + assert_equal 0x895C, JISX0208::Char.new(0x313D).to_sjis + assert_equal 0x895C, JISX0208::Char.from_sjis(0x895C).to_sjis + assert_equal 0xF3DE, JISX0208::Char.from_sjis(0xF3DE).to_sjis + assert_equal 0xFC40, JISX0208::Char.from_sjis(0xFC40).to_sjis + end + + def test_from_sjis + assert_raise(ArgumentError) { JISX0208::Char.from_sjis(-1) } + assert_raise(ArgumentError) { JISX0208::Char.from_sjis(0x10000) } + assert_nothing_raised { JISX0208::Char.from_sjis(0x8140) } + assert_nothing_raised { JISX0208::Char.from_sjis(0xFCFC) } + assert_equal JISX0208::Char.new(0x313D), JISX0208::Char.from_sjis(0x895C) + end + + def test_row + assert_equal 1, JISX0208::Char.new(0x2121).row + assert_equal 94, JISX0208::Char.new(0x7E7E).row + end + + def test_cell + assert_equal 1, JISX0208::Char.new(0x2121).cell + assert_equal 94, JISX0208::Char.new(0x7E7E).cell + end +end diff --git a/tool/transcode-tblgen.rb b/tool/transcode-tblgen.rb new file mode 100644 index 0000000000..f43627b781 --- /dev/null +++ b/tool/transcode-tblgen.rb @@ -0,0 +1,1114 @@ +# frozen_string_literal: true + +require 'optparse' +require 'erb' +require 'fileutils' +require 'pp' + +class Array + unless [].respond_to? :product + def product(*args) + if args.empty? + self.map {|e| [e] } + else + result = [] + self.each {|e0| + result.concat args.first.product(*args[1..-1]).map {|es| [e0, *es] } + } + result + end + end + end +end + +class String + unless "".respond_to? :start_with? + def start_with?(*prefixes) + prefixes.each {|prefix| + return true if prefix.length <= self.length && prefix == self[0, prefix.length] + } + false + end + end +end + +NUM_ELEM_BYTELOOKUP = 2 + +C_ESC = { + "\\" => "\\\\", + '"' => '\"', + "\n" => '\n', +} + +0x00.upto(0x1f) {|ch| C_ESC[[ch].pack("C")] ||= "\\%03o" % ch } +0x7f.upto(0xff) {|ch| C_ESC[[ch].pack("C")] = "\\%03o" % ch } +C_ESC_PAT = Regexp.union(*C_ESC.keys) + +def c_esc(str) + '"' + str.gsub(C_ESC_PAT) { C_ESC[$&] } + '"' +end + +HEX2 = /(?:[0-9A-Fa-f]{2})/ + +class ArrayCode + def initialize(type, name) + @type = type + @name = name + @len = 0; + @content = ''.dup + end + + def length + @len + end + + def insert_at_last(num, str) + newnum = self.length + num + @content << str + @len += num + end + + def to_s + <<"End" +static const #{@type} +#{@name}[#{@len}] = { +#{@content}}; +End + end +end + +class Action + def initialize(value) + @value = value + end + attr_reader :value + + def hash + @value.hash + end + + def eql?(other) + self.class == other.class && + @value == other.value + end + alias == eql? +end + +class Branch + def initialize(byte_min, byte_max, child_tree) + @byte_min = byte_min + @byte_max = byte_max + @child_tree = child_tree + @hash = byte_min.hash ^ byte_max.hash ^ child_tree.hash + end + attr_reader :byte_min, :byte_max, :child_tree, :hash + + def eql?(other) + self.class == other.class && + @hash == other.hash && + @byte_min == other.byte_min && + @byte_max == other.byte_max && + @child_tree == other.child_tree + end + alias == eql? +end + +class ActionMap + def self.parse_to_rects(mapping) + rects = [] + n = 0 + mapping.each {|pat, action| + pat = pat.to_s + if /\A\s*\(empset\)\s*\z/ =~ pat + next + elsif /\A\s*\(empstr\)\s*\z/ =~ pat + rects << ['', '', action] + n += 1 + elsif /\A\s*(#{HEX2}+)\s*\z/o =~ pat + hex = $1.upcase + rects << [hex, hex, action] + elsif /\A\s*((#{HEX2}|\{#{HEX2}(?:-#{HEX2})?(,#{HEX2}(?:-#{HEX2})?)*\})+(\s+|\z))*\z/o =~ pat + pat = pat.upcase + pat.scan(/\S+/) { + pat1 = $& + ranges_list = [] + pat1.scan(/#{HEX2}|\{([^\}]*)\}/o) { + ranges_list << [] + if !$1 + ranges_list.last << [$&,$&] + else + set = {} + $1.scan(/(#{HEX2})(?:-(#{HEX2}))?/o) { + if !$2 + c = $1.to_i(16) + set[c] = true + else + b = $1.to_i(16) + e = $2.to_i(16) + b.upto(e) {|c| set[c] = true } + end + } + i = nil + 0.upto(256) {|j| + if set[j] + if !i + i = j + end + if !set[j+1] + ranges_list.last << ["%02X" % i, "%02X" % j] + i = nil + end + end + } + end + } + first_ranges = ranges_list.shift + first_ranges.product(*ranges_list).each {|range_list| + min = range_list.map {|x, y| x }.join + max = range_list.map {|x, y| y }.join + rects << [min, max, action] + } + } + else + raise ArgumentError, "invalid pattern: #{pat.inspect}" + end + } + rects + end + + def self.unambiguous_action(actions0) + actions = actions0.uniq + if actions.length == 1 + actions[0] + else + actions.delete(:nomap0) + if actions.length == 1 + actions[0] + else + raise ArgumentError, "ambiguous actions: #{actions0.inspect}" + end + end + end + + def self.build_tree(rects) + expand(rects) {|prefix, actions| + unambiguous_action(actions) + } + end + + def self.parse(mapping) + rects = parse_to_rects(mapping) + tree = build_tree(rects) + self.new(tree) + end + + def self.merge_rects(*rects_list) + if rects_list.length < 2 + raise ArgumentError, "not enough arguments" + end + + all_rects = [] + rects_list.each_with_index {|rects, i| + all_rects.concat rects.map {|min, max, action| [min, max, [i, action]] } + } + + tree = expand(all_rects) {|prefix, actions| + args = Array.new(rects_list.length) { [] } + actions.each {|i, action| + args[i] << action + } + yield(prefix, *args) + } + + self.new(tree) + end + + def self.merge(*mappings, &block) + merge_rects(*mappings.map {|m| parse_to_rects(m) }, &block) + end + + def self.merge2(map1, map2, &block) + rects1 = parse_to_rects(map1) + rects2 = parse_to_rects(map2) + + actions = [] + all_rects = [] + + rects1.each {|rect| + min, max, action = rect + rect[2] = actions.length + actions << action + all_rects << rect + } + + boundary = actions.length + + rects2.each {|rect| + min, max, action = rect + rect[2] = actions.length + actions << action + all_rects << rect + } + + tree = expand(all_rects) {|prefix, as0| + as1 = [] + as2 = [] + as0.each {|i| + if i < boundary + as1 << actions[i] + else + as2 << actions[i] + end + } + yield(prefix, as1, as2) + } + + self.new(tree) + end + + def self.expand(rects, &block) + #numsing = numreg = 0 + #rects.each {|min, max, action| if min == max then numsing += 1 else numreg += 1 end } + #puts "#{numsing} singleton mappings and #{numreg} region mappings." + singleton_rects = [] + region_rects = [] + rects.each {|rect| + min, max, action = rect + if min == max + singleton_rects << rect + else + region_rects << rect + end + } + @singleton_rects = singleton_rects.sort_by {|min, max, action| min } + @singleton_rects.reverse! + ret = expand_rec("", region_rects, &block) + @singleton_rects = nil + ret + end + + TMPHASH = {} + def self.expand_rec(prefix, region_rects, &block) + return region_rects if region_rects.empty? && !((s_rect = @singleton_rects.last) && s_rect[0].start_with?(prefix)) + if region_rects.empty? ? s_rect[0].length == prefix.length : region_rects[0][0].empty? + h = TMPHASH + while (s_rect = @singleton_rects.last) && s_rect[0].start_with?(prefix) + min, max, action = @singleton_rects.pop + raise ArgumentError, "ambiguous pattern: #{prefix}" if min.length != prefix.length + h[action] = true + end + region_rects.each {|min, max, action| + raise ArgumentError, "ambiguous pattern: #{prefix}" if !min.empty? + h[action] = true + } + tree = Action.new(block.call(prefix, h.keys)) + h.clear + else + tree = [] + each_firstbyte_range(prefix, region_rects) {|byte_min, byte_max, r_rects2| + if byte_min == byte_max + prefix2 = prefix + "%02X" % byte_min + else + prefix2 = prefix + "{%02X-%02X}" % [byte_min, byte_max] + end + child_tree = expand_rec(prefix2, r_rects2, &block) + tree << Branch.new(byte_min, byte_max, child_tree) + } + end + return tree + end + + def self.each_firstbyte_range(prefix, region_rects) + index_from = TMPHASH + + region_ary = [] + region_rects.each {|min, max, action| + raise ArgumentError, "ambiguous pattern: #{prefix}" if min.empty? + min_firstbyte = min[0,2].to_i(16) + min_rest = min[2..-1] + max_firstbyte = max[0,2].to_i(16) + max_rest = max[2..-1] + region_ary << [min_firstbyte, max_firstbyte, [min_rest, max_rest, action]] + index_from[min_firstbyte] = true + index_from[max_firstbyte+1] = true + } + + byte_from = Array.new(index_from.size) + bytes = index_from.keys + bytes.sort! + bytes.reverse! + bytes.each_with_index {|byte, i| + index_from[byte] = i + byte_from[i] = byte + } + + region_rects_ary = Array.new(index_from.size) { [] } + region_ary.each {|min_firstbyte, max_firstbyte, rest_elt| + index_from[min_firstbyte].downto(index_from[max_firstbyte+1]+1) {|i| + region_rects_ary[i] << rest_elt + } + } + + index_from.clear + + r_rects = region_rects_ary.pop + region_byte = byte_from.pop + prev_r_start = region_byte + prev_r_rects = [] + while r_rects && (s_rect = @singleton_rects.last) && (seq = s_rect[0]).start_with?(prefix) + singleton_byte = seq[prefix.length, 2].to_i(16) + min_byte = singleton_byte < region_byte ? singleton_byte : region_byte + if prev_r_start < min_byte && !prev_r_rects.empty? + yield prev_r_start, min_byte-1, prev_r_rects + end + if region_byte < singleton_byte + prev_r_start = region_byte + prev_r_rects = r_rects + r_rects = region_rects_ary.pop + region_byte = byte_from.pop + elsif region_byte > singleton_byte + yield singleton_byte, singleton_byte, prev_r_rects + prev_r_start = singleton_byte+1 + else # region_byte == singleton_byte + prev_r_start = region_byte+1 + prev_r_rects = r_rects + r_rects = region_rects_ary.pop + region_byte = byte_from.pop + yield singleton_byte, singleton_byte, prev_r_rects + end + end + + while r_rects + if prev_r_start < region_byte && !prev_r_rects.empty? + yield prev_r_start, region_byte-1, prev_r_rects + end + prev_r_start = region_byte + prev_r_rects = r_rects + r_rects = region_rects_ary.pop + region_byte = byte_from.pop + end + + while (s_rect = @singleton_rects.last) && (seq = s_rect[0]).start_with?(prefix) + singleton_byte = seq[prefix.length, 2].to_i(16) + yield singleton_byte, singleton_byte, [] + end + end + + def initialize(tree) + @tree = tree + end + + def inspect + "\#<#{self.class}:" + + @tree.inspect + + ">" + end + + def max_input_length_rec(tree) + case tree + when Action + 0 + else + tree.map {|branch| + max_input_length_rec(branch.child_tree) + }.max + 1 + end + end + + def max_input_length + max_input_length_rec(@tree) + end + + def empty_action + if @tree.kind_of? Action + @tree.value + else + nil + end + end + + OffsetsMemo = {} + InfosMemo = {} + + def format_offsets(min, max, offsets) + offsets = offsets[min..max] + code = "%d, %d,\n" % [min, max] + 0.step(offsets.length-1,16) {|i| + code << " " + code << offsets[i,8].map {|off| "%3d," % off.to_s }.join('') + if i+8 < offsets.length + code << " " + code << offsets[i+8,8].map {|off| "%3d," % off.to_s }.join('') + end + code << "\n" + } + code + end + + UsedName = {} + + StrMemo = {} + + def str_name(bytes) + size = @bytes_code.length + rawbytes = [bytes].pack("H*") + + n = nil + if !n && !(suf = rawbytes.gsub(/[^A-Za-z0-9_]/, '')).empty? && !UsedName[nn = "str1_" + suf] then n = nn end + if !n && !UsedName[nn = "str1_" + bytes] then n = nn end + n ||= "str1s_#{size}" + + StrMemo[bytes] = n + UsedName[n] = true + n + end + + def gen_str(bytes) + if n = StrMemo[bytes] + n + else + len = bytes.length/2 + size = @bytes_code.length + n = str_name(bytes) + @bytes_code.insert_at_last(1 + len, + "\#define #{n} makeSTR1(#{size})\n" + + " makeSTR1LEN(#{len})," + bytes.gsub(/../, ' 0x\&,') + "\n\n") + n + end + end + + def generate_info(info) + case info + when :nomap, :nomap0 + # :nomap0 is low priority. it never collides. + "NOMAP" + when :undef + "UNDEF" + when :invalid + "INVALID" + when :func_ii + "FUNii" + when :func_si + "FUNsi" + when :func_io + "FUNio" + when :func_so + "FUNso" + when /\A(#{HEX2})\z/o + "o1(0x#$1)" + when /\A(#{HEX2})(#{HEX2})\z/o + "o2(0x#$1,0x#$2)" + when /\A(#{HEX2})(#{HEX2})(#{HEX2})\z/o + "o3(0x#$1,0x#$2,0x#$3)" + when /funsio\((\d+)\)/ + "funsio(#{$1})" + when /\A(#{HEX2})(3[0-9])(#{HEX2})(3[0-9])\z/o + "g4(0x#$1,0x#$2,0x#$3,0x#$4)" + when /\A(f[0-7])(#{HEX2})(#{HEX2})(#{HEX2})\z/o + "o4(0x#$1,0x#$2,0x#$3,0x#$4)" + when /\A(#{HEX2}){4,259}\z/o + gen_str(info.upcase) + when /\A\/\*BYTE_LOOKUP\*\// # pointer to BYTE_LOOKUP structure + $'.to_s + else + raise "unexpected action: #{info.inspect}" + end + end + + def format_infos(infos) + infos = infos.map {|info| generate_info(info) } + maxlen = infos.map {|info| info.length }.max + columns = maxlen <= 16 ? 4 : 2 + code = "".dup + 0.step(infos.length-1, columns) {|i| + code << " " + is = infos[i,columns] + is.each {|info| + code << sprintf(" %#{maxlen}s,", info) + } + code << "\n" + } + code + end + + def generate_lookup_node(name, table) + bytes_code = @bytes_code + words_code = @words_code + offsets = [] + infos = [] + infomap = {} + min = max = nil + table.each_with_index {|action, byte| + action ||= :invalid + if action != :invalid + min = byte if !min + max = byte + end + unless o = infomap[action] + infomap[action] = o = infos.length + infos[o] = action + end + offsets[byte] = o + } + infomap.clear + if !min + min = max = 0 + end + + offsets_key = [min, max, offsets[min..max]] + if n = OffsetsMemo[offsets_key] + offsets_name = n + else + offsets_name = "#{name}_offsets" + OffsetsMemo[offsets_key] = offsets_name + size = bytes_code.length + bytes_code.insert_at_last(2+max-min+1, + "\#define #{offsets_name} #{size}\n" + + format_offsets(min,max,offsets) + "\n") + end + + if n = InfosMemo[infos] + infos_name = n + else + infos_name = "#{name}_infos" + InfosMemo[infos] = infos_name + + size = words_code.length + words_code.insert_at_last(infos.length, + "\#define #{infos_name} WORDINDEX2INFO(#{size})\n" + + format_infos(infos) + "\n") + end + + size = words_code.length + words_code.insert_at_last(NUM_ELEM_BYTELOOKUP, + "\#define #{name} WORDINDEX2INFO(#{size})\n" + + <<"End" + "\n") + #{offsets_name}, + #{infos_name}, +End + end + + PreMemo = {} + NextName = "a" + + def generate_node(name_hint=nil) + if n = PreMemo[@tree] + return n + end + + table = Array.new(0x100, :invalid) + @tree.each {|branch| + byte_min, byte_max, child_tree = branch.byte_min, branch.byte_max, branch.child_tree + rest = ActionMap.new(child_tree) + if a = rest.empty_action + table.fill(a, byte_min..byte_max) + else + name_hint2 = nil + if name_hint + name_hint2 = "#{name_hint}_#{byte_min == byte_max ? '%02X' % byte_min : '%02Xto%02X' % [byte_min, byte_max]}" + end + v = "/*BYTE_LOOKUP*/" + rest.gennode(@bytes_code, @words_code, name_hint2) + table.fill(v, byte_min..byte_max) + end + } + + if !name_hint + name_hint = "fun_" + NextName + NextName.succ! + end + + PreMemo[@tree] = name_hint + + generate_lookup_node(name_hint, table) + name_hint + end + + def gennode(bytes_code, words_code, name_hint=nil) + @bytes_code = bytes_code + @words_code = words_code + name = generate_node(name_hint) + @bytes_code = nil + @words_code = nil + return name + end +end + +def citrus_mskanji_cstomb(csid, index) + case csid + when 0 + index + when 1 + index + 0x80 + when 2, 3 + row = index >> 8 + raise "invalid byte sequence" if row < 0x21 + if csid == 3 + if row <= 0x2F + offset = (row == 0x22 || row >= 0x26) ? 0xED : 0xF0 + elsif row >= 0x4D && row <= 0x7E + offset = 0xCE + else + raise "invalid byte sequence" + end + else + raise "invalid byte sequence" if row > 0x97 + offset = (row < 0x5F) ? 0x81 : 0xC1 + end + col = index & 0xFF + raise "invalid byte sequence" if (col < 0x21 || col > 0x7E) + + row -= 0x21 + col -= 0x21 + if (row & 1) == 0 + col += 0x40 + col += 1 if (col >= 0x7F) + else + col += 0x9F; + end + row = row / 2 + offset + (row << 8) | col + end.to_s(16) +end + +def citrus_euc_cstomb(csid, index) + case csid + when 0x0000 + index + when 0x8080 + index | 0x8080 + when 0x0080 + index | 0x8E80 + when 0x8000 + index | 0x8F8080 + end.to_s(16) +end + +def citrus_stateless_iso_cstomb(csid, index) + (index | 0x8080 | (csid << 16)).to_s(16) +end + +def citrus_cstomb(ces, csid, index) + case ces + when 'mskanji' + citrus_mskanji_cstomb(csid, index) + when 'euc' + citrus_euc_cstomb(csid, index) + when 'stateless_iso' + citrus_stateless_iso_cstomb(csid, index) + end +end + +SUBDIR = %w/APPLE AST BIG5 CNS CP EBCDIC EMOJI GB GEORGIAN ISO646 ISO-8859 JIS KAZAKH KOI KS MISC TCVN/ + + +def citrus_decode_mapsrc(ces, csid, mapsrcs) + table = [] + mapsrcs.split(',').each do |mapsrc| + path = [$srcdir] + mode = nil + if mapsrc.rindex(/UCS(?:@[A-Z]+)?/, 0) + mode = :from_ucs + from = mapsrc[$&.size+1..-1] + path << SUBDIR.find{|x| from.rindex(x, 0) } + else + mode = :to_ucs + path << SUBDIR.find{|x| mapsrc.rindex(x, 0) } + end + if /\bUCS@(BMP|SMP|SIP|TIP|SSP)\b/ =~ mapsrc + plane = {"BMP"=>0, "SMP"=>1, "SIP"=>2, "TIP"=>3, "SSP"=>14}[$1] + else + plane = 0 + end + plane <<= 16 + path << mapsrc.gsub(':', '@') + path = File.join(*path) + path << ".src" + path[path.rindex('/')] = '%' + STDERR.puts 'load mapsrc %s' % path if VERBOSE_MODE + open(path, 'rb') do |f| + f.each_line do |l| + break if /^BEGIN_MAP/ =~ l + end + f.each_line do |l| + next if /^\s*(?:#|$)/ =~ l + break if /^END_MAP/ =~ l + case mode + when :from_ucs + case l + when /0x(\w+)\s*-\s*0x(\w+)\s*=\s*INVALID/ + # Citrus OOB_MODE + when /(0x\w+)\s*=\s*(0x\w+)/ + table.push << [plane | $1.hex, citrus_cstomb(ces, csid, $2.hex)] + else + raise "unknown notation '%s'"% l.chomp + end + when :to_ucs + case l + when /(0x\w+)\s*=\s*(0x\w+)/ + table.push << [citrus_cstomb(ces, csid, $1.hex), plane | $2.hex] + else + raise "unknown notation '%s'"% l.chomp + end + end + end + end + end + return table +end + +def import_ucm(path) + to_ucs = [] + from_ucs = [] + File.foreach(File.join($srcdir, "ucm", path)) do |line| + uc, bs, fb = nil + if /^<U([0-9a-fA-F]+)>\s*([\+0-9a-fA-Fx\\]+)\s*\|(\d)/ =~ line + uc = $1.hex + bs = $2.delete('x\\') + fb = $3.to_i + next if uc < 128 && uc == bs.hex + elsif /^([<U0-9a-fA-F>+]+)\s*([\+0-9a-fA-Fx\\]+)\s*\|(\d)/ =~ line + uc = $1.scan(/[0-9a-fA-F]+>/).map(&:hex).pack("U*").unpack("H*")[0] + bs = $2.delete('x\\') + fb = $3.to_i + end + to_ucs << [bs, uc] if fb == 0 || fb == 3 + from_ucs << [uc, bs] if fb == 0 || fb == 1 + end + [to_ucs, from_ucs] +end + +def encode_utf8(map) + r = [] + map.each {|k, v| + # integer means UTF-8 encoded sequence. + k = [k].pack("U").unpack("H*")[0].upcase if Integer === k + v = [v].pack("U").unpack("H*")[0].upcase if Integer === v + r << [k,v] + } + r +end + +UnspecifiedValidEncoding = Object.new + +def transcode_compile_tree(name, from, map, valid_encoding) + map = encode_utf8(map) + h = {} + map.each {|k, v| + h[k] = v unless h[k] # use first mapping + } + if valid_encoding.equal? UnspecifiedValidEncoding + valid_encoding = ValidEncoding.fetch(from) + end + if valid_encoding + am = ActionMap.merge2(h, {valid_encoding => :undef}) {|prefix, as1, as2| + a1 = as1.empty? ? nil : ActionMap.unambiguous_action(as1) + a2 = as2.empty? ? nil : ActionMap.unambiguous_action(as2) + if !a2 + raise "invalid mapping: #{prefix}" + end + a1 || a2 + } + else + am = ActionMap.parse(h) + end + h.clear + + max_input = am.max_input_length + defined_name = am.gennode(TRANSCODE_GENERATED_BYTES_CODE, TRANSCODE_GENERATED_WORDS_CODE, name) + return defined_name, max_input +end + +TRANSCODERS = [] +TRANSCODE_GENERATED_TRANSCODER_CODE = ''.dup + +def transcode_tbl_only(from, to, map, valid_encoding=UnspecifiedValidEncoding) + if VERBOSE_MODE + if from.empty? || to.empty? + STDERR.puts "converter for #{from.empty? ? to : from}" + else + STDERR.puts "converter from #{from} to #{to}" + end + end + id_from = from.tr('^0-9A-Za-z', '_') + id_to = to.tr('^0-9A-Za-z', '_') + if from == "UTF-8" + tree_name = "to_#{id_to}" + elsif to == "UTF-8" + tree_name = "from_#{id_from}" + else + tree_name = "from_#{id_from}_to_#{id_to}" + end + real_tree_name, max_input = transcode_compile_tree(tree_name, from, map, valid_encoding) + return map, tree_name, real_tree_name, max_input +end + +# +# call-seq: +# transcode_tblgen(from_name, to_name, map [, valid_encoding_check [, ascii_compatibility]]) -> '' +# +# Returns an empty string just in case the result is used somewhere. +# Stores the actual product for later output with transcode_generated_code and +# transcode_register_code. +# +# The first argument is a string that will be used for the source (from) encoding. +# The second argument is a string that will be used for the target (to) encoding. +# +# The third argument is the actual data, a map represented as an array of two-element +# arrays. Each element of the array stands for one character being converted. The +# first element of each subarray is the code of the character in the source encoding, +# the second element of each subarray is the code of the character in the target encoding. +# +# Each code (i.e. byte sequence) is represented as a string of hexadecimal characters +# of even length. Codes can also be represented as integers (usually in the form Ox...), +# in which case they are interpreted as Unicode codepoints encoded in UTF-8. So as +# an example, 0x677E is the same as "E69DBE" (but somewhat easier to produce and check). +# +# In addition, the following symbols can also be used instead of actual codes in the +# second element of a subarray: +# :nomap (no mapping, just copy input to output), :nomap0 (same as :nomap, but low priority), +# :undef (input code undefined in the destination encoding), +# :invalid (input code is an invalid byte sequence in the source encoding), +# :func_ii, :func_si, :func_io, :func_so (conversion by function with specific call +# convention). +# +# The forth argument specifies the overall structure of the encoding. For examples, +# see ValidEncoding below. This is used to cross-check the data in the third argument +# and to automatically add :undef and :invalid mappings where necessary. +# +# The fifth argument gives the ascii-compatibility of the transcoding. See +# rb_transcoder_asciicompat_type_t in transcode_data.h for details. In most +# cases, this argument can be left out. +# +def transcode_tblgen(from, to, map, valid_encoding=UnspecifiedValidEncoding, + ascii_compatibility='asciicompat_converter') + map, tree_name, real_tree_name, max_input = transcode_tbl_only(from, to, map, valid_encoding) + transcoder_name = "rb_#{tree_name}" + TRANSCODERS << transcoder_name + input_unit_length = UnitLength[from] + max_output = map.map {|k,v| String === v ? v.length/2 : 1 }.max + transcoder_code = <<"End" +static const rb_transcoder +#{transcoder_name} = { + #{c_esc from}, #{c_esc to}, #{real_tree_name}, + TRANSCODE_TABLE_INFO, + #{input_unit_length}, /* input_unit_length */ + #{max_input}, /* max_input */ + #{max_output}, /* max_output */ + #{ascii_compatibility}, /* asciicompat_type */ + 0, NULL, NULL, /* state_size, state_init, state_fini */ + NULL, NULL, NULL, NULL, + NULL, NULL, NULL +}; +End + TRANSCODE_GENERATED_TRANSCODER_CODE << transcoder_code + '' +end + +def transcode_generate_node(am, name_hint=nil) + STDERR.puts "converter for #{name_hint}" if VERBOSE_MODE + name = am.gennode(TRANSCODE_GENERATED_BYTES_CODE, TRANSCODE_GENERATED_WORDS_CODE, name_hint) + '' +end + +def transcode_generated_code + TRANSCODE_GENERATED_BYTES_CODE.to_s + + TRANSCODE_GENERATED_WORDS_CODE.to_s + + "\#define TRANSCODE_TABLE_INFO " + + "#{OUTPUT_PREFIX}byte_array, #{TRANSCODE_GENERATED_BYTES_CODE.length}, " + + "#{OUTPUT_PREFIX}word_array, #{TRANSCODE_GENERATED_WORDS_CODE.length}, " + + "((int)sizeof(unsigned int))\n" + + TRANSCODE_GENERATED_TRANSCODER_CODE +end + +def transcode_register_code + code = ''.dup + TRANSCODERS.each {|transcoder_name| + code << " rb_register_transcoder(&#{transcoder_name});\n" + } + code +end + +UnitLength = { + 'UTF-16BE' => 2, + 'UTF-16LE' => 2, + 'UTF-32BE' => 4, + 'UTF-32LE' => 4, +} +UnitLength.default = 1 + +ValidEncoding = { + '1byte' => '{00-ff}', + '2byte' => '{00-ff}{00-ff}', + '4byte' => '{00-ff}{00-ff}{00-ff}{00-ff}', + 'US-ASCII' => '{00-7f}', + 'UTF-8' => '{00-7f} + {c2-df}{80-bf} + e0{a0-bf}{80-bf} + {e1-ec}{80-bf}{80-bf} + ed{80-9f}{80-bf} + {ee-ef}{80-bf}{80-bf} + f0{90-bf}{80-bf}{80-bf} + {f1-f3}{80-bf}{80-bf}{80-bf} + f4{80-8f}{80-bf}{80-bf}', + 'UTF-16BE' => '{00-d7,e0-ff}{00-ff} + {d8-db}{00-ff}{dc-df}{00-ff}', + 'UTF-16LE' => '{00-ff}{00-d7,e0-ff} + {00-ff}{d8-db}{00-ff}{dc-df}', + 'UTF-32BE' => '0000{00-d7,e0-ff}{00-ff} + 00{01-10}{00-ff}{00-ff}', + 'UTF-32LE' => '{00-ff}{00-d7,e0-ff}0000 + {00-ff}{00-ff}{01-10}00', + 'EUC-JP' => '{00-7f} + {a1-fe}{a1-fe} + 8e{a1-fe} + 8f{a1-fe}{a1-fe}', + 'CP51932' => '{00-7f} + {a1-fe}{a1-fe} + 8e{a1-fe}', + 'EUC-JIS-2004' => '{00-7f} + {a1-fe}{a1-fe} + 8e{a1-fe} + 8f{a1-fe}{a1-fe}', + 'Shift_JIS' => '{00-7f} + {81-9f,e0-fc}{40-7e,80-fc} + {a1-df}', + 'EUC-KR' => '{00-7f} + {a1-fe}{a1-fe}', + 'CP949' => '{00-7f} + {81-fe}{41-5a,61-7a,81-fe}', + 'Big5' => '{00-7f} + {81-fe}{40-7e,a1-fe}', + 'EUC-TW' => '{00-7f} + {a1-fe}{a1-fe} + 8e{a1-b0}{a1-fe}{a1-fe}', + 'GBK' => '{00-80} + {81-fe}{40-7e,80-fe}', + 'GB18030' => '{00-7f} + {81-fe}{40-7e,80-fe} + {81-fe}{30-39}{81-fe}{30-39}', +} + +def ValidEncoding(enc) + ValidEncoding.fetch(enc) +end + +def set_valid_byte_pattern(encoding, pattern_or_label) + pattern = + if ValidEncoding[pattern_or_label] + ValidEncoding[pattern_or_label] + else + pattern_or_label + end + if ValidEncoding[encoding] and ValidEncoding[encoding]!=pattern + raise ArgumentError, "trying to change valid byte pattern for encoding #{encoding} from #{ValidEncoding[encoding]} to #{pattern}" + end + ValidEncoding[encoding] = pattern +end + +# the following may be used in different places, so keep them here for the moment +set_valid_byte_pattern 'ASCII-8BIT', '1byte' +set_valid_byte_pattern 'Windows-31J', 'Shift_JIS' +set_valid_byte_pattern 'eucJP-ms', 'EUC-JP' + +def make_signature(filename, src) + "src=#{filename.dump}, len=#{src.length}, checksum=#{src.sum}" +end + +if __FILE__ == $0 + start_time = Time.now + + output_filename = nil + verbose_mode = false + force_mode = false + + op = OptionParser.new + op.def_option("--help", "show help message") { puts op; exit 0 } + op.def_option("--verbose", "verbose mode") { verbose_mode = true } + op.def_option("--force", "force table generation") { force_mode = true } + op.def_option("--output=FILE", "specify output file") {|arg| output_filename = arg } + op.parse! + + VERBOSE_MODE = verbose_mode + + OUTPUT_FILENAME = output_filename + OUTPUT_PREFIX = output_filename ? File.basename(output_filename)[/\A[A-Za-z0-9_]*/] : "".dup + OUTPUT_PREFIX.sub!(/\A_+/, '') + OUTPUT_PREFIX.sub!(/_*\z/, '_') + + TRANSCODE_GENERATED_BYTES_CODE = ArrayCode.new("unsigned char", "#{OUTPUT_PREFIX}byte_array") + TRANSCODE_GENERATED_WORDS_CODE = ArrayCode.new("unsigned int", "#{OUTPUT_PREFIX}word_array") + + arg = ARGV.shift + $srcdir = File.dirname(arg) + $:.unshift $srcdir unless $:.include? $srcdir + src = File.read(arg) + src.force_encoding("ascii-8bit") if src.respond_to? :force_encoding + this_script = File.read(__FILE__) + this_script.force_encoding("ascii-8bit") if this_script.respond_to? :force_encoding + + base_signature = "/* autogenerated. */\n".dup + base_signature << "/* #{make_signature(File.basename(__FILE__), this_script)} */\n" + base_signature << "/* #{make_signature(File.basename(arg), src)} */\n" + + if !force_mode && output_filename && File.readable?(output_filename) + old_signature = File.open(output_filename) {|f| f.gets("").chomp } + chk_signature = base_signature.dup + old_signature.each_line {|line| + if %r{/\* src="([0-9a-z_.-]+)",} =~ line + name = $1 + next if name == File.basename(arg) || name == File.basename(__FILE__) + path = File.join($srcdir, name) + if File.readable? path + chk_signature << "/* #{make_signature(name, File.read(path))} */\n" + end + end + } + if old_signature == chk_signature + now = Time.now + File.utime(now, now, output_filename) + STDERR.puts "already up-to-date: #{output_filename}" if VERBOSE_MODE + exit + end + end + + if VERBOSE_MODE + if output_filename + STDERR.puts "generating #{output_filename} ..." + end + end + + libs1 = $".dup + erb = ERB.new(src, nil, '%') + erb.filename = arg + erb_result = erb.result(binding) + libs2 = $".dup + + libs = libs2 - libs1 + lib_sigs = ''.dup + libs.each {|lib| + lib = File.basename(lib) + path = File.join($srcdir, lib) + if File.readable? path + lib_sigs << "/* #{make_signature(lib, File.read(path))} */\n" + end + } + + result = ''.dup + result << base_signature + result << lib_sigs + result << "\n" + result << erb_result + result << "\n" + + if output_filename + new_filename = output_filename + ".new" + FileUtils.mkdir_p(File.dirname(output_filename)) + File.open(new_filename, "wb") {|f| f << result } + File.rename(new_filename, output_filename) + tms = Process.times + elapsed = Time.now - start_time + STDERR.puts "done. (#{'%.2f' % tms.utime}user #{'%.2f' % tms.stime}system #{'%.2f' % elapsed}elapsed)" if VERBOSE_MODE + else + print result + end +end diff --git a/tool/update-deps b/tool/update-deps new file mode 100755 index 0000000000..c308f35195 --- /dev/null +++ b/tool/update-deps @@ -0,0 +1,625 @@ +#!/usr/bin/ruby + +# tool/update-deps verify makefile dependencies. + +# Requirements: +# gcc 4.5 (for -save-temps=obj option) +# GNU make (for -p option) +# +# Warning: ccache (and similar tools) must be disabled for +# -save-temps=obj to work properly. +# +# Usage: +# 1. Compile ruby with -save-temps=obj option. +# Ex. ./configure debugflags='-save-temps=obj -g' && make all golf +# 2. Run tool/update-deps to show dependency problems. +# Ex. ruby tool/update-deps +# 3. Use --fix to fix makefiles. +# Ex. ruby tool/update-deps --fix +# +# Other usages: +# * Fix makefiles using previously detected dependency problems +# Ex. ruby tool/update-deps --actual-fix [file] +# "ruby tool/update-deps --fix" is same as "ruby tool/update-deps | ruby tool/update-deps --actual-fix". + +require 'optparse' +require 'stringio' +require 'pathname' +require 'open3' +require 'pp' + +# When out-of-place build, files may be built in source directory or +# build directory. +# Some files are always built in the source directory. +# Some files are always built in the build directory. +# Some files are built in the source directory for tarball but build directory for repository (svn). + +=begin +How to build test directories. + +VER=2.2.0 +REV=48577 +tar xf ruby-$VER-r$REV.tar.xz +cp -a ruby-$VER-r$REV tarball_source_dir_original +mv ruby-$VER-r$REV tarball_source_dir_after_build +svn co -q -r$REV http://svn.ruby-lang.org/repos/ruby/trunk ruby +(cd ruby; autoconf) +cp -a ruby repo_source_dir_original +mv ruby repo_source_dir_after_build +mkdir tarball_build_dir repo_build_dir tarball_install_dir repo_install_dir +(cd tarball_build_dir; ../tarball_source_dir_after_build/configure --prefix=$(cd ../tarball_install_dir; pwd) && make all golf install) > tarball.log 2>&1 +(cd repo_build_dir; ../repo_source_dir_after_build/configure --prefix=$(cd ../repo_install_dir; pwd) && make all golf install) > repo.log 2>&1 +ruby -rpp -rfind -e ' +ds = %w[ + repo_source_dir_original + repo_source_dir_after_build + repo_build_dir + tarball_source_dir_original + tarball_source_dir_after_build + tarball_build_dir +] +files = {} +ds.each {|d| + files[d] = {} + Dir.chdir(d) { Find.find(".") {|f| files[d][f] = true if %r{\.(c|h|inc|dmyh)\z} =~ f } } +} +result = {} +files_union = files.values.map {|h| h.keys }.flatten.uniq.sort +files_union.each {|f| + k = files.map {|d,h| h[f] ? d : nil }.compact.sort + next if k == %w[repo_source_dir_after_build repo_source_dir_original tarball_source_dir_after_build tarball_source_dir_original] + next if k == %w[repo_build_dir tarball_build_dir] && File.basename(f) == "extconf.h" + result[k] ||= [] + result[k] << f +} +result.each {|k,v| + k.each {|d| + puts d + } + v.each {|f| + puts " " + f.sub(%r{\A\./}, "") + } + puts +} +' | tee compare.log +=end + +# Files built in the source directory. +# They can be referenced as $(top_srcdir)/filename. +# % ruby -e 'def g(d) Dir.chdir(d) { Dir["**/*.{c,h,inc,dmyh}"] } end; puts((g("repo_source_dir_after_build") - g("repo_source_dir_original")).sort)' +FILES_IN_SOURCE_DIRECTORY = %w[ + revision.h +] + +# Files built in the build directory (except extconf.h). +# They can be referenced as $(topdir)/filename. +# % ruby -e 'def g(d) Dir.chdir(d) { Dir["**/*.{c,h,inc,dmyh}"] } end; puts(g("tarball_build_dir").reject {|f| %r{/extconf.h\z} =~ f }.sort)' +FILES_IN_BUILD_DIRECTORY = %w[ + encdb.h + ext/etc/constdefs.h + ext/socket/constdefs.c + ext/socket/constdefs.h + probes.h + transdb.h + verconf.h +] + +# They are built in the build directory if the source is obtained from the repository. +# However they are pre-built for tarball and they exist in the source directory extracted from the tarball. +# % ruby -e 'def g(d) Dir.chdir(d) { Dir["**/*.{c,h,inc,dmyh}"] } end; puts((g("repo_build_dir") & g("tarball_source_dir_original")).sort)' +FILES_NEED_VPATH = %w[ + ext/rbconfig/sizeof/sizes.c + ext/ripper/eventids1.c + ext/ripper/eventids2table.c + ext/ripper/ripper.c + golf_prelude.c + id.c + id.h + insns.inc + insns_info.inc + known_errors.inc + lex.c + miniprelude.c + newline.c + node_name.inc + opt_sc.inc + optinsn.inc + optunifs.inc + parse.c + parse.h + prelude.c + probes.dmyh + vm.inc + vmtc.inc + + enc/trans/big5.c + enc/trans/chinese.c + enc/trans/emoji.c + enc/trans/emoji_iso2022_kddi.c + enc/trans/emoji_sjis_docomo.c + enc/trans/emoji_sjis_kddi.c + enc/trans/emoji_sjis_softbank.c + enc/trans/escape.c + enc/trans/gb18030.c + enc/trans/gbk.c + enc/trans/iso2022.c + enc/trans/japanese.c + enc/trans/japanese_euc.c + enc/trans/japanese_sjis.c + enc/trans/korean.c + enc/trans/single_byte.c + enc/trans/utf8_mac.c + enc/trans/utf_16_32.c +] + +# Multiple files with same filename. +# It is not good idea to refer them using VPATH. +# Files in FILES_SAME_NAME_INC is referenced using $(hdrdir). +# Files in FILES_SAME_NAME_TOP is referenced using $(top_srcdir). +# include/ruby.h is referenced using $(top_srcdir) because mkmf.rb replaces +# $(hdrdir)/ruby.h to $(hdrdir)/ruby/ruby.h + +FILES_SAME_NAME_INC = %w[ + include/ruby/ruby.h + include/ruby/version.h +] + +FILES_SAME_NAME_TOP = %w[ + include/ruby.h + version.h +] + +# Other source files exist in the source directory. + +def in_makefile(target, source) + target = target.to_s + source = source.to_s + case target + when %r{\A[^/]*\z} + target2 = "#{target.sub(/\.o\z/, '.$(OBJEXT)')}" + case source + when *FILES_IN_SOURCE_DIRECTORY then source2 = "$(top_srcdir)/#{source}" + when *FILES_IN_BUILD_DIRECTORY then source2 = "{$(VPATH)}#{source}" # VPATH is not used now but it may changed in future. + when *FILES_NEED_VPATH then source2 = "{$(VPATH)}#{source}" + when *FILES_SAME_NAME_INC then source2 = "$(hdrdir)/#{source.sub(%r{\Ainclude/},'')}" + when *FILES_SAME_NAME_TOP then source2 = "$(top_srcdir)/#{source}" + when 'thread_pthread.c' then source2 = '{$(VPATH)}thread_$(THREAD_MODEL).c' + when 'thread_pthread.h' then source2 = '{$(VPATH)}thread_$(THREAD_MODEL).h' + when %r{\A[^/]*\z} then source2 = "{$(VPATH)}#{File.basename source}" + when %r{\A\.ext/include/[^/]+/ruby/} then source2 = "{$(VPATH)}#{$'}" + when %r{\Ainclude/ruby/} then source2 = "{$(VPATH)}#{$'}" + when %r{\Aenc/} then source2 = "{$(VPATH)}#{$'}" + when %r{\Amissing/} then source2 = "{$(VPATH)}#{$'}" + when %r{\Accan/} then source2 = "$(CCAN_DIR)/#{$'}" + when %r{\Adefs/} then source2 = "{$(VPATH)}#{source}" + else source2 = "$(top_srcdir)/#{source}" + end + ["common.mk", target2, source2] + when %r{\Aenc/} + target2 = "#{target.sub(/\.o\z/, '.$(OBJEXT)')}" + case source + when *FILES_IN_SOURCE_DIRECTORY then source2 = "$(top_srcdir)/#{source}" + when *FILES_IN_BUILD_DIRECTORY then source2 = source + when *FILES_NEED_VPATH then source2 = source + when *FILES_SAME_NAME_INC then source2 = "$(hdrdir)/#{source.sub(%r{\Ainclude/},'')}" + when *FILES_SAME_NAME_TOP then source2 = "$(top_srcdir)/#{source}" + when %r{\A\.ext/include/[^/]+/ruby/} then source2 = $' + when %r{\Ainclude/ruby/} then source2 = $' + when %r{\Aenc/} then source2 = source + else source2 = "$(top_srcdir)/#{source}" + end + ["enc/depend", target2, source2] + when %r{\Aext/} + unless File.exist?("#{File.dirname(target)}/extconf.rb") + warn "warning: not found: #{File.dirname(target)}/extconf.rb" + end + target2 = File.basename(target) + relpath = Pathname(source).relative_path_from(Pathname(target).dirname).to_s + case source + when *FILES_IN_SOURCE_DIRECTORY then source2 = "$(top_srcdir)/#{source}" + when *FILES_IN_BUILD_DIRECTORY then source2 = relpath + when *FILES_NEED_VPATH then source2 = "{$(VPATH)}#{File.basename source}" + when *FILES_SAME_NAME_INC then source2 = "$(hdrdir)/#{source.sub(%r{\Ainclude/},'')}" + when *FILES_SAME_NAME_TOP then source2 = "$(top_srcdir)/#{source}" + when %r{\A\.ext/include/[^/]+/ruby/} then source2 = "$(arch_hdrdir)/ruby/#{$'}" + when %r{\Ainclude/} then source2 = "$(hdrdir)/#{$'}" + when %r{\A#{Regexp.escape File.dirname(target)}/extconf\.h\z} then source2 = "$(RUBY_EXTCONF_H)" + when %r{\A#{Regexp.escape File.dirname(target)}/} then source2 = $' + else source2 = "$(top_srcdir)/#{source}" + end + ["#{File.dirname(target)}/depend", target2, source2] + else + raise "unexpected target: #{target}" + end +end + +DEPENDENCIES_SECTION_START_MARK = "\# AUTOGENERATED DEPENDENCIES START\n" +DEPENDENCIES_SECTION_END_MARK = "\# AUTOGENERATED DEPENDENCIES END\n" + +def init_global + ENV['LC_ALL'] = 'C' + + $opt_fix = false + $opt_a = false + $opt_actual_fix = false + $i_not_found = false +end + +def optionparser + op = OptionParser.new + op.banner = 'Usage: ruby tool/update-deps' + op.def_option('-a', 'show valid dependencies') { $opt_a = true } + op.def_option('--fix') { $opt_fix = true } + op.def_option('--actual-fix') { $opt_actual_fix = true } + op +end + +def read_make_deps(cwd) + dependencies = {} + make_p, make_p_stderr, make_p_status = Open3.capture3("make -p all miniruby ruby golf") + File.open('update-deps.make.out.log', 'w') {|f| f.print make_p } + File.open('update-deps.make.err.log', 'w') {|f| f.print make_p_stderr } + if !make_p_status.success? + puts make_p_stderr + raise "make failed" + end + dirstack = [cwd] + curdir = nil + make_p.scan(%r{Entering\ directory\ ['`](.*)'| + ^\#\ (GNU\ Make)\ | + ^CURDIR\ :=\ (.*)| + ^([/0-9a-zA-Z._-]+):(.*)\n((?:\#.*\n)*)| + ^\#\ (Finished\ Make\ data\ base\ on)\ | + Leaving\ directory\ ['`](.*)'}x) { + directory_enter = $1 + data_base_start = $2 + data_base_curdir = $3 + rule_target = $4 + rule_sources = $5 + rule_desc = $6 + data_base_end = $7 + directory_leave = $8 + #p $~ + if directory_enter + enter_dir = Pathname(directory_enter) + #p [:enter, enter_dir] + dirstack.push enter_dir + elsif data_base_start + curdir = nil + elsif data_base_curdir + curdir = Pathname(data_base_curdir) + elsif rule_target && rule_sources && rule_desc && + /Modification time never checked/ !~ rule_desc # This pattern match eliminates rules which VPATH is not expanded. + target = rule_target + deps = rule_sources + deps = deps.scan(%r{[/0-9a-zA-Z._-]+}) + next if /\.o\z/ !~ target.to_s + next if /\A\./ =~ target.to_s # skip rules such as ".c.o" + #p [curdir, target, deps] + dir = curdir || dirstack.last + dependencies[dir + target] ||= [] + dependencies[dir + target] |= deps.map {|dep| dir + dep } + elsif data_base_end + curdir = nil + elsif directory_leave + leave_dir = Pathname(directory_leave) + #p [:leave, leave_dir] + if leave_dir != dirstack.last + warn "unexpected leave_dir : #{dirstack.last.inspect} != #{leave_dir.inspect}" + end + dirstack.pop + end + } + dependencies +end + +#def guess_compiler_wd(filename, hint0) +# hint = hint0 +# begin +# guess = hint + filename +# if guess.file? +# return hint +# end +# hint = hint.parent +# end while hint.to_s != '.' +# raise ArgumentError, "can not find #{filename} (hint: #{hint0})" +#end + +def read_single_cc_deps(path_i, cwd) + files = {} + compiler_wd = nil + path_i.each_line {|line| + next if /\A\# \d+ "(.*)"/ !~ line + dep = $1 + next if %r{\A<.*>\z} =~ dep # omit <command-line>, etc. + compiler_wd ||= dep + files[dep] = true + } + # gcc emits {# 1 "/absolute/directory/of/the/source/file//"} at 2nd line. + if %r{\A/.*//\z} =~ compiler_wd + files.delete compiler_wd + compiler_wd = Pathname(compiler_wd.sub(%r{//\z}, '')) + elsif !(compiler_wd = yield) + raise "compiler working directory not found: #{path_i}" + end + deps = [] + files.each_key {|dep| + dep = Pathname(dep) + if dep.relative? + dep = compiler_wd + dep + end + if !dep.file? + warn "warning: file not found: #{dep}" + next + end + next if !dep.to_s.start_with?(cwd.to_s) # omit system headers. + deps << dep + } + deps +end + +def read_cc_deps(cwd) + deps = {} + Pathname.glob('**/*.o').sort.each {|fn_o| + fn_i = fn_o.sub_ext('.i') + if !fn_i.exist? + warn "warning: not found: #{fn_i}" + $i_not_found = true + next + end + path_o = cwd + fn_o + path_i = cwd + fn_i + deps[path_o] = read_single_cc_deps(path_i, cwd) do + if fn_o.to_s.start_with?("enc/") + cwd + else + path_i.parent + end + end + } + deps +end + +def concentrate(dependencies, cwd) + deps = {} + dependencies.keys.sort.each {|target| + sources = dependencies[target] + target = target.relative_path_from(cwd) + sources = sources.map {|s| + rel = s.relative_path_from(cwd) + rel + } + if %r{\A\.\.(/|\z)} =~ target.to_s + warn "warning: out of tree target: #{target}" + next + end + sources = sources.reject {|s| + if %r{\A\.\.(/|\z)} =~ s.to_s + warn "warning: out of tree source: #{s}" + true + elsif %r{/\.time\z} =~ s.to_s + true + else + false + end + } + deps[target] = sources + } + deps +end + +def sort_paths(paths) + paths.sort_by {|t| + ary = t.to_s.split(%r{/}) + ary.map.with_index {|e, i| i == ary.length-1 ? [0, e] : [1, e] } # regular file first, directories last. + } +end + +def show_deps(tag, deps) + targets = sort_paths(deps.keys) + targets.each {|t| + sources = sort_paths(deps[t]) + sources.each {|s| + puts "#{tag} #{t}: #{s}" + } + } +end + +def detect_dependencies(out=$stdout) + cwd = Pathname.pwd + make_deps = read_make_deps(cwd) + #pp make_deps + make_deps = concentrate(make_deps, cwd) + #pp make_deps + cc_deps = read_cc_deps(cwd) + #pp cc_deps + cc_deps = concentrate(cc_deps, cwd) + #pp cc_deps + return make_deps, cc_deps +end + +def compare_deps(make_deps, cc_deps, out=$stdout) + targets = make_deps.keys | cc_deps.keys + + makefiles = {} + + make_lines_hash = {} + make_deps.each {|t, sources| + sources.each {|s| + makefile, t2, s2 = in_makefile(t, s) + makefiles[makefile] = true + make_lines_hash[makefile] ||= Hash.new(false) + make_lines_hash[makefile]["#{t2}: #{s2}"] = true + } + } + + cc_lines_hash = {} + cc_deps.each {|t, sources| + sources.each {|s| + makefile, t2, s2 = in_makefile(t, s) + makefiles[makefile] = true + cc_lines_hash[makefile] ||= Hash.new(false) + cc_lines_hash[makefile]["#{t2}: #{s2}"] = true + } + } + + makefiles.keys.sort.each {|makefile| + cc_lines = cc_lines_hash[makefile] || Hash.new(false) + make_lines = make_lines_hash[makefile] || Hash.new(false) + content = begin + File.read(makefile) + rescue Errno::ENOENT + '' + end + if /^#{Regexp.escape DEPENDENCIES_SECTION_START_MARK} + ((?:.*\n)*) + #{Regexp.escape DEPENDENCIES_SECTION_END_MARK}/x =~ content + pre_post_part = [$`, $'] + current_lines = Hash.new(false) + $1.each_line {|line| current_lines[line.chomp] = true } + (cc_lines.keys | current_lines.keys | make_lines.keys).sort.each {|line| + status = [cc_lines[line], current_lines[line], make_lines[line]] + case status + when [true, true, true] + # no problem + when [true, true, false] + out.puts "warning #{makefile} : #{line} (make doesn't detect written dependency)" + when [true, false, true] + out.puts "add_auto #{makefile} : #{line} (harmless)" # This is automatically updatable. + when [true, false, false] + out.puts "add_auto #{makefile} : #{line} (harmful)" # This is automatically updatable. + when [false, true, true] + out.puts "del_cc #{makefile} : #{line}" # Not automatically updatable because build on other OS may need the dependency. + when [false, true, false] + out.puts "del_cc #{makefile} : #{line} (Curious. make doesn't detect this dependency.)" # Not automatically updatable because build on other OS may need the dependency. + when [false, false, true] + out.puts "del_make #{makefile} : #{line}" # Not automatically updatable because the dependency is written manually. + else + raise "unexpected status: #{status.inspect}" + end + } + else + (cc_lines.keys | make_lines.keys).sort.each {|line| + status = [cc_lines[line], make_lines[line]] + case status + when [true, true] + # no problem + when [true, false] + out.puts "add_manual #{makefile} : #{line}" # Not automatically updatable because makefile has no section to update automatically. + when [false, true] + out.puts "del_manual #{makefile} : #{line}" # Not automatically updatable because makefile has no section to update automatically. + else + raise "unexpected status: #{status.inspect}" + end + } + end + } +end + +def main_show(out=$stdout) + make_deps, cc_deps = detect_dependencies(out) + compare_deps(make_deps, cc_deps, out) +end + +def extract_deplines(problems) + adds = {} + others = {} + problems.each_line {|line| + case line + when /\Aadd_auto (\S+) : ((\S+): (\S+))/ + (adds[$1] ||= []) << [line, "#{$2}\n"] + when /\A(?:del_cc|del_make|add_manual|del_manual|warning) (\S+) : / + (others[$1] ||= []) << line + else + raise "unexpected line: #{line.inspect}" + end + } + return adds, others +end + +def main_actual_fix(problems) + adds, others = extract_deplines(problems) + (adds.keys | others.keys).sort.each {|makefile| + content = begin + File.read(makefile) + rescue Errno::ENOENT + nil + end + + if content && + /^#{Regexp.escape DEPENDENCIES_SECTION_START_MARK} + ((?:.*\n)*) + #{Regexp.escape DEPENDENCIES_SECTION_END_MARK}/x =~ content + pre_dep_post = [$`, $1, $'] + else + pre_dep_post = nil + end + + if pre_dep_post && adds[makefile] + pre_lines, dep_lines, post_lines = pre_dep_post + dep_lines = dep_lines.lines.to_a + add_lines = adds[makefile].map(&:last) + new_lines = (dep_lines | add_lines).sort.uniq + new_content = [ + pre_lines, + DEPENDENCIES_SECTION_START_MARK, + *new_lines, + DEPENDENCIES_SECTION_END_MARK, + post_lines + ].join + if content != new_content + puts "modified: #{makefile}" + tmp_makefile = "#{makefile}.new.#{$$}" + File.write(tmp_makefile, new_content) + File.rename tmp_makefile, makefile + (add_lines - dep_lines).each {|line| puts " added #{line}" } + else + puts "not modified: #{makefile}" + end + if others[makefile] + others[makefile].each {|line| puts " #{line}" } + end + else + if pre_dep_post + puts "no additional lines: #{makefile}" + elsif content + puts "no dependencies section: #{makefile}" + else + puts "no makefile: #{makefile}" + end + if adds[makefile] + puts " warning: dependencies section was exist at previous phase." + end + if adds[makefile] + adds[makefile].map(&:first).each {|line| puts " #{line}" } + end + if others[makefile] + others[makefile].each {|line| puts " #{line}" } + end + end + } +end + +def main_fix + problems = StringIO.new + main_show(problems) + main_actual_fix(problems.string) +end + +def run + op = optionparser + op.parse!(ARGV) + if $opt_actual_fix + main_actual_fix(ARGF.read) + elsif $opt_fix + main_fix + else + main_show + end +end + +init_global +run +if $i_not_found + warn "warning: missing *.i files, see help in #$0 and ensure ccache is disabled" +end diff --git a/tool/vcs.rb b/tool/vcs.rb new file mode 100644 index 0000000000..c3a5b3a0f8 --- /dev/null +++ b/tool/vcs.rb @@ -0,0 +1,495 @@ +# vcs +require 'fileutils' + +# This library is used by several other tools/ scripts to detect the current +# VCS in use (e.g. SVN, Git) or to interact with that VCS. + +ENV.delete('PWD') + +unless File.respond_to? :realpath + require 'pathname' + def File.realpath(arg) + Pathname(arg).realpath.to_s + end +end + +def IO.pread(*args) + STDERR.puts(*args.inspect) if $DEBUG + popen(*args) {|f|f.read} +end + +if RUBY_VERSION < "2.0" + class IO + @orig_popen = method(:popen) + + if defined?(fork) + def self.popen(command, *rest, &block) + if command.kind_of?(Hash) + env = command + command = rest.shift + end + opts = rest.last + if opts.kind_of?(Hash) + dir = opts.delete(:chdir) + rest.pop if opts.empty? + end + + if block + @orig_popen.call("-", *rest) do |f| + if f + yield(f) + else + Dir.chdir(dir) if dir + ENV.replace(env) if env + exec(*command) + end + end + else + f = @orig_popen.call("-", *rest) + unless f + Dir.chdir(dir) if dir + ENV.replace(env) if env + exec(*command) + end + f + end + end + else + require 'shellwords' + def self.popen(command, *rest, &block) + if command.kind_of?(Hash) + env = command + oldenv = ENV.to_hash + command = rest.shift + end + opts = rest.last + if opts.kind_of?(Hash) + dir = opts.delete(:chdir) + rest.pop if opts.empty? + end + + command = command.shelljoin if Array === command + Dir.chdir(dir || ".") do + ENV.replace(env) if env + @orig_popen.call(command, *rest, &block) + ENV.replace(oldenv) if oldenv + end + end + end + end +else + module DebugPOpen + verbose, $VERBOSE = $VERBOSE, nil if RUBY_VERSION < "2.1" + refine IO.singleton_class do + def popen(*args) + STDERR.puts args.inspect if $DEBUG + super + end + end + ensure + $VERBOSE = verbose unless verbose.nil? + end + using DebugPOpen +end + +class VCS + class NotFoundError < RuntimeError; end + + @@dirs = [] + def self.register(dir, &pred) + @@dirs << [dir, self, pred] + end + + def self.detect(path) + @@dirs.each do |dir, klass, pred| + curr = path + loop { + return klass.new(curr) if pred ? pred[curr, dir] : File.directory?(File.join(curr, dir)) + prev, curr = curr, File.realpath(File.join(curr, '..')) + break if curr == prev # stop at the root directory + } + end + raise VCS::NotFoundError, "does not seem to be under a vcs: #{path}" + end + + def self.local_path?(path) + String === path or path.respond_to?(:to_path) + end + + attr_reader :srcdir + + def initialize(path) + @srcdir = path + super() + end + + NullDevice = defined?(IO::NULL) ? IO::NULL : + %w[/dev/null NUL NIL: NL:].find {|dev| File.exist?(dev)} + + # return a pair of strings, the last revision and the last revision in which + # +path+ was modified. + def get_revisions(path) + if self.class.local_path?(path) + path = relative_to(path) + end + last, changed, modified, *rest = ( + begin + if NullDevice + save_stderr = STDERR.dup + STDERR.reopen NullDevice, 'w' + end + self.class.get_revisions(path, @srcdir) + rescue Errno::ENOENT => e + raise VCS::NotFoundError, e.message + ensure + if save_stderr + STDERR.reopen save_stderr + save_stderr.close + end + end + ) + last or raise VCS::NotFoundError, "last revision not found" + changed or raise VCS::NotFoundError, "changed revision not found" + if modified + /\A(\d+)-(\d+)-(\d+)\D(\d+):(\d+):(\d+(?:\.\d+)?)\s*(?:Z|([-+]\d\d)(\d\d))\z/ =~ modified or + raise "unknown time format - #{modified}" + match = $~[1..6].map { |x| x.to_i } + off = $7 ? "#{$7}:#{$8}" : "+00:00" + match << off + begin + modified = Time.new(*match) + rescue ArgumentError + modified = Time.utc(*$~[1..6]) + $7.to_i * 3600 + $8.to_i * 60 + end + end + return last, changed, modified, *rest + end + + def modified(path) + _, _, modified, * = get_revisions(path) + modified + end + + def relative_to(path) + if path + srcdir = File.realpath(@srcdir) + path = File.realdirpath(path) + list1 = srcdir.split(%r{/}) + list2 = path.split(%r{/}) + while !list1.empty? && !list2.empty? && list1.first == list2.first + list1.shift + list2.shift + end + if list1.empty? && list2.empty? + "." + else + ([".."] * list1.length + list2).join("/") + end + else + '.' + end + end + + def after_export(dir) + end + + class SVN < self + register(".svn") + COMMAND = ENV['SVN'] || 'svn' + + def self.get_revisions(path, srcdir = nil) + if srcdir and local_path?(path) + path = File.join(srcdir, path) + end + if srcdir + info_xml = IO.pread(%W"#{COMMAND} info --xml #{srcdir}") + info_xml = nil unless info_xml[/<url>(.*)<\/url>/, 1] == path.to_s + end + info_xml ||= IO.pread(%W"#{COMMAND} info --xml #{path}") + _, last, _, changed, _ = info_xml.split(/revision="(\d+)"/) + modified = info_xml[/<date>([^<>]*)/, 1] + branch = info_xml[%r'<relative-url>\^/(?:branches/|tags/)?([^<>]+)', 1] + [last, changed, modified, branch] + end + + def self.search_root(path) + return unless local_path?(path) + parent = File.realpath(path) + begin + parent = File.dirname(wkdir = parent) + return wkdir if File.directory?(wkdir + "/.svn") + end until parent == wkdir + end + + def get_info + @info ||= IO.pread(%W"#{COMMAND} info --xml #{@srcdir}") + end + + def url + unless @url + url = get_info[/<root>(.*)<\/root>/, 1] + @url = URI.parse(url+"/") if url + end + @url + end + + def wcroot + unless @wcroot + info = get_info + @wcroot = info[/<wcroot-abspath>(.*)<\/wcroot-abspath>/, 1] + @wcroot ||= self.class.search_root(@srcdir) + end + @wcroot + end + + def branch(name) + url + "branches/#{name}" + end + + def tag(name) + url + "tags/#{name}" + end + + def trunk + url + "trunk" + end + + def branch_list(pat) + IO.popen(%W"#{COMMAND} ls #{branch('')}") do |f| + f.each do |line| + line.chomp! + line.chomp!('/') + yield(line) if File.fnmatch?(pat, line) + end + end + end + + def grep(pat, tag, *files, &block) + cmd = %W"#{COMMAND} cat" + files.map! {|n| File.join(tag, n)} if tag + set = block.binding.eval("proc {|match| $~ = match}") + IO.popen([cmd, *files]) do |f| + f.grep(pat) do |s| + set[$~] + yield s + end + end + end + + def export(revision, url, dir, keep_temp = false) + if @srcdir and (rootdir = wcroot) + srcdir = File.realpath(@srcdir) + rootdir << "/" + if srcdir.start_with?(rootdir) + subdir = srcdir[rootdir.size..-1] + subdir = nil if subdir.empty? + FileUtils.mkdir_p(svndir = dir+"/.svn") + FileUtils.ln_s(Dir.glob(rootdir+"/.svn/*"), svndir) + system(COMMAND, "-q", "revert", "-R", subdir || ".", :chdir => dir) or return false + FileUtils.rm_rf(svndir) unless keep_temp + if subdir + tmpdir = Dir.mktmpdir("tmp-co.", "#{dir}/#{subdir}") + File.rename(tmpdir, tmpdir = "#{dir}/#{File.basename(tmpdir)}") + FileUtils.mv(Dir.glob("#{dir}/#{subdir}/{.[^.]*,..?*,*}"), tmpdir) + begin + Dir.rmdir("#{dir}/#{subdir}") + end until (subdir = File.dirname(subdir)) == '.' + FileUtils.mv(Dir.glob("#{tmpdir}/#{subdir}/{.[^.]*,..?*,*}"), dir) + Dir.rmdir(tmpdir) + end + return true + end + end + IO.popen(%W"#{COMMAND} export -r #{revision} #{url} #{dir}") do |pipe| + pipe.each {|line| /^A/ =~ line or yield line} + end + $?.success? + end + + def after_export(dir) + FileUtils.rm_rf(dir+"/.svn") + end + + def export_changelog(url, from, to, path) + range = [to, (from+1 if from)].compact.join(':') + IO.popen({'TZ' => 'JST-9', 'LANG' => 'C', 'LC_ALL' => 'C'}, + %W"#{COMMAND} log -r#{range} #{url}") do |r| + open(path, 'w') do |w| + IO.copy_stream(r, w) + end + end + end + + def commit + system(*%W"#{COMMAND} commit") + end + end + + class GIT < self + register(".git") {|path, dir| File.exist?(File.join(path, dir))} + COMMAND = ENV["GIT"] || 'git' + + def self.cmd_args(cmds, srcdir = nil) + if srcdir and local_path?(srcdir) + (opts = cmds.last).kind_of?(Hash) or cmds << (opts = {}) + opts[:chdir] ||= srcdir + end + cmds + end + + def self.cmd_pipe_at(srcdir, cmds, &block) + IO.popen(*cmd_args(cmds, srcdir), &block) + end + + def self.cmd_read_at(srcdir, cmds) + IO.pread(*cmd_args(cmds, srcdir)) + end + + def self.get_revisions(path, srcdir = nil) + gitcmd = [COMMAND] + desc = cmd_read_at(srcdir, [gitcmd + %w[describe --tags --match REV_*]]) + if /\AREV_(\d+)(?:-(\d+)-g\h+)?\Z/ =~ desc + last = ($1.to_i + $2.to_i).to_s + end + logcmd = gitcmd + %W[log -n1 --date=iso] + logcmd << "--grep=^ *git-svn-id: .*@[0-9][0-9]*" unless last + idpat = /git-svn-id: .*?@(\d+) \S+\Z/ + log = cmd_read_at(srcdir, [logcmd]) + commit = log[/\Acommit (\w+)/, 1] + last ||= log[idpat, 1] + if path + cmd = logcmd + cmd += [path] unless path == '.' + log = cmd_read_at(srcdir, [cmd]) + changed = log[idpat, 1] || last + else + changed = last + end + modified = log[/^Date:\s+(.*)/, 1] + branch = cmd_read_at(srcdir, [gitcmd + %W[symbolic-ref HEAD]])[%r'\A(?:refs/heads/)?(.+)', 1] + title = cmd_read_at(srcdir, [gitcmd + %W[log --format=%s -n1 #{commit}..HEAD]]) + title = nil if title.empty? + [last, changed, modified, branch, title] + end + + def initialize(*) + super + if srcdir = @srcdir and self.class.local_path?(srcdir) + @srcdir = File.realpath(srcdir) + end + self + end + + def cmd_pipe(*cmds, &block) + self.class.cmd_pipe_at(@srcdir, cmds, &block) + end + + def cmd_read(*cmds) + self.class.cmd_read_at(@srcdir, cmds) + end + + Branch = Struct.new(:to_str) + + def branch(name) + Branch.new(name) + end + + alias tag branch + + def trunk + branch("trunk") + end + + def stable + cmd = %W"#{COMMAND} for-each-ref --format=\%(refname:short) refs/heads/ruby_[0-9]*" + branch(cmd_read(cmd)[/.*^(ruby_\d+_\d+)$/m, 1]) + end + + def branch_list(pat) + cmd = %W"#{COMMAND} for-each-ref --format=\%(refname:short) refs/heads/#{pat}" + cmd_pipe(cmd) {|f| + f.each {|line| + line.chomp! + yield line + } + } + end + + def grep(pat, tag, *files, &block) + cmd = %W[#{COMMAND} grep -h --perl-regexp #{tag} --] + set = block.binding.eval("proc {|match| $~ = match}") + cmd_pipe(cmd+files) do |f| + f.grep(pat) do |s| + set[$~] + yield s + end + end + end + + def export(revision, url, dir, keep_temp = false) + ret = system(COMMAND, "clone", "-s", (@srcdir || '.').to_s, "-b", url, dir) + FileUtils.rm_rf("#{dir}/.git") if ret and !keep_temp + ret + end + + def after_export(dir) + FileUtils.rm_rf(Dir.glob("#{dir}/.git*")) + end + + def export_changelog(url, from, to, path) + range = [from, to].map do |rev| + rev or next + rev = cmd_read({'LANG' => 'C', 'LC_ALL' => 'C'}, + %W"#{COMMAND} log -n1 --format=format:%H" << + "--grep=^ *git-svn-id: .*@#{rev} ") + rev unless rev.empty? + end.join('..') + cmd_pipe({'TZ' => 'JST-9', 'LANG' => 'C', 'LC_ALL' => 'C'}, + %W"#{COMMAND} log --date=iso-local --topo-order #{range}", "rb") do |r| + open(path, 'w') do |w| + sep = "-"*72 + w.puts sep + while s = r.gets('') + author = s[/^Author:\s*(\S+)/, 1] + time = s[/^Date:\s*(.+)/, 1] + s = r.gets('') + s.gsub!(/^ {4}/, '') + s.sub!(/^git-svn-id: .*@(\d+) .*\n+\z/, '') + rev = $1 + s.gsub!(/^ {8}/, '') if /^(?! {8}|$)/ !~ s + if /\A(\d+)-(\d+)-(\d+)/ =~ time + date = Time.new($1.to_i, $2.to_i, $3.to_i).strftime("%a, %d %b %Y") + end + w.puts "r#{rev} | #{author} | #{time} (#{date}) | #{s.count("\n")} lines\n\n" + w.puts s, sep + end + end + end + end + + def commit + rev = cmd_read(%W"#{COMMAND} svn info"+[STDERR=>[:child, :out]])[/^Last Changed Rev: (\d+)/, 1] + com = cmd_read(%W"#{COMMAND} svn find-rev r#{rev}").chomp + + # TODO: dcommit necessary commits only with --add-author-from + same = true + cmd_pipe([COMMAND, "log", "--format=%ae %ce", "#{com}..@"], "rb") do |r| + r.each do |l| + same &&= /^(\S+) +\1$/ =~ l + end + end + ret = system(COMMAND, "svn", "dcommit", *(["--add-author-from"] unless same)) + + if ret and rev + old = [cmd_read(%W"#{COMMAND} log -1 --format=%H").chomp] + old << cmd_read(%W"#{COMMAND} svn reset -r#{rev}")[/^r#{rev} = (\h+)/, 1] + 3.times do + sleep 2 + system(*%W"#{COMMAND} pull --no-edit --rebase") + break unless old.include?(cmd_read(%W"#{COMMAND} log -1 --format=%H").chomp) + end + end + ret + end + end +end diff --git a/tool/vpath.rb b/tool/vpath.rb new file mode 100644 index 0000000000..48ab148405 --- /dev/null +++ b/tool/vpath.rb @@ -0,0 +1,87 @@ +# -*- coding: us-ascii -*- + +class VPath + attr_accessor :separator + + def initialize(*list) + @list = list + @additional = [] + @separator = nil + end + + def inspect + list.inspect + end + + def search(meth, base, *rest) + begin + meth.call(base, *rest) + rescue Errno::ENOENT => error + list.each do |dir| + return meth.call(File.join(dir, base), *rest) rescue nil + end + raise error + end + end + + def process(*args, &block) + search(File.method(__callee__), *args, &block) + end + + alias stat process + alias lstat process + + def open(*args) + f = search(File.method(:open), *args) + if block_given? + begin + yield f + ensure + f.close unless f.closed? + end + else + f + end + end + + def read(*args) + open(*args) {|f| f.read} + end + + def foreach(file, *args, &block) + open(file) {|f| f.each(*args, &block)} + end + + def def_options(opt) + opt.on("-I", "--srcdir=DIR", "add a directory to search path") {|dir| + @additional << dir + } + opt.on("-L", "--vpath=PATH LIST", "add directories to search path") {|dirs| + @additional << [dirs] + } + opt.on("--path-separator=SEP", /\A(?:\W\z|\.(\W).+)/, "separator for vpath") {|sep, vsep| + # hack for msys make. + @separator = vsep || sep + } + end + + def list + @additional.reject! do |dirs| + case dirs + when String + @list << dirs + when Array + raise "--path-separator option is needed for vpath list" unless @separator + # @separator ||= (require 'rbconfig'; RbConfig::CONFIG["PATH_SEPARATOR"]) + @list.concat(dirs[0].split(@separator)) + end + true + end + @list + end + + def strip(path) + prefix = list.map {|dir| Regexp.quote(dir)} + path.sub(/\A#{prefix.join('|')}(?:\/|\z)/, '') + end +end diff --git a/tool/vtlh.rb b/tool/vtlh.rb new file mode 100644 index 0000000000..2e1faf2ce8 --- /dev/null +++ b/tool/vtlh.rb @@ -0,0 +1,17 @@ +# Convert addresses to line numbers for MiniRuby. + +# ARGF = open('ha') +cd = `pwd`.chomp + '/' +ARGF.each{|line| + if /^0x([a-z0-9]+),/ =~ line + stat = line.split(',') + addr = stat[0].hex + 0x00400000 + retired = stat[2].to_i + ticks = stat[3].to_i + + src = `addr2line -e miniruby.exe #{addr.to_s(16)}`.chomp + src.sub!(cd, '') + puts '%-40s 0x%08x %8d %8d' % [src, addr, retired, ticks] + end +} + diff --git a/tool/ytab.sed b/tool/ytab.sed new file mode 100755 index 0000000000..a3b411aa8f --- /dev/null +++ b/tool/ytab.sed @@ -0,0 +1,61 @@ +#!/bin/sed -f +# This file is used when generating code for the Ruby parser. +/^int yydebug;/{ +i\ +#ifndef yydebug +a\ +#endif +} +/^extern int yydebug;/{ +i\ +#ifndef yydebug +a\ +#endif +} +/^yydestruct.*yymsg/,/#endif/{ + /^yydestruct/{ + /parser/!{ + H + s/^/ruby_parser_&/ + s/)$/, parser)/ + /\*/s/parser)$/struct parser_params *&/ + } + } + /^#endif/{ + x + /yydestruct/{ + i\ + struct parser_params *parser; + a\ +#define yydestruct(m, t, v) ruby_parser_yydestruct(m, t, v, parser) + } + x + } +} +/^yy_stack_print/{ + /parser/!{ + H + s/)$/, parser)/ + /\*/s/parser)$/struct parser_params *&/ + } +} +/yy_stack_print.*;/{ + x + /yy_stack_print/{ + x + s/\(yy_stack_print *\)(\(.*\));/\1(\2, parser);/ + x + } + x +} +/^yy_reduce_print/,/^}/{ + s/fprintf *(stderr,/YYFPRINTF (parser,/g +} +s/\( YYFPRINTF *(\)yyoutput,/\1parser,/ +s/\( YYFPRINTF *(\)yyo,/\1parser,/ +s/\( YYFPRINTF *(\)stderr,/\1parser,/ +s/\( YYDPRINTF *((\)stderr,/\1parser,/ +s/^\([ ]*\)\(yyerror[ ]*([ ]*parser,\)/\1parser_\2/ +s!^ *extern char \*getenv();!/* & */! +s/^\(#.*\)".*\.tab\.c"/\1"parse.c"/ +/^\(#.*\)".*\.y"/s:\\\\:/:g |