diff options
Diffstat (limited to 'tool/downloader.rb')
| -rw-r--r-- | tool/downloader.rb | 132 |
1 files changed, 104 insertions, 28 deletions
diff --git a/tool/downloader.rb b/tool/downloader.rb index e9ea00c4c4..3a91ea0b93 100644 --- a/tool/downloader.rb +++ b/tool/downloader.rb @@ -36,6 +36,12 @@ else end class Downloader + def self.find(dlname) + constants.find do |name| + return const_get(name) if dlname.casecmp(name.to_s) == 0 + end + end + def self.https=(https) @@https = https end @@ -48,13 +54,18 @@ class Downloader @@https end + def self.get_option(argv, options) + false + end + class GNU < self def self.download(name, *rest) if https? begin super("https://cdn.jsdelivr.net/gh/gcc-mirror/gcc@master/#{name}", name, *rest) rescue => e - STDERR.puts "Download failed (#{e.message}), try another URL" + m1, m2 = e.message.split("\n", 2) + STDERR.puts "Download failed (#{m1}), try another URL\n#{m2}" super("https://raw.githubusercontent.com/gcc-mirror/gcc/master/#{name}", name, *rest) end else @@ -68,6 +79,9 @@ class Downloader require 'rubygems' options = options.dup options[:ssl_ca_cert] = Dir.glob(File.expand_path("../lib/rubygems/ssl_certs/**/*.pem", File.dirname(__FILE__))) + if Gem::Version.new(name[/-\K[^-]*(?=\.gem\z)/]).prerelease? + options[:ignore_http_client_errors] = true + end super("https://rubygems.org/downloads/#{name}", name, dir, since, options) end end @@ -78,6 +92,21 @@ class Downloader INDEX = {} # cache index file information across files in the same directory UNICODE_PUBLIC = "https://www.unicode.org/Public/" + def self.get_option(argv, options) + case argv[0] + when '--unicode-beta' + options[:unicode_beta] = argv[1] + argv.shift(2) + true + when /\A--unicode-beta=(.*)/m + options[:unicode_beta] = $1 + argv.shift + true + else + super + end + end + def self.download(name, dir = nil, since = true, options = {}) options = options.dup unicode_beta = options.delete(:unicode_beta) @@ -173,7 +202,6 @@ class Downloader options = options.dup url = URI(url) dryrun = options.delete(:dryrun) - options.delete(:unicode_beta) # just to be on the safe side for gems and gcc if name file = Pathname.new(under(dir, name)) @@ -211,17 +239,30 @@ class Downloader $stdout.print "downloading #{name} ... " $stdout.flush end + mtime = nil + ignore_http_client_errors = options.delete(:ignore_http_client_errors) + options = options.merge(http_options(file, since.nil? ? true : since)) begin data = with_retry(10) do - url.read(options.merge(http_options(file, since.nil? ? true : since))) + data = url.read(options) + if mtime = data.meta["last-modified"] + mtime = Time.httpdate(mtime) + end + data end rescue OpenURI::HTTPError => http_error - if http_error.message =~ /^304 / # 304 Not Modified + case http_error.message + when /^304 / # 304 Not Modified if $VERBOSE $stdout.puts "#{name} not modified" $stdout.flush end return file.to_path + when /^40/ # Net::HTTPClientError: 403 Forbidden, 404 Not Found + if ignore_http_client_errors + puts "Ignore #{url}: #{http_error.message}" + return file.to_path + end end raise rescue Timeout::Error @@ -237,16 +278,14 @@ class Downloader end raise end - mtime = nil dest = (cache_save && cache && !cache.exist? ? cache : file) dest.parent.mkpath + dest.unlink if dest.symlink? && !dest.exist? dest.open("wb", 0600) do |f| f.write(data) f.chmod(mode_for(data)) - mtime = data.meta["last-modified"] end if mtime - mtime = httpdate(mtime) dest.utime(mtime, mtime) end if $VERBOSE @@ -285,7 +324,16 @@ class Downloader return true if cache.eql?(file) if /cygwin/ !~ RUBY_PLATFORM or /winsymlink:nativestrict/ =~ ENV['CYGWIN'] begin - file.make_symlink(cache.relative_path_from(file.parent)) + link = cache.relative_path_from(file.parent) + rescue ArgumentError + abs = cache.expand_path + link = abs.relative_path_from(file.parent.expand_path) + if link.to_s.count("/") > abs.to_s.count("/") + link = abs + end + end + begin + file.make_symlink(link) rescue SystemCallError else if verbose @@ -328,7 +376,7 @@ class Downloader times = 0 begin block.call - rescue Errno::ETIMEDOUT, SocketError, OpenURI::HTTPError, Net::ReadTimeout, Net::OpenTimeout => e + rescue Errno::ETIMEDOUT, SocketError, OpenURI::HTTPError, Net::ReadTimeout, Net::OpenTimeout, ArgumentError => e raise if e.is_a?(OpenURI::HTTPError) && e.message !~ /^50[023] / # retry only 500, 502, 503 for http error times += 1 if times <= max_times @@ -348,45 +396,73 @@ Downloader.https = https.freeze if $0 == __FILE__ since = true options = {} + dl = nil + (args = []).singleton_class.__send__(:define_method, :downloader?) do |arg| + !dl and args.empty? and (dl = Downloader.find(arg)) + end until ARGV.empty? + if ARGV[0] == '--' + ARGV.shift + break if ARGV.empty? + ARGV.shift if args.downloader? ARGV[0] + args.concat(ARGV) + break + end + + if dl and dl.get_option(ARGV, options) + # the downloader dealt with the arguments, and should be removed + # from ARGV. + next + end + case ARGV[0] - when '-d' + when '-d', '--destdir' + ## -d, --destdir DIRECTORY Download into the directory destdir = ARGV[1] ARGV.shift - when '-p' - # strip directory names from the name to download, and add the - # prefix instead. + when '-p', '--prefix' + ## -p, --prefix Strip directory names from the name to download, + ## and add the prefix instead. prefix = ARGV[1] ARGV.shift - when '-e' + when '-e', '--exist', '--non-existent-only' + ## -e, --exist, --non-existent-only Skip already existent files. since = nil - when '-a' + when '-a', '--always' + ## -a, --always Download all files. since = false - when '-n', '--dryrun' + when '-u', '--update', '--if-modified' + ## -u, --update, --if-modified Download newer files only. + since = true + when '-n', '--dry-run', '--dryrun' + ## -n, --dry-run Do not download actually. options[:dryrun] = true when '--cache-dir' + ## --cache-dir DIRECTORY Cache downloaded files in the directory. options[:cache_dir] = ARGV[1] ARGV.shift - when '--unicode-beta' - options[:unicode_beta] = ARGV[1] - ARGV.shift when /\A--cache-dir=(.*)/m options[:cache_dir] = $1 + when /\A--help\z/ + ## --help Print this message + puts "Usage: #$0 [options] relative-url..." + File.foreach(__FILE__) do |line| + line.sub!(/^ *## /, "") or next + break if line.chomp!.empty? + opt, desc = line.split(/ {2,}/, 2) + printf " %-28s %s\n", opt, desc + end + exit when /\A-/ abort "#{$0}: unknown option #{ARGV[0]}" else - break + args << ARGV[0] unless args.downloader? ARGV[0] end ARGV.shift end - dl = Downloader.constants.find do |name| - ARGV[0].casecmp(name.to_s) == 0 - end unless ARGV.empty? $VERBOSE = true if dl - dl = Downloader.const_get(dl) - ARGV.shift - ARGV.each do |name| + args.each do |name| dir = destdir if prefix name = name.sub(/\A\.\//, '') @@ -406,7 +482,7 @@ if $0 == __FILE__ dl.download(name, dir, since, options) end else - abort "usage: #{$0} url name" unless ARGV.size == 2 - Downloader.download(ARGV[0], ARGV[1], destdir, since, options) + abort "usage: #{$0} url name" unless args.size == 2 + Downloader.download(args[0], args[1], destdir, since, options) end end |