From 9c66bad9f3d522d50d4a45ef8a3a92abbf93229f Mon Sep 17 00:00:00 2001 From: drbrain Date: Thu, 15 Nov 2012 21:59:37 +0000 Subject: * lib/rake*: Updated to rake 0.9.3 * test/rake*: ditto * bin/rake: ditto * NEWS: ditto git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@37664 b2dd03c8-39d4-4d8f-98ff-823fe69b080e --- lib/rake.rb | 1 + lib/rake/application.rb | 384 +++++++++++++++++++++++-------------- lib/rake/backtrace.rb | 18 ++ lib/rake/clean.rb | 2 +- lib/rake/cloneable.rb | 23 +-- lib/rake/contrib/ftptools.rb | 3 +- lib/rake/contrib/sys.rb | 12 +- lib/rake/dsl_definition.rb | 20 +- lib/rake/ext/module.rb | 2 +- lib/rake/ext/string.rb | 3 +- lib/rake/ext/time.rb | 3 +- lib/rake/file_list.rb | 4 +- lib/rake/file_utils_ext.rb | 7 +- lib/rake/multi_task.rb | 7 +- lib/rake/phony.rb | 13 ++ lib/rake/private_reader.rb | 20 ++ lib/rake/promise.rb | 99 ++++++++++ lib/rake/rake_module.rb | 15 ++ lib/rake/rdoctask.rb | 2 +- lib/rake/runtest.rb | 2 +- lib/rake/task.rb | 36 +++- lib/rake/task_arguments.rb | 2 +- lib/rake/task_manager.rb | 2 +- lib/rake/testtask.rb | 6 +- lib/rake/thread_history_display.rb | 48 +++++ lib/rake/thread_pool.rb | 155 +++++++++++++++ lib/rake/version.rb | 10 +- 27 files changed, 699 insertions(+), 200 deletions(-) create mode 100644 lib/rake/backtrace.rb create mode 100644 lib/rake/phony.rb create mode 100644 lib/rake/private_reader.rb create mode 100644 lib/rake/promise.rb create mode 100644 lib/rake/thread_history_display.rb create mode 100644 lib/rake/thread_pool.rb (limited to 'lib') diff --git a/lib/rake.rb b/lib/rake.rb index fc1a6a5165..8180a5e996 100644 --- a/lib/rake.rb +++ b/lib/rake.rb @@ -58,6 +58,7 @@ require 'rake/early_time' require 'rake/name_space' require 'rake/task_manager' require 'rake/application' +require 'rake/backtrace' $trace = false diff --git a/lib/rake/application.rb b/lib/rake/application.rb index 2079fb9be6..50930938bf 100644 --- a/lib/rake/application.rb +++ b/lib/rake/application.rb @@ -2,10 +2,14 @@ require 'shellwords' require 'optparse' require 'rake/task_manager' +require 'rake/thread_pool' +require 'rake/thread_history_display' require 'rake/win32' module Rake + CommandLineOptionError = Class.new(StandardError) + ###################################################################### # Rake main application object. When invoking +rake+ from the # command line, a Rake::Application object is created and run. @@ -54,7 +58,7 @@ module Rake # # * Initialize the command line options (+init+). # * Define the tasks (+load_rakefile+). - # * Run the top level tasks (+run_tasks+). + # * Run the top level tasks (+top_level+). # # If you wish to build a custom rake command, you should call # +init+ on your application. Then define any tasks. Finally, @@ -85,7 +89,7 @@ module Rake # Run the top level tasks of a Rake application. def top_level - standard_exception_handling do + run_with_threads do if options.show_tasks display_tasks_and_comments elsif options.show_prereqs @@ -96,6 +100,21 @@ module Rake end end + # Run the given block with the thread startup and shutdown. + def run_with_threads + thread_pool.gather_history if options.job_stats == :history + + yield + + thread_pool.join + if options.job_stats + stats = thread_pool.statistics + puts "Maximum active threads: #{stats[:max_active_threads]}" + puts "Total threads in play: #{stats[:total_threads_in_play]}" + end + ThreadHistoryDisplay.new(thread_pool.history).show if options.job_stats == :history + end + # Add a loader to handle imported files ending in the extension # +ext+. def add_loader(ext, loader) @@ -108,6 +127,11 @@ module Rake @options ||= OpenStruct.new end + # Return the thread pool used for multithreaded processing. + def thread_pool # :nodoc: + @thread_pool ||= ThreadPool.new(options.thread_pool_size||FIXNUM_MAX) + end + # private ---------------------------------------------------------------- def invoke_task(task_string) @@ -146,15 +170,15 @@ module Rake # Display the error message that caused the exception. def display_error_message(ex) - $stderr.puts "#{name} aborted!" - $stderr.puts ex.message - if options.trace - $stderr.puts ex.backtrace.join("\n") + trace "#{name} aborted!" + trace ex.message + if options.backtrace + trace ex.backtrace.join("\n") else - $stderr.puts rakefile_location(ex.backtrace) + trace Backtrace.collapse(ex.backtrace) end - $stderr.puts "Tasks: #{ex.chain}" if has_chain?(ex) - $stderr.puts "(See full trace by running task with --trace)" unless options.trace + trace "Tasks: #{ex.chain}" if has_chain?(ex) + trace "(See full trace by running task with --trace)" unless options.backtrace end # Warn about deprecated usage. @@ -180,7 +204,7 @@ module Rake def have_rakefile @rakefiles.each do |fn| if File.exist?(fn) - others = Dir.glob(fn, File::FNM_CASEFOLD) + others = Rake.glob(fn, File::FNM_CASEFOLD) return others.size == 1 ? others.first : fn elsif fn == '' return fn @@ -208,7 +232,7 @@ module Rake # Display the tasks and comments. def display_tasks_and_comments displayable_tasks = tasks.select { |t| - t.comment && t.name =~ options.show_task_pattern + (options.show_all_tasks || t.comment) && t.name =~ options.show_task_pattern } case options.show_tasks when :tasks @@ -222,7 +246,8 @@ module Rake when :describe displayable_tasks.each do |t| puts "#{name} #{t.name_with_args}" - t.full_comment.split("\n").each do |line| + comment = t.full_comment || "" + comment.split("\n").each do |line| puts " #{line}" end puts @@ -271,7 +296,9 @@ module Rake end def truncate(string, width) - if string.length <= width + if string.nil? + "" + elsif string.length <= width string else ( string[0, width-3] || "" ) + "..." @@ -286,141 +313,214 @@ module Rake end end + def trace(*str) + options.trace_output ||= $stderr + options.trace_output.puts(*str) + end + + def sort_options(options) + options.sort_by { |opt| + opt.select { |o| o =~ /^-/ }.map { |o| o.downcase }.sort.reverse + } + end + private :sort_options + # A list of all the standard options used in rake, suitable for # passing to OptionParser. def standard_rake_options - [ - ['--classic-namespace', '-C', "Put Task and FileTask in the top level namespace", - lambda { |value| - require 'rake/classic_namespace' - options.classic_namespace = true - } - ], - ['--describe', '-D [PATTERN]', "Describe the tasks (matching optional PATTERN), then exit.", - lambda { |value| - options.show_tasks = :describe - options.show_task_pattern = Regexp.new(value || '') - TaskManager.record_task_metadata = true - } - ], - ['--dry-run', '-n', "Do a dry run without executing actions.", - lambda { |value| - Rake.verbose(true) - Rake.nowrite(true) - options.dryrun = true - options.trace = true - } - ], - ['--execute', '-e CODE', "Execute some Ruby code and exit.", - lambda { |value| - eval(value) - exit - } - ], - ['--execute-print', '-p CODE', "Execute some Ruby code, print the result, then exit.", - lambda { |value| - puts eval(value) - exit - } - ], - ['--execute-continue', '-E CODE', - "Execute some Ruby code, then continue with normal task processing.", - lambda { |value| eval(value) } - ], - ['--libdir', '-I LIBDIR', "Include LIBDIR in the search path for required modules.", - lambda { |value| $:.push(value) } - ], - ['--no-search', '--nosearch', '-N', "Do not search parent directories for the Rakefile.", - lambda { |value| options.nosearch = true } - ], - ['--prereqs', '-P', "Display the tasks and dependencies, then exit.", - lambda { |value| options.show_prereqs = true } - ], - ['--quiet', '-q', "Do not log messages to standard output.", - lambda { |value| Rake.verbose(false) } - ], - ['--rakefile', '-f [FILE]', "Use FILE as the rakefile.", - lambda { |value| - value ||= '' - @rakefiles.clear - @rakefiles << value - } - ], - ['--rakelibdir', '--rakelib', '-R RAKELIBDIR', - "Auto-import any .rake files in RAKELIBDIR. (default is 'rakelib')", - # HACK Use File::PATH_SEPARATOR - lambda { |value| options.rakelib = value.split(':') } - ], - ['--require', '-r MODULE', "Require MODULE before executing rakefile.", - lambda { |value| - begin - require value - rescue LoadError => ex + sort_options( + [ + ['--all', '-A', "Show all tasks, even uncommented ones", + lambda { |value| + options.show_all_tasks = value + } + ], + ['--backtrace [OUT]', "Enable full backtrace. OUT can be stderr (default) or stdout.", + lambda { |value| + options.backtrace = true + select_trace_output(options, 'backtrace', value) + } + ], + ['--classic-namespace', '-C', "Put Task and FileTask in the top level namespace", + lambda { |value| + require 'rake/classic_namespace' + options.classic_namespace = true + } + ], + ['--comments', "Show commented tasks only", + lambda { |value| + options.show_all_tasks = !value + } + ], + ['--describe', '-D [PATTERN]', "Describe the tasks (matching optional PATTERN), then exit.", + lambda { |value| + select_tasks_to_show(options, :describe, value) + } + ], + ['--dry-run', '-n', "Do a dry run without executing actions.", + lambda { |value| + Rake.verbose(true) + Rake.nowrite(true) + options.dryrun = true + options.trace = true + } + ], + ['--execute', '-e CODE', "Execute some Ruby code and exit.", + lambda { |value| + eval(value) + exit + } + ], + ['--execute-print', '-p CODE', "Execute some Ruby code, print the result, then exit.", + lambda { |value| + puts eval(value) + exit + } + ], + ['--execute-continue', '-E CODE', + "Execute some Ruby code, then continue with normal task processing.", + lambda { |value| eval(value) } + ], + ['--jobs', '-j [NUMBER]', + "Specifies the maximum number of tasks to execute in parallel. (default:2)", + lambda { |value| options.thread_pool_size = [(value || 2).to_i,2].max } + ], + ['--job-stats [LEVEL]', + "Display job statistics. LEVEL=history displays a complete job list", + lambda { |value| + if value =~ /^history/i + options.job_stats = :history + else + options.job_stats = true + end + } + ], + ['--libdir', '-I LIBDIR', "Include LIBDIR in the search path for required modules.", + lambda { |value| $:.push(value) } + ], + ['--multitask', '-m', "Treat all tasks as multitasks.", + lambda { |value| options.always_multitask = true } + ], + ['--no-search', '--nosearch', '-N', "Do not search parent directories for the Rakefile.", + lambda { |value| options.nosearch = true } + ], + ['--prereqs', '-P', "Display the tasks and dependencies, then exit.", + lambda { |value| options.show_prereqs = true } + ], + ['--quiet', '-q', "Do not log messages to standard output.", + lambda { |value| Rake.verbose(false) } + ], + ['--rakefile', '-f [FILE]', "Use FILE as the rakefile.", + lambda { |value| + value ||= '' + @rakefiles.clear + @rakefiles << value + } + ], + ['--rakelibdir', '--rakelib', '-R RAKELIBDIR', + "Auto-import any .rake files in RAKELIBDIR. (default is 'rakelib')", + lambda { |value| options.rakelib = value.split(File::PATH_SEPARATOR) } + ], + ['--reduce-compat', "Remove DSL in Object; remove Module#const_missing which defines ::Task etc.", + # Load-time option. + # Handled in bin/rake where Rake::REDUCE_COMPAT is defined (or not). + lambda { |_| } + ], + ['--require', '-r MODULE', "Require MODULE before executing rakefile.", + lambda { |value| begin - rake_require value - rescue LoadError - raise ex + require value + rescue LoadError => ex + begin + rake_require value + rescue LoadError + raise ex + end end - end - } - ], - ['--rules', "Trace the rules resolution.", - lambda { |value| options.trace_rules = true } - ], - ['--silent', '-s', "Like --quiet, but also suppresses the 'in directory' announcement.", - lambda { |value| - Rake.verbose(false) - options.silent = true - } - ], - ['--system', '-g', - "Using system wide (global) rakefiles (usually '~/.rake/*.rake').", - lambda { |value| options.load_system = true } - ], - ['--no-system', '--nosystem', '-G', - "Use standard project Rakefile search paths, ignore system wide rakefiles.", - lambda { |value| options.ignore_system = true } - ], - ['--tasks', '-T [PATTERN]', "Display the tasks (matching optional PATTERN) with descriptions, then exit.", - lambda { |value| - options.show_tasks = :tasks - options.show_task_pattern = Regexp.new(value || '') - Rake::TaskManager.record_task_metadata = true - } - ], - ['--trace', '-t', "Turn on invoke/execute tracing, enable full backtrace.", - lambda { |value| - options.trace = true - Rake.verbose(true) - } - ], - ['--verbose', '-v', "Log message to standard output.", - lambda { |value| Rake.verbose(true) } - ], - ['--version', '-V', "Display the program version.", - lambda { |value| - puts "rake, version #{RAKEVERSION}" - exit - } - ], - ['--where', '-W [PATTERN]', "Describe the tasks (matching optional PATTERN), then exit.", - lambda { |value| - options.show_tasks = :lines - options.show_task_pattern = Regexp.new(value || '') - Rake::TaskManager.record_task_metadata = true - } - ], - ['--no-deprecation-warnings', '-X', "Disable the deprecation warnings.", - lambda { |value| - options.ignore_deprecate = true - } - ], - ] + } + ], + ['--rules', "Trace the rules resolution.", + lambda { |value| options.trace_rules = true } + ], + ['--silent', '-s', "Like --quiet, but also suppresses the 'in directory' announcement.", + lambda { |value| + Rake.verbose(false) + options.silent = true + } + ], + ['--suppress-backtrace PATTERN', "Suppress backtrace lines matching regexp PATTERN. Ignored if --trace is on.", + lambda { |value| + options.suppress_backtrace_pattern = Regexp.new(value) + } + ], + ['--system', '-g', + "Using system wide (global) rakefiles (usually '~/.rake/*.rake').", + lambda { |value| options.load_system = true } + ], + ['--no-system', '--nosystem', '-G', + "Use standard project Rakefile search paths, ignore system wide rakefiles.", + lambda { |value| options.ignore_system = true } + ], + ['--tasks', '-T [PATTERN]', "Display the tasks (matching optional PATTERN) with descriptions, then exit.", + lambda { |value| + select_tasks_to_show(options, :tasks, value) + } + ], + ['--trace', '-t [OUT]', "Turn on invoke/execute tracing, enable full backtrace. OUT can be stderr (default) or stdout.", + lambda { |value| + options.trace = true + options.backtrace = true + select_trace_output(options, 'trace', value) + Rake.verbose(true) + } + ], + ['--verbose', '-v', "Log message to standard output.", + lambda { |value| Rake.verbose(true) } + ], + ['--version', '-V', "Display the program version.", + lambda { |value| + puts "rake, version #{RAKEVERSION}" + exit + } + ], + ['--where', '-W [PATTERN]', "Describe the tasks (matching optional PATTERN), then exit.", + lambda { |value| + select_tasks_to_show(options, :lines, value) + options.show_all_tasks = true + } + ], + ['--no-deprecation-warnings', '-X', "Disable the deprecation warnings.", + lambda { |value| + options.ignore_deprecate = true + } + ], + ]) + end + + def select_tasks_to_show(options, show_tasks, value) + options.show_tasks = show_tasks + options.show_task_pattern = Regexp.new(value || '') + Rake::TaskManager.record_task_metadata = true + end + private :select_tasks_to_show + + def select_trace_output(options, trace_option, value) + value = value.strip unless value.nil? + case value + when 'stdout' + options.trace_output = $stdout + when 'stderr', nil + options.trace_output = $stderr + else + fail CommandLineOptionError, "Unrecognized --#{trace_option} option '#{value}'" + end end + private :select_trace_output # Read and handle the command line options. def handle_options options.rakelib = ['rakelib'] + options.trace_output = $stderr OptionParser.new do |opts| opts.banner = "rake [-f rakefile] {options} targets..." @@ -509,7 +609,7 @@ module Rake end def glob(path, &block) - Dir[path.gsub("\\", '/')].each(&block) + Rake.glob(path.gsub("\\", '/')).each(&block) end private :glob @@ -583,7 +683,7 @@ module Rake @const_warning = true end - def rakefile_location backtrace = caller + def rakefile_location(backtrace=caller) backtrace.map { |t| t[/([^:]+):/,1] } re = /^#{@rakefile}$/ @@ -591,5 +691,9 @@ module Rake backtrace.find { |str| str =~ re } || '' end + + private + FIXNUM_MAX = (2**(0.size * 8 - 2) - 1) # :nodoc: + end end diff --git a/lib/rake/backtrace.rb b/lib/rake/backtrace.rb new file mode 100644 index 0000000000..038ca57906 --- /dev/null +++ b/lib/rake/backtrace.rb @@ -0,0 +1,18 @@ +module Rake + module Backtrace + SUPPRESSED_PATHS = + RbConfig::CONFIG.values_at(*RbConfig::CONFIG. + keys.grep(/(prefix|libdir)/)) + [ + File.join(File.dirname(__FILE__), ".."), + ].map { |f| Regexp.quote(File.expand_path(f)) } + SUPPRESSED_PATHS.reject! { |s| s.nil? || s =~ /^ *$/ } + + SUPPRESS_PATTERN = %r!(\A#{SUPPRESSED_PATHS.join('|')}|bin/rake:\d+)!i + + def self.collapse(backtrace) + pattern = Rake.application.options.suppress_backtrace_pattern || + SUPPRESS_PATTERN + backtrace.reject { |elem| elem =~ pattern } + end + end +end diff --git a/lib/rake/clean.rb b/lib/rake/clean.rb index 5c9cbcdb24..32846d4a6d 100644 --- a/lib/rake/clean.rb +++ b/lib/rake/clean.rb @@ -16,7 +16,7 @@ require 'rake' # :stopdoc: CLEAN = Rake::FileList["**/*~", "**/*.bak", "**/core"] CLEAN.clear_exclude.exclude { |fn| - fn.pathmap("%f") == 'core' && File.directory?(fn) + fn.pathmap("%f").downcase == 'core' && File.directory?(fn) } desc "Remove any temporary products." diff --git a/lib/rake/cloneable.rb b/lib/rake/cloneable.rb index 19c780bff6..ac67471232 100644 --- a/lib/rake/cloneable.rb +++ b/lib/rake/cloneable.rb @@ -3,23 +3,14 @@ module Rake # Mixin for creating easily cloned objects. # module Cloneable - # Clone an object by making a new object and setting all the instance - # variables to the same values. - def dup - sibling = self.class.new - instance_variables.each do |ivar| - value = self.instance_variable_get(ivar) - new_value = value.clone rescue value - sibling.instance_variable_set(ivar, new_value) + # The hook that invoked by 'clone' and 'dup' methods. + def initialize_copy(source) + super + source.instance_variables.each do |var| + src_value = source.instance_variable_get(var) + value = src_value.clone rescue src_value + instance_variable_set(var, value) end - sibling.taint if tainted? - sibling - end - - def clone - sibling = dup - sibling.freeze if frozen? - sibling end end end diff --git a/lib/rake/contrib/ftptools.rb b/lib/rake/contrib/ftptools.rb index 78420c7412..eaf8885262 100644 --- a/lib/rake/contrib/ftptools.rb +++ b/lib/rake/contrib/ftptools.rb @@ -127,7 +127,8 @@ module Rake # :nodoc: # Upload all files matching +wildcard+ to the uploader's root # path. def upload_files(wildcard) - Dir[wildcard].each do |fn| + fail "OUCH" + Rake.glob(wildcard).each do |fn| upload(fn) end end diff --git a/lib/rake/contrib/sys.rb b/lib/rake/contrib/sys.rb index 41963f1fef..aefd4a1913 100644 --- a/lib/rake/contrib/sys.rb +++ b/lib/rake/contrib/sys.rb @@ -27,7 +27,7 @@ module Sys # Install all the files matching +wildcard+ into the +dest_dir+ # directory. The permission mode is set to +mode+. def install(wildcard, dest_dir, mode) - Dir[wildcard].each do |fn| + Rake.glob(wildcard).each do |fn| File.install(fn, dest_dir, mode, $verbose) end end @@ -81,7 +81,7 @@ module Sys # recursively delete directories. def delete(*wildcards) wildcards.each do |wildcard| - Dir[wildcard].each do |fn| + Rake.glob(wildcard).each do |fn| if File.directory?(fn) log "Deleting directory #{fn}" Dir.delete(fn) @@ -96,10 +96,10 @@ module Sys # Recursively delete all files and directories matching +wildcard+. def delete_all(*wildcards) wildcards.each do |wildcard| - Dir[wildcard].each do |fn| + Rake.glob(wildcard).each do |fn| next if ! File.exist?(fn) if File.directory?(fn) - Dir["#{fn}/*"].each do |subfn| + Rake.glob("#{fn}/*").each do |subfn| next if subfn=='.' || subfn=='..' delete_all(subfn) end @@ -161,7 +161,7 @@ module Sys # Perform a block with each file matching a set of wildcards. def for_files(*wildcards) wildcards.each do |wildcard| - Dir[wildcard].each do |fn| + Rake.glob(wildcard).each do |fn| yield(fn) end end @@ -172,7 +172,7 @@ module Sys private # ---------------------------------------------------------- def for_matching_files(wildcard, dest_dir) - Dir[wildcard].each do |fn| + Rake.glob(wildcard).each do |fn| dest_file = File.join(dest_dir, fn) parent = File.dirname(dest_file) makedirs(parent) if ! File.directory?(parent) diff --git a/lib/rake/dsl_definition.rb b/lib/rake/dsl_definition.rb index 6d9a6b88f3..143b0bcf7a 100644 --- a/lib/rake/dsl_definition.rb +++ b/lib/rake/dsl_definition.rb @@ -52,8 +52,8 @@ module Rake # Declare a file creation task. # (Mainly used for the directory command). - def file_create(args, &block) - Rake::FileCreationTask.define_task(args, &block) + def file_create(*args, &block) + Rake::FileCreationTask.define_task(*args, &block) end # Declare a set of files tasks to create the given directories on @@ -62,12 +62,15 @@ module Rake # Example: # directory "testdata/doc" # - def directory(dir) + def directory(*args, &block) + result = file_create(*args, &block) + dir, _ = *Rake.application.resolve_args(args) Rake.each_dir_parent(dir) do |d| file_create d do |t| mkdir_p t.name if ! File.exist?(t.name) end end + result end # Declare a task that performs its prerequisites in @@ -78,8 +81,8 @@ module Rake # Example: # multitask :deploy => [:deploy_gem, :deploy_rdoc] # - def multitask(args, &block) - Rake::MultiTask.define_task(args, &block) + def multitask(*args, &block) + Rake::MultiTask.define_task(*args, &block) end # Create a new rake namespace and use it for evaluating the given @@ -167,10 +170,13 @@ module Rake private :#{name} }, __FILE__, line end - end + end unless defined? Rake::REDUCE_COMPAT extend FileUtilsExt end +# Extend the main object with the DSL commands. This allows top-level +# calls to task, etc. to work from a Rakefile without polluting the +# object inheritance tree. self.extend Rake::DSL -include Rake::DeprecatedObjectDSL +include Rake::DeprecatedObjectDSL unless defined? Rake::REDUCE_COMPAT diff --git a/lib/rake/ext/module.rb b/lib/rake/ext/module.rb index 3f64aef6c8..fc61bea555 100644 --- a/lib/rake/ext/module.rb +++ b/lib/rake/ext/module.rb @@ -36,4 +36,4 @@ class Module rake_original_const_missing(const_name) end end -end +end unless defined? Rake::REDUCE_COMPAT diff --git a/lib/rake/ext/string.rb b/lib/rake/ext/string.rb index fb22a9deb1..be8b463e1a 100644 --- a/lib/rake/ext/string.rb +++ b/lib/rake/ext/string.rb @@ -4,6 +4,7 @@ require 'rake/ext/core' # Rake extension methods for String. # class String + rake_extension("ext") do # Replace the file extension with +newext+. If there is no extension on # the string, append the new extension to the end. If the new extension @@ -163,5 +164,5 @@ class String result end end -end # class String +end diff --git a/lib/rake/ext/time.rb b/lib/rake/ext/time.rb index 7877abf0ce..ea8b037e39 100644 --- a/lib/rake/ext/time.rb +++ b/lib/rake/ext/time.rb @@ -1,6 +1,8 @@ #-- # Extensions to time to allow comparisons with an early time class. +require 'rake/early_time' + class Time alias rake_original_time_compare :<=> def <=>(other) @@ -11,4 +13,3 @@ class Time end end end - diff --git a/lib/rake/file_list.rb b/lib/rake/file_list.rb index e49ccd0147..b74ecac4cc 100644 --- a/lib/rake/file_list.rb +++ b/lib/rake/file_list.rb @@ -286,7 +286,7 @@ module Rake matched = 0 each do |fn| begin - open(fn, "r:ascii-8bit", *options) do |inf| + open(fn, "r", *options) do |inf| count = 0 inf.each do |line| count += 1 @@ -340,7 +340,7 @@ module Rake # Add matching glob patterns. def add_matching(pattern) - Dir[pattern].each do |fn| + Rake.glob(pattern).each do |fn| self << fn unless exclude?(fn) end end diff --git a/lib/rake/file_utils_ext.rb b/lib/rake/file_utils_ext.rb index 557420df80..386af441d8 100644 --- a/lib/rake/file_utils_ext.rb +++ b/lib/rake/file_utils_ext.rb @@ -21,12 +21,13 @@ module Rake $fileutils_verbose = true $fileutils_nowrite = false - FileUtils::OPT_TABLE.each do |name, opts| + FileUtils.commands.each do |name| + opts = FileUtils.options_of name default_options = [] - if opts.include?(:verbose) || opts.include?("verbose") + if opts.include?("verbose") default_options << ':verbose => FileUtilsExt.verbose_flag' end - if opts.include?(:noop) || opts.include?("noop") + if opts.include?("noop") default_options << ':noop => FileUtilsExt.nowrite_flag' end diff --git a/lib/rake/multi_task.rb b/lib/rake/multi_task.rb index 21c8de732f..5418a7a7b0 100644 --- a/lib/rake/multi_task.rb +++ b/lib/rake/multi_task.rb @@ -5,11 +5,8 @@ module Rake # class MultiTask < Task private - def invoke_prerequisites(args, invocation_chain) - threads = @prerequisites.collect { |p| - Thread.new(p) { |r| application[r, @scope].invoke_with_call_chain(args, invocation_chain) } - } - threads.each { |t| t.join } + def invoke_prerequisites(task_args, invocation_chain) # :nodoc: + invoke_prerequisites_concurrently(task_args, invocation_chain) end end diff --git a/lib/rake/phony.rb b/lib/rake/phony.rb new file mode 100644 index 0000000000..0552c26a33 --- /dev/null +++ b/lib/rake/phony.rb @@ -0,0 +1,13 @@ +# Defines a :phony task that you can use as a dependency. This allows +# file-based tasks to use non-file-based tasks as prerequisites +# without forcing them to rebuild. +# +# See FileTask#out_of_date? and Task#timestamp for more info. + +require 'rake' + +task :phony + +def (Rake::Task[:phony]).timestamp + Time.at 0 +end diff --git a/lib/rake/private_reader.rb b/lib/rake/private_reader.rb new file mode 100644 index 0000000000..1620978576 --- /dev/null +++ b/lib/rake/private_reader.rb @@ -0,0 +1,20 @@ +module Rake + + # Include PrivateReader to use +private_reader+. + module PrivateReader # :nodoc: all + + def self.included(base) + base.extend(ClassMethods) + end + + module ClassMethods + + # Declare a list of private accessors + def private_reader(*names) + attr_reader(*names) + private(*names) + end + end + + end +end diff --git a/lib/rake/promise.rb b/lib/rake/promise.rb new file mode 100644 index 0000000000..3258b91139 --- /dev/null +++ b/lib/rake/promise.rb @@ -0,0 +1,99 @@ +module Rake + + # A Promise object represents a promise to do work (a chore) in the + # future. The promise is created with a block and a list of + # arguments for the block. Calling value will return the value of + # the promised chore. + # + # Used by ThreadPool. + # + class Promise # :nodoc: all + NOT_SET = Object.new.freeze # :nodoc: + + attr_accessor :recorder + + # Create a promise to do the chore specified by the block. + def initialize(args, &block) + @mutex = Mutex.new + @result = NOT_SET + @error = NOT_SET + @args = args.collect { |a| begin; a.dup; rescue; a; end } + @block = block + end + + # Return the value of this promise. + # + # If the promised chore is not yet complete, then do the work + # synchronously. We will wait. + def value + unless complete? + stat :sleeping_on, :item_id => object_id + @mutex.synchronize do + stat :has_lock_on, :item_id => object_id + chore + stat :releasing_lock_on, :item_id => object_id + end + end + error? ? raise(@error) : @result + end + + # If no one else is working this promise, go ahead and do the chore. + def work + stat :attempting_lock_on, :item_id => object_id + if @mutex.try_lock + stat :has_lock_on, :item_id => object_id + chore + stat :releasing_lock_on, :item_id => object_id + @mutex.unlock + else + stat :bailed_on, :item_id => object_id + end + end + + private + + # Perform the chore promised + def chore + if complete? + stat :found_completed, :item_id => object_id + return + end + stat :will_execute, :item_id => object_id + begin + @result = @block.call(*@args) + rescue Exception => e + @error = e + end + stat :did_execute, :item_id => object_id + discard + end + + # Do we have a result for the promise + def result? + ! @result.equal?(NOT_SET) + end + + # Did the promise throw an error + def error? + ! @error.equal?(NOT_SET) + end + + # Are we done with the promise + def complete? + result? || error? + end + + # free up these items for the GC + def discard + @args = nil + @block = nil + end + + # Record execution statistics if there is a recorder + def stat(*args) + @recorder.call(*args) if @recorder + end + + end + +end diff --git a/lib/rake/rake_module.rb b/lib/rake/rake_module.rb index a9d210c637..6f77d1b674 100644 --- a/lib/rake/rake_module.rb +++ b/lib/rake/rake_module.rb @@ -24,6 +24,21 @@ module Rake def load_rakefile(path) load(path) end + + # Add files to the rakelib list + def add_rakelib(*files) + application.options.rakelib ||= [] + files.each do |file| + application.options.rakelib << file + end + end + + # Get a sorted list of files matching the pattern. This method + # should be prefered to Dir[pattern] and Dir.glob[pattern] because + # the files returned are guaranteed to be sorted. + def glob(pattern, *args) + Dir.glob(pattern, *args).sort + end end end diff --git a/lib/rake/rdoctask.rb b/lib/rake/rdoctask.rb index b6ae224a9e..261fa69b4d 100644 --- a/lib/rake/rdoctask.rb +++ b/lib/rake/rdoctask.rb @@ -1,7 +1,7 @@ # rake/rdoctask is deprecated in favor of rdoc/task if Rake.application - Rake.application.deprecate('require \'rake/rdoctask\'', 'require \'rdoc/task\' (in RDoc 2.4.2+)', __FILE__) + Rake.application.deprecate('require \'rake/rdoctask\'', 'require \'rdoc/task\' (in RDoc 2.4.2+)', caller.first) end require 'rubygems' diff --git a/lib/rake/runtest.rb b/lib/rake/runtest.rb index 2b98a60cae..9c6469d45e 100644 --- a/lib/rake/runtest.rb +++ b/lib/rake/runtest.rb @@ -5,7 +5,7 @@ module Rake include Test::Unit::Assertions def run_tests(pattern='test/test*.rb', log_enabled=false) - Dir["#{pattern}"].each { |fn| + Rake.glob(pattern).each { |fn| $stderr.puts fn if log_enabled begin require fn diff --git a/lib/rake/task.rb b/lib/rake/task.rb index f977d18711..afa1b6153d 100644 --- a/lib/rake/task.rb +++ b/lib/rake/task.rb @@ -123,6 +123,7 @@ module Rake def clear clear_prerequisites clear_actions + clear_comments self end @@ -138,6 +139,13 @@ module Rake self end + # Clear the existing comments on a rake task. + def clear_comments + @full_comment = nil + @comment = nil + self + end + # Invoke the task if it is needed. Prerequisites are invoked first. def invoke(*args) task_args = TaskArguments.new(arg_names, args) @@ -150,7 +158,7 @@ module Rake new_chain = InvocationChain.append(self, invocation_chain) @lock.synchronize do if application.options.trace - $stderr.puts "** Invoke #{name} #{format_trace_flags}" + application.trace "** Invoke #{name} #{format_trace_flags}" end return if @already_invoked @already_invoked = true @@ -171,10 +179,24 @@ module Rake # Invoke all the prerequisites of a task. def invoke_prerequisites(task_args, invocation_chain) # :nodoc: - prerequisite_tasks.each { |prereq| - prereq_args = task_args.new_scope(prereq.arg_names) - prereq.invoke_with_call_chain(prereq_args, invocation_chain) - } + if application.options.always_multitask + invoke_prerequisites_concurrently(task_args, invocation_chain) + else + prerequisite_tasks.each { |prereq| + prereq_args = task_args.new_scope(prereq.arg_names) + prereq.invoke_with_call_chain(prereq_args, invocation_chain) + } + end + end + + # Invoke all the prerequisites of a task in parallel. + def invoke_prerequisites_concurrently(args, invocation_chain) # :nodoc: + futures = @prerequisites.collect do |p| + application.thread_pool.future(p) do |r| + application[r, @scope].invoke_with_call_chain(args, invocation_chain) + end + end + futures.each { |f| f.value } end # Format the trace flags for display. @@ -190,11 +212,11 @@ module Rake def execute(args=nil) args ||= EMPTY_TASK_ARGS if application.options.dryrun - $stderr.puts "** Execute (dry run) #{name}" + application.trace "** Execute (dry run) #{name}" return end if application.options.trace - $stderr.puts "** Execute #{name}" + application.trace "** Execute #{name}" end application.enhance_with_matching_rule(name) if @actions.empty? @actions.each do |act| diff --git a/lib/rake/task_arguments.rb b/lib/rake/task_arguments.rb index 02d01b99f9..4417af2f8e 100644 --- a/lib/rake/task_arguments.rb +++ b/lib/rake/task_arguments.rb @@ -47,7 +47,7 @@ module Rake keys.map { |k| lookup(k) } end - def method_missing(sym, *args, &block) + def method_missing(sym, *args) lookup(sym.to_sym) end diff --git a/lib/rake/task_manager.rb b/lib/rake/task_manager.rb index 4c3c26aa3a..5a9419d536 100644 --- a/lib/rake/task_manager.rb +++ b/lib/rake/task_manager.rb @@ -238,7 +238,7 @@ module Rake end def trace_rule(level, message) - $stderr.puts "#{" "*level}#{message}" if Rake.application.options.trace_rules + options.trace_output.puts "#{" "*level}#{message}" if Rake.application.options.trace_rules end # Attempt to create a rule given the list of prerequisites. diff --git a/lib/rake/testtask.rb b/lib/rake/testtask.rb index 04d3ae473a..99094df1c8 100644 --- a/lib/rake/testtask.rb +++ b/lib/rake/testtask.rb @@ -96,7 +96,11 @@ module Rake desc "Run tests" + (@name==:test ? "" : " for #{@name}") task @name do FileUtilsExt.verbose(@verbose) do - ruby "#{ruby_opts_string} #{run_code} #{file_list_string} #{option_list}" + ruby "#{ruby_opts_string} #{run_code} #{file_list_string} #{option_list}" do |ok, status| + if !ok && status.respond_to?(:signaled?) && status.signaled? + raise SignalException.new(status.termsig) + end + end end end self diff --git a/lib/rake/thread_history_display.rb b/lib/rake/thread_history_display.rb new file mode 100644 index 0000000000..917e951064 --- /dev/null +++ b/lib/rake/thread_history_display.rb @@ -0,0 +1,48 @@ +require 'rake/private_reader' + +module Rake + + class ThreadHistoryDisplay # :nodoc: all + include Rake::PrivateReader + + private_reader :stats, :items, :threads + + def initialize(stats) + @stats = stats + @items = { :_seq_ => 1 } + @threads = { :_seq_ => "A" } + end + + def show + puts "Job History:" + stats.each do |stat| + stat[:data] ||= {} + rename(stat, :thread, threads) + rename(stat[:data], :item_id, items) + rename(stat[:data], :new_thread, threads) + rename(stat[:data], :deleted_thread, threads) + printf("%8d %2s %-20s %s\n", + (stat[:time] * 1_000_000).round, + stat[:thread], + stat[:event], + stat[:data].map { |k,v| "#{k}:#{v}" }.join(" ")) + end + end + + private + + def rename(hash, key, renames) + if hash && hash[key] + original = hash[key] + value = renames[original] + unless value + value = renames[:_seq_] + renames[:_seq_] = renames[:_seq_].succ + renames[original] = value + end + hash[key] = value + end + end + end + +end diff --git a/lib/rake/thread_pool.rb b/lib/rake/thread_pool.rb new file mode 100644 index 0000000000..983a67a514 --- /dev/null +++ b/lib/rake/thread_pool.rb @@ -0,0 +1,155 @@ +require 'thread' +require 'set' + +require 'rake/promise' + +module Rake + + class ThreadPool # :nodoc: all + + # Creates a ThreadPool object. + # The parameter is the size of the pool. + def initialize(thread_count) + @max_active_threads = [thread_count, 0].max + @threads = Set.new + @threads_mon = Monitor.new + @queue = Queue.new + @join_cond = @threads_mon.new_cond + + @history_start_time = nil + @history = [] + @history_mon = Monitor.new + @total_threads_in_play = 0 + end + + # Creates a future executed by the +ThreadPool+. + # + # The args are passed to the block when executing (similarly to + # Thread#new) The return value is an object representing + # a future which has been created and added to the queue in the + # pool. Sending #value to the object will sleep the + # current thread until the future is finished and will return the + # result (or raise an exception thrown from the future) + def future(*args, &block) + promise = Promise.new(args, &block) + promise.recorder = lambda { |*stats| stat(*stats) } + + @queue.enq promise + stat :queued, :item_id => promise.object_id + start_thread + promise + end + + # Waits until the queue of futures is empty and all threads have exited. + def join + @threads_mon.synchronize do + begin + stat :joining + @join_cond.wait unless @threads.empty? + stat :joined + rescue Exception => e + stat :joined + $stderr.puts e + $stderr.print "Queue contains #{@queue.size} items. Thread pool contains #{@threads.count} threads\n" + $stderr.print "Current Thread #{Thread.current} status = #{Thread.current.status}\n" + $stderr.puts e.backtrace.join("\n") + @threads.each do |t| + $stderr.print "Thread #{t} status = #{t.status}\n" + # 1.8 doesn't support Thread#backtrace + $stderr.puts t.backtrace.join("\n") if t.respond_to? :backtrace + end + raise e + end + end + end + + # Enable the gathering of history events. + def gather_history #:nodoc: + @history_start_time = Time.now if @history_start_time.nil? + end + + # Return a array of history events for the thread pool. + # + # History gathering must be enabled to be able to see the events + # (see #gather_history). Best to call this when the job is + # complete (i.e. after ThreadPool#join is called). + def history # :nodoc: + @history_mon.synchronize { @history.dup }. + sort_by { |i| i[:time] }. + each { |i| i[:time] -= @history_start_time } + end + + # Return a hash of always collected statistics for the thread pool. + def statistics # :nodoc: + { + :total_threads_in_play => @total_threads_in_play, + :max_active_threads => @max_active_threads, + } + end + + private + + # processes one item on the queue. Returns true if there was an + # item to process, false if there was no item + def process_queue_item #:nodoc: + return false if @queue.empty? + + # Even though we just asked if the queue was empty, it + # still could have had an item which by this statement + # is now gone. For this reason we pass true to Queue#deq + # because we will sleep indefinitely if it is empty. + promise = @queue.deq(true) + stat :dequeued, :item_id => promise.object_id + promise.work + return true + + rescue ThreadError # this means the queue is empty + false + end + + def start_thread # :nodoc: + @threads_mon.synchronize do + next unless @threads.count < @max_active_threads + + t = Thread.new do + begin + while @threads.count <= @max_active_threads + break unless process_queue_item + end + ensure + @threads_mon.synchronize do + @threads.delete Thread.current + stat :ended, :thread_count => @threads.count + @join_cond.broadcast if @threads.empty? + end + end + end + @threads << t + stat :spawned, :new_thread => t.object_id, :thread_count => @threads.count + @total_threads_in_play = @threads.count if @threads.count > @total_threads_in_play + end + end + + def stat(event, data=nil) # :nodoc: + return if @history_start_time.nil? + info = { + :event => event, + :data => data, + :time => Time.now, + :thread => Thread.current.object_id, + } + @history_mon.synchronize { @history << info } + end + + # for testing only + + def __queue__ # :nodoc: + @queue + end + + def __threads__ # :nodoc: + @threads.dup + end + end + +end diff --git a/lib/rake/version.rb b/lib/rake/version.rb index 6c43493df9..2515e25663 100644 --- a/lib/rake/version.rb +++ b/lib/rake/version.rb @@ -1,8 +1,10 @@ module Rake - VERSION = '0.9.2.2' - module Version # :nodoc: all - MAJOR, MINOR, BUILD = VERSION.split '.' - NUMBERS = [ MAJOR, MINOR, BUILD ] + NUMBERS = [ + MAJOR = 0, + MINOR = 9, + BUILD = 3, + ] end + VERSION = Version::NUMBERS.join('.') end -- cgit v1.2.3