diff options
Diffstat (limited to 'trunk/lib/rubygems/indexer.rb')
| -rw-r--r-- | trunk/lib/rubygems/indexer.rb | 370 |
1 files changed, 0 insertions, 370 deletions
diff --git a/trunk/lib/rubygems/indexer.rb b/trunk/lib/rubygems/indexer.rb deleted file mode 100644 index e2dd57d3fe..0000000000 --- a/trunk/lib/rubygems/indexer.rb +++ /dev/null @@ -1,370 +0,0 @@ -require 'fileutils' -require 'tmpdir' -require 'zlib' - -require 'rubygems' -require 'rubygems/format' - -begin - require 'builder/xchar' -rescue LoadError -end - -## -# Top level class for building the gem repository index. - -class Gem::Indexer - - include Gem::UserInteraction - - ## - # Index install location - - attr_reader :dest_directory - - ## - # Index build directory - - attr_reader :directory - - ## - # Create an indexer that will index the gems in +directory+. - - def initialize(directory) - unless ''.respond_to? :to_xs then - fail "Gem::Indexer requires that the XML Builder library be installed:" \ - "\n\tgem install builder" - end - - @dest_directory = directory - @directory = File.join Dir.tmpdir, "gem_generate_index_#{$$}" - - marshal_name = "Marshal.#{Gem.marshal_version}" - - @master_index = File.join @directory, 'yaml' - @marshal_index = File.join @directory, marshal_name - - @quick_dir = File.join @directory, 'quick' - - @quick_marshal_dir = File.join @quick_dir, marshal_name - - @quick_index = File.join @quick_dir, 'index' - @latest_index = File.join @quick_dir, 'latest_index' - - @specs_index = File.join @directory, "specs.#{Gem.marshal_version}" - @latest_specs_index = File.join @directory, - "latest_specs.#{Gem.marshal_version}" - - files = [ - @specs_index, - "#{@specs_index}.gz", - @latest_specs_index, - "#{@latest_specs_index}.gz", - @quick_dir, - @master_index, - "#{@master_index}.Z", - @marshal_index, - "#{@marshal_index}.Z", - ] - - @files = files.map do |path| - path.sub @directory, '' - end - end - - ## - # Abbreviate the spec for downloading. Abbreviated specs are only used for - # searching, downloading and related activities and do not need deployment - # specific information (e.g. list of files). So we abbreviate the spec, - # making it much smaller for quicker downloads. - - def abbreviate(spec) - spec.files = [] - spec.test_files = [] - spec.rdoc_options = [] - spec.extra_rdoc_files = [] - spec.cert_chain = [] - spec - end - - ## - # Build various indicies - - def build_indicies(index) - progress = ui.progress_reporter index.size, - "Generating quick index gemspecs for #{index.size} gems", - "Complete" - - index.each do |original_name, spec| - spec_file_name = "#{original_name}.gemspec.rz" - yaml_name = File.join @quick_dir, spec_file_name - marshal_name = File.join @quick_marshal_dir, spec_file_name - - yaml_zipped = Gem.deflate spec.to_yaml - open yaml_name, 'wb' do |io| io.write yaml_zipped end - - marshal_zipped = Gem.deflate Marshal.dump(spec) - open marshal_name, 'wb' do |io| io.write marshal_zipped end - - progress.updated original_name - end - - progress.done - - say "Generating specs index" - - open @specs_index, 'wb' do |io| - specs = index.sort.map do |_, spec| - platform = spec.original_platform - platform = Gem::Platform::RUBY if platform.nil? or platform.empty? - [spec.name, spec.version, platform] - end - - specs = compact_specs specs - - Marshal.dump specs, io - end - - say "Generating latest specs index" - - open @latest_specs_index, 'wb' do |io| - specs = index.latest_specs.sort.map do |spec| - platform = spec.original_platform - platform = Gem::Platform::RUBY if platform.nil? or platform.empty? - [spec.name, spec.version, platform] - end - - specs = compact_specs specs - - Marshal.dump specs, io - end - - say "Generating quick index" - - quick_index = File.join @quick_dir, 'index' - open quick_index, 'wb' do |io| - io.puts index.sort.map { |_, spec| spec.original_name } - end - - say "Generating latest index" - - latest_index = File.join @quick_dir, 'latest_index' - open latest_index, 'wb' do |io| - io.puts index.latest_specs.sort.map { |spec| spec.original_name } - end - - say "Generating Marshal master index" - - open @marshal_index, 'wb' do |io| - io.write index.dump - end - - progress = ui.progress_reporter index.size, - "Generating YAML master index for #{index.size} gems (this may take a while)", - "Complete" - - open @master_index, 'wb' do |io| - io.puts "--- !ruby/object:#{index.class}" - io.puts "gems:" - - gems = index.sort_by { |name, gemspec| gemspec.sort_obj } - gems.each do |original_name, gemspec| - yaml = gemspec.to_yaml.gsub(/^/, ' ') - yaml = yaml.sub(/\A ---/, '') # there's a needed extra ' ' here - io.print " #{original_name}:" - io.puts yaml - - progress.updated original_name - end - end - - progress.done - - say "Compressing indicies" - # use gzip for future files. - - compress quick_index, 'rz' - paranoid quick_index, 'rz' - - compress latest_index, 'rz' - paranoid latest_index, 'rz' - - compress @marshal_index, 'Z' - paranoid @marshal_index, 'Z' - - compress @master_index, 'Z' - paranoid @master_index, 'Z' - - gzip @specs_index - gzip @latest_specs_index - end - - ## - # Collect specifications from .gem files from the gem directory. - - def collect_specs - index = Gem::SourceIndex.new - - progress = ui.progress_reporter gem_file_list.size, - "Loading #{gem_file_list.size} gems from #{@dest_directory}", - "Loaded all gems" - - gem_file_list.each do |gemfile| - if File.size(gemfile.to_s) == 0 then - alert_warning "Skipping zero-length gem: #{gemfile}" - next - end - - begin - spec = Gem::Format.from_file_by_path(gemfile).spec - - unless gemfile =~ /\/#{Regexp.escape spec.original_name}.*\.gem\z/i then - alert_warning "Skipping misnamed gem: #{gemfile} => #{spec.full_name} (#{spec.original_name})" - next - end - - abbreviate spec - sanitize spec - - index.gems[spec.original_name] = spec - - progress.updated spec.original_name - - rescue SignalException => e - alert_error "Received signal, exiting" - raise - rescue Exception => e - alert_error "Unable to process #{gemfile}\n#{e.message} (#{e.class})\n\t#{e.backtrace.join "\n\t"}" - end - end - - progress.done - - index - end - - ## - # Compacts Marshal output for the specs index data source by using identical - # objects as much as possible. - - def compact_specs(specs) - names = {} - versions = {} - platforms = {} - - specs.map do |(name, version, platform)| - names[name] = name unless names.include? name - versions[version] = version unless versions.include? version - platforms[platform] = platform unless platforms.include? platform - - [names[name], versions[version], platforms[platform]] - end - end - - ## - # Compress +filename+ with +extension+. - - def compress(filename, extension) - data = Gem.read_binary filename - - zipped = Gem.deflate data - - open "#{filename}.#{extension}", 'wb' do |io| - io.write zipped - end - end - - ## - # List of gem file names to index. - - def gem_file_list - Dir.glob(File.join(@dest_directory, "gems", "*.gem")) - end - - ## - # Builds and installs indexicies. - - def generate_index - make_temp_directories - index = collect_specs - build_indicies index - install_indicies - rescue SignalException - ensure - FileUtils.rm_rf @directory - end - - ## - # Zlib::GzipWriter wrapper that gzips +filename+ on disk. - - def gzip(filename) - Zlib::GzipWriter.open "#{filename}.gz" do |io| - io.write Gem.read_binary(filename) - end - end - - ## - # Install generated indicies into the destination directory. - - def install_indicies - verbose = Gem.configuration.really_verbose - - say "Moving index into production dir #{@dest_directory}" if verbose - - @files.each do |file| - src_name = File.join @directory, file - dst_name = File.join @dest_directory, file - - FileUtils.rm_rf dst_name, :verbose => verbose - FileUtils.mv src_name, @dest_directory, :verbose => verbose, - :force => true - end - end - - ## - # Make directories for index generation - - def make_temp_directories - FileUtils.rm_rf @directory - FileUtils.mkdir_p @directory, :mode => 0700 - FileUtils.mkdir_p @quick_marshal_dir - end - - ## - # Ensure +path+ and path with +extension+ are identical. - - def paranoid(path, extension) - data = Gem.read_binary path - compressed_data = Gem.read_binary "#{path}.#{extension}" - - unless data == Gem.inflate(compressed_data) then - raise "Compressed file #{compressed_path} does not match uncompressed file #{path}" - end - end - - ## - # Sanitize the descriptive fields in the spec. Sometimes non-ASCII - # characters will garble the site index. Non-ASCII characters will - # be replaced by their XML entity equivalent. - - def sanitize(spec) - spec.summary = sanitize_string(spec.summary) - spec.description = sanitize_string(spec.description) - spec.post_install_message = sanitize_string(spec.post_install_message) - spec.authors = spec.authors.collect { |a| sanitize_string(a) } - - spec - end - - ## - # Sanitize a single string. - - def sanitize_string(string) - # HACK the #to_s is in here because RSpec has an Array of Arrays of - # Strings for authors. Need a way to disallow bad values on gempsec - # generation. (Probably won't happen.) - string ? string.to_s.to_xs : string - end - -end - |