diff options
author | yugui <yugui@b2dd03c8-39d4-4d8f-98ff-823fe69b080e> | 2008-08-25 15:02:05 +0000 |
---|---|---|
committer | yugui <yugui@b2dd03c8-39d4-4d8f-98ff-823fe69b080e> | 2008-08-25 15:02:05 +0000 |
commit | 0dc342de848a642ecce8db697b8fecd83a63e117 (patch) | |
tree | 2b7ed4724aff1f86073e4740134bda9c4aac1a39 /trunk/lib/rubygems/indexer.rb | |
parent | ef70cf7138ab8034b5b806f466e4b484b24f0f88 (diff) |
added tag v1_9_0_4
git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/tags/v1_9_0_4@18845 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
Diffstat (limited to 'trunk/lib/rubygems/indexer.rb')
-rw-r--r-- | trunk/lib/rubygems/indexer.rb | 370 |
1 files changed, 370 insertions, 0 deletions
diff --git a/trunk/lib/rubygems/indexer.rb b/trunk/lib/rubygems/indexer.rb new file mode 100644 index 0000000000..e2dd57d3fe --- /dev/null +++ b/trunk/lib/rubygems/indexer.rb @@ -0,0 +1,370 @@ +require 'fileutils' +require 'tmpdir' +require 'zlib' + +require 'rubygems' +require 'rubygems/format' + +begin + require 'builder/xchar' +rescue LoadError +end + +## +# Top level class for building the gem repository index. + +class Gem::Indexer + + include Gem::UserInteraction + + ## + # Index install location + + attr_reader :dest_directory + + ## + # Index build directory + + attr_reader :directory + + ## + # Create an indexer that will index the gems in +directory+. + + def initialize(directory) + unless ''.respond_to? :to_xs then + fail "Gem::Indexer requires that the XML Builder library be installed:" \ + "\n\tgem install builder" + end + + @dest_directory = directory + @directory = File.join Dir.tmpdir, "gem_generate_index_#{$$}" + + marshal_name = "Marshal.#{Gem.marshal_version}" + + @master_index = File.join @directory, 'yaml' + @marshal_index = File.join @directory, marshal_name + + @quick_dir = File.join @directory, 'quick' + + @quick_marshal_dir = File.join @quick_dir, marshal_name + + @quick_index = File.join @quick_dir, 'index' + @latest_index = File.join @quick_dir, 'latest_index' + + @specs_index = File.join @directory, "specs.#{Gem.marshal_version}" + @latest_specs_index = File.join @directory, + "latest_specs.#{Gem.marshal_version}" + + files = [ + @specs_index, + "#{@specs_index}.gz", + @latest_specs_index, + "#{@latest_specs_index}.gz", + @quick_dir, + @master_index, + "#{@master_index}.Z", + @marshal_index, + "#{@marshal_index}.Z", + ] + + @files = files.map do |path| + path.sub @directory, '' + end + end + + ## + # Abbreviate the spec for downloading. Abbreviated specs are only used for + # searching, downloading and related activities and do not need deployment + # specific information (e.g. list of files). So we abbreviate the spec, + # making it much smaller for quicker downloads. + + def abbreviate(spec) + spec.files = [] + spec.test_files = [] + spec.rdoc_options = [] + spec.extra_rdoc_files = [] + spec.cert_chain = [] + spec + end + + ## + # Build various indicies + + def build_indicies(index) + progress = ui.progress_reporter index.size, + "Generating quick index gemspecs for #{index.size} gems", + "Complete" + + index.each do |original_name, spec| + spec_file_name = "#{original_name}.gemspec.rz" + yaml_name = File.join @quick_dir, spec_file_name + marshal_name = File.join @quick_marshal_dir, spec_file_name + + yaml_zipped = Gem.deflate spec.to_yaml + open yaml_name, 'wb' do |io| io.write yaml_zipped end + + marshal_zipped = Gem.deflate Marshal.dump(spec) + open marshal_name, 'wb' do |io| io.write marshal_zipped end + + progress.updated original_name + end + + progress.done + + say "Generating specs index" + + open @specs_index, 'wb' do |io| + specs = index.sort.map do |_, spec| + platform = spec.original_platform + platform = Gem::Platform::RUBY if platform.nil? or platform.empty? + [spec.name, spec.version, platform] + end + + specs = compact_specs specs + + Marshal.dump specs, io + end + + say "Generating latest specs index" + + open @latest_specs_index, 'wb' do |io| + specs = index.latest_specs.sort.map do |spec| + platform = spec.original_platform + platform = Gem::Platform::RUBY if platform.nil? or platform.empty? + [spec.name, spec.version, platform] + end + + specs = compact_specs specs + + Marshal.dump specs, io + end + + say "Generating quick index" + + quick_index = File.join @quick_dir, 'index' + open quick_index, 'wb' do |io| + io.puts index.sort.map { |_, spec| spec.original_name } + end + + say "Generating latest index" + + latest_index = File.join @quick_dir, 'latest_index' + open latest_index, 'wb' do |io| + io.puts index.latest_specs.sort.map { |spec| spec.original_name } + end + + say "Generating Marshal master index" + + open @marshal_index, 'wb' do |io| + io.write index.dump + end + + progress = ui.progress_reporter index.size, + "Generating YAML master index for #{index.size} gems (this may take a while)", + "Complete" + + open @master_index, 'wb' do |io| + io.puts "--- !ruby/object:#{index.class}" + io.puts "gems:" + + gems = index.sort_by { |name, gemspec| gemspec.sort_obj } + gems.each do |original_name, gemspec| + yaml = gemspec.to_yaml.gsub(/^/, ' ') + yaml = yaml.sub(/\A ---/, '') # there's a needed extra ' ' here + io.print " #{original_name}:" + io.puts yaml + + progress.updated original_name + end + end + + progress.done + + say "Compressing indicies" + # use gzip for future files. + + compress quick_index, 'rz' + paranoid quick_index, 'rz' + + compress latest_index, 'rz' + paranoid latest_index, 'rz' + + compress @marshal_index, 'Z' + paranoid @marshal_index, 'Z' + + compress @master_index, 'Z' + paranoid @master_index, 'Z' + + gzip @specs_index + gzip @latest_specs_index + end + + ## + # Collect specifications from .gem files from the gem directory. + + def collect_specs + index = Gem::SourceIndex.new + + progress = ui.progress_reporter gem_file_list.size, + "Loading #{gem_file_list.size} gems from #{@dest_directory}", + "Loaded all gems" + + gem_file_list.each do |gemfile| + if File.size(gemfile.to_s) == 0 then + alert_warning "Skipping zero-length gem: #{gemfile}" + next + end + + begin + spec = Gem::Format.from_file_by_path(gemfile).spec + + unless gemfile =~ /\/#{Regexp.escape spec.original_name}.*\.gem\z/i then + alert_warning "Skipping misnamed gem: #{gemfile} => #{spec.full_name} (#{spec.original_name})" + next + end + + abbreviate spec + sanitize spec + + index.gems[spec.original_name] = spec + + progress.updated spec.original_name + + rescue SignalException => e + alert_error "Received signal, exiting" + raise + rescue Exception => e + alert_error "Unable to process #{gemfile}\n#{e.message} (#{e.class})\n\t#{e.backtrace.join "\n\t"}" + end + end + + progress.done + + index + end + + ## + # Compacts Marshal output for the specs index data source by using identical + # objects as much as possible. + + def compact_specs(specs) + names = {} + versions = {} + platforms = {} + + specs.map do |(name, version, platform)| + names[name] = name unless names.include? name + versions[version] = version unless versions.include? version + platforms[platform] = platform unless platforms.include? platform + + [names[name], versions[version], platforms[platform]] + end + end + + ## + # Compress +filename+ with +extension+. + + def compress(filename, extension) + data = Gem.read_binary filename + + zipped = Gem.deflate data + + open "#{filename}.#{extension}", 'wb' do |io| + io.write zipped + end + end + + ## + # List of gem file names to index. + + def gem_file_list + Dir.glob(File.join(@dest_directory, "gems", "*.gem")) + end + + ## + # Builds and installs indexicies. + + def generate_index + make_temp_directories + index = collect_specs + build_indicies index + install_indicies + rescue SignalException + ensure + FileUtils.rm_rf @directory + end + + ## + # Zlib::GzipWriter wrapper that gzips +filename+ on disk. + + def gzip(filename) + Zlib::GzipWriter.open "#{filename}.gz" do |io| + io.write Gem.read_binary(filename) + end + end + + ## + # Install generated indicies into the destination directory. + + def install_indicies + verbose = Gem.configuration.really_verbose + + say "Moving index into production dir #{@dest_directory}" if verbose + + @files.each do |file| + src_name = File.join @directory, file + dst_name = File.join @dest_directory, file + + FileUtils.rm_rf dst_name, :verbose => verbose + FileUtils.mv src_name, @dest_directory, :verbose => verbose, + :force => true + end + end + + ## + # Make directories for index generation + + def make_temp_directories + FileUtils.rm_rf @directory + FileUtils.mkdir_p @directory, :mode => 0700 + FileUtils.mkdir_p @quick_marshal_dir + end + + ## + # Ensure +path+ and path with +extension+ are identical. + + def paranoid(path, extension) + data = Gem.read_binary path + compressed_data = Gem.read_binary "#{path}.#{extension}" + + unless data == Gem.inflate(compressed_data) then + raise "Compressed file #{compressed_path} does not match uncompressed file #{path}" + end + end + + ## + # Sanitize the descriptive fields in the spec. Sometimes non-ASCII + # characters will garble the site index. Non-ASCII characters will + # be replaced by their XML entity equivalent. + + def sanitize(spec) + spec.summary = sanitize_string(spec.summary) + spec.description = sanitize_string(spec.description) + spec.post_install_message = sanitize_string(spec.post_install_message) + spec.authors = spec.authors.collect { |a| sanitize_string(a) } + + spec + end + + ## + # Sanitize a single string. + + def sanitize_string(string) + # HACK the #to_s is in here because RSpec has an Array of Arrays of + # Strings for authors. Need a way to disallow bad values on gempsec + # generation. (Probably won't happen.) + string ? string.to_s.to_xs : string + end + +end + |