summaryrefslogtreecommitdiff
path: root/lib/rubygems/indexer.rb
blob: 5496e452cc7938f0e672aeabebbbeba476cc838c (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
require 'fileutils'
require 'tmpdir'

require 'rubygems'
require 'rubygems/format'

begin
  require 'builder/xchar'
rescue LoadError
end

##
# Top level class for building the gem repository index.

class Gem::Indexer

  include Gem::UserInteraction

  ##
  # Index install location

  attr_reader :dest_directory

  ##
  # Index build directory

  attr_reader :directory

  ##
  # Create an indexer that will index the gems in +directory+.

  def initialize(directory)
    unless ''.respond_to? :to_xs then
      fail "Gem::Indexer requires that the XML Builder library be installed:" \
           "\n\tgem install builder"
    end

    @dest_directory = directory
    @directory = File.join Dir.tmpdir, "gem_generate_index_#{$$}"

    marshal_name = "Marshal.#{Gem.marshal_version}"

    @master_index = Gem::Indexer::MasterIndexBuilder.new "yaml", @directory
    @marshal_index = Gem::Indexer::MarshalIndexBuilder.new marshal_name, @directory
    @quick_index = Gem::Indexer::QuickIndexBuilder.new 'index', @directory

    quick_dir = File.join @directory, 'quick'
    @latest_index = Gem::Indexer::LatestIndexBuilder.new 'latest_index', quick_dir
  end

  ##
  # Build the index.

  def build_index
    @master_index.build do
      @quick_index.build do
        @marshal_index.build do
          @latest_index.build do
            progress = ui.progress_reporter gem_file_list.size,
                                          "Generating index for #{gem_file_list.size} gems in #{@dest_directory}",
                                          "Loaded all gems"

                                          gem_file_list.each do |gemfile|
              if File.size(gemfile.to_s) == 0 then
                alert_warning "Skipping zero-length gem: #{gemfile}"
                next
              end

              begin
                spec = Gem::Format.from_file_by_path(gemfile).spec

                unless gemfile =~ /\/#{Regexp.escape spec.original_name}.*\.gem\z/i then
                  alert_warning "Skipping misnamed gem: #{gemfile} => #{spec.full_name} (#{spec.original_name})"
                  next
                end

                abbreviate spec
                sanitize spec

                @master_index.add spec
                @quick_index.add spec
                @marshal_index.add spec
                @latest_index.add spec

                progress.updated spec.original_name

              rescue SignalException => e
                alert_error "Recieved signal, exiting"
                raise
              rescue Exception => e
                alert_error "Unable to process #{gemfile}\n#{e.message} (#{e.class})\n\t#{e.backtrace.join "\n\t"}"
              end
                                          end

            progress.done

            say "Generating master indexes (this may take a while)"
          end
        end
      end
    end
  end

  def install_index
    verbose = Gem.configuration.really_verbose

    say "Moving index into production dir #{@dest_directory}" if verbose

    files = @master_index.files + @quick_index.files + @marshal_index.files +
            @latest_index.files

    files.each do |file|
      src_name = File.join @directory, file
      dst_name = File.join @dest_directory, file

      FileUtils.rm_rf dst_name, :verbose => verbose
      FileUtils.mv src_name, @dest_directory, :verbose => verbose
    end
  end

  def generate_index
    FileUtils.rm_rf @directory
    FileUtils.mkdir_p @directory, :mode => 0700

    build_index
    install_index
  rescue SignalException
  ensure
    FileUtils.rm_rf @directory
  end

  # List of gem file names to index.
  def gem_file_list
    Dir.glob(File.join(@dest_directory, "gems", "*.gem"))
  end

  # Abbreviate the spec for downloading.  Abbreviated specs are only
  # used for searching, downloading and related activities and do not
  # need deployment specific information (e.g. list of files).  So we
  # abbreviate the spec, making it much smaller for quicker downloads.
  def abbreviate(spec)
    spec.files = []
    spec.test_files = []
    spec.rdoc_options = []
    spec.extra_rdoc_files = []
    spec.cert_chain = []
    spec
  end

  # Sanitize the descriptive fields in the spec.  Sometimes non-ASCII
  # characters will garble the site index.  Non-ASCII characters will
  # be replaced by their XML entity equivalent.
  def sanitize(spec)
    spec.summary = sanitize_string(spec.summary)
    spec.description = sanitize_string(spec.description)
    spec.post_install_message = sanitize_string(spec.post_install_message)
    spec.authors = spec.authors.collect { |a| sanitize_string(a) }
    spec
  end

  # Sanitize a single string.
  def sanitize_string(string)
    # HACK the #to_s is in here because RSpec has an Array of Arrays of
    # Strings for authors.  Need a way to disallow bad values on gempsec
    # generation.  (Probably won't happen.)
    string ? string.to_s.to_xs : string
  end

end

require 'rubygems/indexer/abstract_index_builder'
require 'rubygems/indexer/master_index_builder'
require 'rubygems/indexer/quick_index_builder'
require 'rubygems/indexer/marshal_index_builder'
require 'rubygems/indexer/latest_index_builder'