summaryrefslogtreecommitdiff
path: root/lib/rubygems/source_index.rb
blob: b940b83cf872bd0b74cab7a4e75300af60c8c4fe (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
#--
# Copyright 2006 by Chad Fowler, Rich Kilmer, Jim Weirich and others.
# All rights reserved.
# See LICENSE.txt for permissions.
#++

require 'rubygems'
require 'rubygems/user_interaction'
require 'rubygems/specification'
require 'rubygems/spec_fetcher'

##
# The SourceIndex object indexes all the gems available from a
# particular source (e.g. a list of gem directories, or a remote
# source).  A SourceIndex maps a gem full name to a gem
# specification.
#
# NOTE:: The class used to be named Cache, but that became
#        confusing when cached source fetchers where introduced. The
#        constant Gem::Cache is an alias for this class to allow old
#        YAMLized source index objects to load properly.

class Gem::SourceIndex

  include Enumerable

  include Gem::UserInteraction

  attr_reader :gems # :nodoc:

  ##
  # Directories to use to refresh this SourceIndex when calling refresh!

  attr_accessor :spec_dirs

  class << self
    include Gem::UserInteraction

    ##
    # Factory method to construct a source index instance for a given
    # path.
    #
    # deprecated::
    #   If supplied, from_installed_gems will act just like
    #   +from_gems_in+.  This argument is deprecated and is provided
    #   just for backwards compatibility, and should not generally
    #   be used.
    #
    # return::
    #   SourceIndex instance

    def from_installed_gems(*deprecated)
      if deprecated.empty?
        from_gems_in(*installed_spec_directories)
      else
        from_gems_in(*deprecated) # HACK warn
      end
    end

    ##
    # Returns a list of directories from Gem.path that contain specifications.

    def installed_spec_directories
      Gem.path.collect { |dir| File.join(dir, "specifications") }
    end

    ##
    # Creates a new SourceIndex from the ruby format gem specifications in
    # +spec_dirs+.

    def from_gems_in(*spec_dirs)
      source_index = new
      source_index.spec_dirs = spec_dirs
      source_index.refresh!
    end

    ##
    # Loads a ruby-format specification from +file_name+ and returns the
    # loaded spec.

    def load_specification(file_name)
      begin
        spec_code = File.read(file_name).untaint
        gemspec = eval spec_code, binding, file_name
        if gemspec.is_a?(Gem::Specification)
          gemspec.loaded_from = file_name
          return gemspec
        end
        alert_warning "File '#{file_name}' does not evaluate to a gem specification"
      rescue SignalException, SystemExit
        raise
      rescue SyntaxError => e
        alert_warning e
        alert_warning spec_code
      rescue Exception => e
        alert_warning(e.inspect.to_s + "\n" + spec_code)
        alert_warning "Invalid .gemspec format in '#{file_name}'"
      end
      return nil
    end

  end

  ##
  # Constructs a source index instance from the provided
  # specifications
  #
  # specifications::
  #   [Hash] hash of [Gem name, Gem::Specification] pairs

  def initialize(specifications={})
    @gems = specifications
    @spec_dirs = nil
  end

  ##
  # Reconstruct the source index from the specifications in +spec_dirs+.

  def load_gems_in(*spec_dirs)
    @gems.clear

    spec_dirs.reverse_each do |spec_dir|
      spec_files = Dir.glob File.join(spec_dir, '*.gemspec')

      spec_files.each do |spec_file|
        gemspec = self.class.load_specification spec_file.untaint
        add_spec gemspec if gemspec
      end
    end

    self
  end

  ##
  # Returns an Array specifications for the latest versions of each gem in
  # this index.

  def latest_specs
    result = Hash.new { |h,k| h[k] = [] }
    latest = {}

    sort.each do |_, spec|
      name = spec.name
      curr_ver = spec.version
      prev_ver = latest.key?(name) ? latest[name].version : nil

      next unless prev_ver.nil? or curr_ver >= prev_ver or
                  latest[name].platform != Gem::Platform::RUBY

      if prev_ver.nil? or
         (curr_ver > prev_ver and spec.platform == Gem::Platform::RUBY) then
        result[name].clear
        latest[name] = spec
      end

      if spec.platform != Gem::Platform::RUBY then
        result[name].delete_if do |result_spec|
          result_spec.platform == spec.platform
        end
      end

      result[name] << spec
    end

    result.values.flatten
  end

  ##
  # Add a gem specification to the source index.

  def add_spec(gem_spec)
    @gems[gem_spec.full_name] = gem_spec
  end

  ##
  # Add gem specifications to the source index.

  def add_specs(*gem_specs)
    gem_specs.each do |spec|
      add_spec spec
    end
  end

  ##
  # Remove a gem specification named +full_name+.

  def remove_spec(full_name)
    @gems.delete(full_name)
  end

  ##
  # Iterate over the specifications in the source index.

  def each(&block) # :yields: gem.full_name, gem
    @gems.each(&block)
  end

  ##
  # The gem specification given a full gem spec name.

  def specification(full_name)
    @gems[full_name]
  end

  ##
  # The signature for the source index.  Changes in the signature indicate a
  # change in the index.

  def index_signature
    require 'rubygems/digest/sha2'

    Gem::SHA256.new.hexdigest(@gems.keys.sort.join(',')).to_s
  end

  ##
  # The signature for the given gem specification.

  def gem_signature(gem_full_name)
    require 'rubygems/digest/sha2'

    Gem::SHA256.new.hexdigest(@gems[gem_full_name].to_yaml).to_s
  end

  def size
    @gems.size
  end
  alias length size

  ##
  # Find a gem by an exact match on the short name.

  def find_name(gem_name, version_requirement = Gem::Requirement.default)
    search(/^#{gem_name}$/, version_requirement)
  end

  ##
  # Search for a gem by Gem::Dependency +gem_pattern+.  If +only_platform+
  # is true, only gems matching Gem::Platform.local will be returned.  An
  # Array of matching Gem::Specification objects is returned.
  #
  # For backwards compatibility, a String or Regexp pattern may be passed as
  # +gem_pattern+, and a Gem::Requirement for +platform_only+.  This
  # behavior is deprecated and will be removed.

  def search(gem_pattern, platform_only = false)
    version_requirement = nil
    only_platform = false

    case gem_pattern # TODO warn after 2008/03, remove three months after
    when Regexp then
      version_requirement = platform_only || Gem::Requirement.default
    when Gem::Dependency then
      only_platform = platform_only
      version_requirement = gem_pattern.version_requirements
      gem_pattern = if Regexp === gem_pattern.name then
                      gem_pattern.name
                    elsif gem_pattern.name.empty? then
                      //
                    else
                      /^#{Regexp.escape gem_pattern.name}$/
                    end
    else
      version_requirement = platform_only || Gem::Requirement.default
      gem_pattern = /#{gem_pattern}/i
    end

    unless Gem::Requirement === version_requirement then
      version_requirement = Gem::Requirement.create version_requirement
    end

    specs = @gems.values.select do |spec|
      spec.name =~ gem_pattern and
      version_requirement.satisfied_by? spec.version
    end

    if only_platform then
      specs = specs.select do |spec|
        Gem::Platform.match spec.platform
      end
    end

    specs.sort_by { |s| s.sort_obj }
  end

  ##
  # Replaces the gems in the source index from specifications in the
  # directories this source index was created from.  Raises an exception if
  # this source index wasn't created from a directory (via from_gems_in or
  # from_installed_gems, or having spec_dirs set).

  def refresh!
    raise 'source index not created from disk' if @spec_dirs.nil?
    load_gems_in(*@spec_dirs)
  end

  ##
  # Returns an Array of Gem::Specifications that are not up to date.

  def outdated
    outdateds = []

    latest_specs.each do |local|
      name = local.name

      dependency = Gem::Dependency.new name, ">= #{local.version}"

      begin
        fetcher = Gem::SpecFetcher.fetcher
        remotes = fetcher.find_matching dependency
        remotes = remotes.map { |(name, version,),| version }
      rescue Gem::RemoteFetcher::FetchError => e
        raise unless fetcher.warn_legacy e do
          require 'rubygems/source_info_cache'

          specs = Gem::SourceInfoCache.search_with_source dependency, true

          remotes = specs.map { |spec,| spec.version }
        end
      end

      latest = remotes.sort.last

      outdateds << name if latest and local.version < latest
    end

    outdateds
  end

  ##
  # Updates this SourceIndex from +source_uri+.  If +all+ is false, only the
  # latest gems are fetched.

  def update(source_uri, all)
    source_uri = URI.parse source_uri unless URI::Generic === source_uri
    source_uri.path += '/' unless source_uri.path =~ /\/$/

    use_incremental = false

    begin
      gem_names = fetch_quick_index source_uri, all
      remove_extra gem_names
      missing_gems = find_missing gem_names

      return false if missing_gems.size.zero?

      say "Missing metadata for #{missing_gems.size} gems" if
      missing_gems.size > 0 and Gem.configuration.really_verbose

      use_incremental = missing_gems.size <= Gem.configuration.bulk_threshold
    rescue Gem::OperationNotSupportedError => ex
      alert_error "Falling back to bulk fetch: #{ex.message}" if
      Gem.configuration.really_verbose
      use_incremental = false
    end

    if use_incremental then
      update_with_missing(source_uri, missing_gems)
    else
      new_index = fetch_bulk_index(source_uri)
      @gems.replace(new_index.gems)
    end

    true
  end

  def ==(other) # :nodoc:
    self.class === other and @gems == other.gems 
  end

  def dump
    Marshal.dump(self)
  end

  private

  def fetcher
    require 'rubygems/remote_fetcher'

    Gem::RemoteFetcher.fetcher
  end

  def fetch_index_from(source_uri)
    @fetch_error = nil

    indexes = %W[
        Marshal.#{Gem.marshal_version}.Z
        Marshal.#{Gem.marshal_version}
        yaml.Z
        yaml
      ]

    indexes.each do |name|
      spec_data = nil
      index = source_uri + name
      begin
        spec_data = fetcher.fetch_path index
        spec_data = unzip(spec_data) if name =~ /\.Z$/

        if name =~ /Marshal/ then
          return Marshal.load(spec_data)
        else
          return YAML.load(spec_data)
        end
      rescue => e
        if Gem.configuration.really_verbose then
          alert_error "Unable to fetch #{name}: #{e.message}"
        end

        @fetch_error = e
      end
    end

    nil
  end

  def fetch_bulk_index(source_uri)
    say "Bulk updating Gem source index for: #{source_uri}" if
      Gem.configuration.verbose

    index = fetch_index_from(source_uri)
    if index.nil? then
      raise Gem::RemoteSourceException,
              "Error fetching remote gem cache: #{@fetch_error}"
    end
    @fetch_error = nil
    index
  end

  ##
  # Get the quick index needed for incremental updates.

  def fetch_quick_index(source_uri, all)
    index = all ? 'index' : 'latest_index'

    zipped_index = fetcher.fetch_path source_uri + "quick/#{index}.rz"

    unzip(zipped_index).split("\n")
  rescue ::Exception => e
    unless all then
      say "Latest index not found, using quick index" if
        Gem.configuration.really_verbose

      fetch_quick_index source_uri, true
    else
      raise Gem::OperationNotSupportedError,
            "No quick index found: #{e.message}"
    end
  end

  ##
  # Make a list of full names for all the missing gemspecs.

  def find_missing(spec_names)
    unless defined? @originals then
      @originals = {}
      each do |full_name, spec|
        @originals[spec.original_name] = spec
      end
    end

    spec_names.find_all { |full_name|
      @originals[full_name].nil?
    }
  end

  def remove_extra(spec_names)
    dictionary = spec_names.inject({}) { |h, k| h[k] = true; h }
    each do |name, spec|
      remove_spec name unless dictionary.include? spec.original_name
    end
  end

  ##
  # Unzip the given string.

  def unzip(string)
    require 'zlib'
    Gem.inflate string
  end

  ##
  # Tries to fetch Marshal representation first, then YAML

  def fetch_single_spec(source_uri, spec_name)
    @fetch_error = nil

    begin
      marshal_uri = source_uri + "quick/Marshal.#{Gem.marshal_version}/#{spec_name}.gemspec.rz"
      zipped = fetcher.fetch_path marshal_uri
      return Marshal.load(unzip(zipped))
    rescue => ex
      @fetch_error = ex

      if Gem.configuration.really_verbose then
        say "unable to fetch marshal gemspec #{marshal_uri}: #{ex.class} - #{ex}"
      end
    end

    begin
      yaml_uri = source_uri + "quick/#{spec_name}.gemspec.rz"
      zipped = fetcher.fetch_path yaml_uri
      return YAML.load(unzip(zipped))
    rescue => ex
      @fetch_error = ex
      if Gem.configuration.really_verbose then
        say "unable to fetch YAML gemspec #{yaml_uri}: #{ex.class} - #{ex}"
      end
    end

    nil
  end

  ##
  # Update the cached source index with the missing names.

  def update_with_missing(source_uri, missing_names)
    progress = ui.progress_reporter(missing_names.size,
        "Updating metadata for #{missing_names.size} gems from #{source_uri}")
    missing_names.each do |spec_name|
      gemspec = fetch_single_spec(source_uri, spec_name)
      if gemspec.nil? then
        ui.say "Failed to download spec #{spec_name} from #{source_uri}:\n" \
                 "\t#{@fetch_error.message}"
      else
        add_spec gemspec
        progress.updated spec_name
      end
      @fetch_error = nil
    end
    progress.done
    progress.count
  end

end

module Gem

  # :stopdoc:

  # Cache is an alias for SourceIndex to allow older YAMLized source index
  # objects to load properly.
  Cache = SourceIndex

  # :starddoc:

end