1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
|
require 'fileutils'
require 'tmpdir'
require 'zlib'
require 'rubygems'
require 'rubygems/format'
begin
require 'builder/xchar'
rescue LoadError
end
##
# Top level class for building the gem repository index.
class Gem::Indexer
include Gem::UserInteraction
##
# Index install location
attr_reader :dest_directory
##
# Index build directory
attr_reader :directory
##
# Create an indexer that will index the gems in +directory+.
def initialize(directory)
unless ''.respond_to? :to_xs then
fail "Gem::Indexer requires that the XML Builder library be installed:" \
"\n\tgem install builder"
end
@dest_directory = directory
@directory = File.join Dir.tmpdir, "gem_generate_index_#{$$}"
marshal_name = "Marshal.#{Gem.marshal_version}"
@master_index = File.join @directory, 'yaml'
@marshal_index = File.join @directory, marshal_name
@quick_dir = File.join @directory, 'quick'
@quick_marshal_dir = File.join @quick_dir, marshal_name
@quick_index = File.join @quick_dir, 'index'
@latest_index = File.join @quick_dir, 'latest_index'
@specs_index = File.join @directory, "specs.#{Gem.marshal_version}"
@latest_specs_index = File.join @directory,
"latest_specs.#{Gem.marshal_version}"
files = [
@specs_index,
"#{@specs_index}.gz",
@latest_specs_index,
"#{@latest_specs_index}.gz",
@quick_dir,
@master_index,
"#{@master_index}.Z",
@marshal_index,
"#{@marshal_index}.Z",
]
@files = files.map do |path|
path.sub @directory, ''
end
end
##
# Abbreviate the spec for downloading. Abbreviated specs are only used for
# searching, downloading and related activities and do not need deployment
# specific information (e.g. list of files). So we abbreviate the spec,
# making it much smaller for quicker downloads.
def abbreviate(spec)
spec.files = []
spec.test_files = []
spec.rdoc_options = []
spec.extra_rdoc_files = []
spec.cert_chain = []
spec
end
##
# Build various indicies
def build_indicies(index)
progress = ui.progress_reporter index.size,
"Generating quick index gemspecs for #{index.size} gems",
"Complete"
index.each do |original_name, spec|
spec_file_name = "#{original_name}.gemspec.rz"
yaml_name = File.join @quick_dir, spec_file_name
marshal_name = File.join @quick_marshal_dir, spec_file_name
yaml_zipped = Gem.deflate spec.to_yaml
open yaml_name, 'wb' do |io| io.write yaml_zipped end
marshal_zipped = Gem.deflate Marshal.dump(spec)
open marshal_name, 'wb' do |io| io.write marshal_zipped end
progress.updated original_name
end
progress.done
say "Generating specs index"
open @specs_index, 'wb' do |io|
specs = index.sort.map do |_, spec|
platform = spec.original_platform
platform = Gem::Platform::RUBY if platform.nil? or platform.empty?
[spec.name, spec.version, platform]
end
specs = compact_specs specs
Marshal.dump specs, io
end
say "Generating latest specs index"
open @latest_specs_index, 'wb' do |io|
specs = index.latest_specs.sort.map do |spec|
platform = spec.original_platform
platform = Gem::Platform::RUBY if platform.nil? or platform.empty?
[spec.name, spec.version, platform]
end
specs = compact_specs specs
Marshal.dump specs, io
end
say "Generating quick index"
quick_index = File.join @quick_dir, 'index'
open quick_index, 'wb' do |io|
io.puts index.sort.map { |_, spec| spec.original_name }
end
say "Generating latest index"
latest_index = File.join @quick_dir, 'latest_index'
open latest_index, 'wb' do |io|
io.puts index.latest_specs.sort.map { |spec| spec.original_name }
end
say "Generating Marshal master index"
open @marshal_index, 'wb' do |io|
io.write index.dump
end
progress = ui.progress_reporter index.size,
"Generating YAML master index for #{index.size} gems (this may take a while)",
"Complete"
open @master_index, 'wb' do |io|
io.puts "--- !ruby/object:#{index.class}"
io.puts "gems:"
gems = index.sort_by { |name, gemspec| gemspec.sort_obj }
gems.each do |original_name, gemspec|
yaml = gemspec.to_yaml.gsub(/^/, ' ')
yaml = yaml.sub(/\A ---/, '') # there's a needed extra ' ' here
io.print " #{original_name}:"
io.puts yaml
progress.updated original_name
end
end
progress.done
say "Compressing indicies"
# use gzip for future files.
compress quick_index, 'rz'
paranoid quick_index, 'rz'
compress latest_index, 'rz'
paranoid latest_index, 'rz'
compress @marshal_index, 'Z'
paranoid @marshal_index, 'Z'
compress @master_index, 'Z'
paranoid @master_index, 'Z'
gzip @specs_index
gzip @latest_specs_index
end
##
# Collect specifications from .gem files from the gem directory.
def collect_specs
index = Gem::SourceIndex.new
progress = ui.progress_reporter gem_file_list.size,
"Loading #{gem_file_list.size} gems from #{@dest_directory}",
"Loaded all gems"
gem_file_list.each do |gemfile|
if File.size(gemfile.to_s) == 0 then
alert_warning "Skipping zero-length gem: #{gemfile}"
next
end
begin
spec = Gem::Format.from_file_by_path(gemfile).spec
unless gemfile =~ /\/#{Regexp.escape spec.original_name}.*\.gem\z/i then
alert_warning "Skipping misnamed gem: #{gemfile} => #{spec.full_name} (#{spec.original_name})"
next
end
abbreviate spec
sanitize spec
index.gems[spec.original_name] = spec
progress.updated spec.original_name
rescue SignalException => e
alert_error "Received signal, exiting"
raise
rescue Exception => e
alert_error "Unable to process #{gemfile}\n#{e.message} (#{e.class})\n\t#{e.backtrace.join "\n\t"}"
end
end
progress.done
index
end
##
# Compacts Marshal output for the specs index data source by using identical
# objects as much as possible.
def compact_specs(specs)
names = {}
versions = {}
platforms = {}
specs.map do |(name, version, platform)|
names[name] = name unless names.include? name
versions[version] = version unless versions.include? version
platforms[platform] = platform unless platforms.include? platform
[names[name], versions[version], platforms[platform]]
end
end
##
# Compress +filename+ with +extension+.
def compress(filename, extension)
data = Gem.read_binary filename
zipped = Gem.deflate data
open "#{filename}.#{extension}", 'wb' do |io|
io.write zipped
end
end
##
# List of gem file names to index.
def gem_file_list
Dir.glob(File.join(@dest_directory, "gems", "*.gem"))
end
##
# Builds and installs indexicies.
def generate_index
make_temp_directories
index = collect_specs
build_indicies index
install_indicies
rescue SignalException
ensure
FileUtils.rm_rf @directory
end
##
# Zlib::GzipWriter wrapper that gzips +filename+ on disk.
def gzip(filename)
Zlib::GzipWriter.open "#{filename}.gz" do |io|
io.write Gem.read_binary(filename)
end
end
##
# Install generated indicies into the destination directory.
def install_indicies
verbose = Gem.configuration.really_verbose
say "Moving index into production dir #{@dest_directory}" if verbose
@files.each do |file|
src_name = File.join @directory, file
dst_name = File.join @dest_directory, file
FileUtils.rm_rf dst_name, :verbose => verbose
FileUtils.mv src_name, @dest_directory, :verbose => verbose,
:force => true
end
end
##
# Make directories for index generation
def make_temp_directories
FileUtils.rm_rf @directory
FileUtils.mkdir_p @directory, :mode => 0700
FileUtils.mkdir_p @quick_marshal_dir
end
##
# Ensure +path+ and path with +extension+ are identical.
def paranoid(path, extension)
data = Gem.read_binary path
compressed_data = Gem.read_binary "#{path}.#{extension}"
unless data == Gem.inflate(compressed_data) then
raise "Compressed file #{compressed_path} does not match uncompressed file #{path}"
end
end
##
# Sanitize the descriptive fields in the spec. Sometimes non-ASCII
# characters will garble the site index. Non-ASCII characters will
# be replaced by their XML entity equivalent.
def sanitize(spec)
spec.summary = sanitize_string(spec.summary)
spec.description = sanitize_string(spec.description)
spec.post_install_message = sanitize_string(spec.post_install_message)
spec.authors = spec.authors.collect { |a| sanitize_string(a) }
spec
end
##
# Sanitize a single string.
def sanitize_string(string)
# HACK the #to_s is in here because RSpec has an Array of Arrays of
# Strings for authors. Need a way to disallow bad values on gempsec
# generation. (Probably won't happen.)
string ? string.to_s.to_xs : string
end
end
|