Object
Top level class for building the gem repository index.
Create an indexer that will index the gems in directory.
# File lib/rubygems/indexer.rb, line 56 56: def initialize(directory, options = {}) 57: require 'fileutils' 58: require 'tmpdir' 59: require 'zlib' 60: 61: unless defined?(Builder::XChar) then 62: raise "Gem::Indexer requires that the XML Builder library be installed:" "\n\tgem install builder" 63: end 64: 65: options = { :build_legacy => true, :build_modern => true }.merge options 66: 67: @build_legacy = options[:build_legacy] 68: @build_modern = options[:build_modern] 69: 70: @rss_title = options[:rss_title] 71: @rss_host = options[:rss_host] 72: @rss_gems_host = options[:rss_gems_host] 73: 74: @dest_directory = directory 75: @directory = File.join(Dir.tmpdir, "gem_generate_index_#{$$}") 76: 77: marshal_name = "Marshal.#{Gem.marshal_version}" 78: 79: @master_index = File.join @directory, 'yaml' 80: @marshal_index = File.join @directory, marshal_name 81: 82: @quick_dir = File.join @directory, 'quick' 83: @quick_marshal_dir = File.join @quick_dir, marshal_name 84: @quick_marshal_dir_base = File.join "quick", marshal_name # FIX: UGH 85: 86: @quick_index = File.join @quick_dir, 'index' 87: @latest_index = File.join @quick_dir, 'latest_index' 88: 89: @specs_index = File.join @directory, "specs.#{Gem.marshal_version}" 90: @latest_specs_index = 91: File.join(@directory, "latest_specs.#{Gem.marshal_version}") 92: @prerelease_specs_index = 93: File.join(@directory, "prerelease_specs.#{Gem.marshal_version}") 94: @dest_specs_index = 95: File.join(@dest_directory, "specs.#{Gem.marshal_version}") 96: @dest_latest_specs_index = 97: File.join(@dest_directory, "latest_specs.#{Gem.marshal_version}") 98: @dest_prerelease_specs_index = 99: File.join(@dest_directory, "prerelease_specs.#{Gem.marshal_version}") 100: 101: @rss_index = File.join @directory, 'index.rss' 102: 103: @files = [] 104: end
Abbreviate the spec for downloading. Abbreviated specs are only used for searching, downloading and related activities and do not need deployment specific information (e.g. list of files). So we abbreviate the spec, making it much smaller for quicker downloads.
# File lib/rubygems/indexer.rb, line 113 113: def abbreviate(spec) 114: spec.files = [] 115: spec.test_files = [] 116: spec.rdoc_options = [] 117: spec.extra_rdoc_files = [] 118: spec.cert_chain = [] 119: spec 120: end
Build various indicies
# File lib/rubygems/indexer.rb, line 125 125: def build_indicies 126: # Marshal gemspecs are used by both modern and legacy RubyGems 127: 128: Gem::Specification.dirs = [] 129: Gem::Specification.add_specs(*map_gems_to_specs(gem_file_list)) 130: 131: build_marshal_gemspecs 132: build_legacy_indicies if @build_legacy 133: build_modern_indicies if @build_modern 134: build_rss 135: 136: compress_indicies 137: end
Builds indicies for RubyGems older than 1.2.x
# File lib/rubygems/indexer.rb, line 142 142: def build_legacy_indicies 143: index = collect_specs 144: 145: say "Generating Marshal master index" 146: 147: Gem.time 'Generated Marshal master index' do 148: open @marshal_index, 'wb' do |io| 149: io.write index.dump 150: end 151: end 152: 153: @files << @marshal_index 154: @files << "#{@marshal_index}.Z" 155: end
Builds Marshal quick index gemspecs.
# File lib/rubygems/indexer.rb, line 160 160: def build_marshal_gemspecs 161: count = Gem::Specification.count 162: progress = ui.progress_reporter count, 163: "Generating Marshal quick index gemspecs for #{count} gems", 164: "Complete" 165: 166: files = [] 167: 168: Gem.time 'Generated Marshal quick index gemspecs' do 169: Gem::Specification.each do |spec| 170: spec_file_name = "#{spec.original_name}.gemspec.rz" 171: marshal_name = File.join @quick_marshal_dir, spec_file_name 172: 173: marshal_zipped = Gem.deflate Marshal.dump(spec) 174: open marshal_name, 'wb' do |io| io.write marshal_zipped end 175: 176: files << marshal_name 177: 178: progress.updated spec.original_name 179: end 180: 181: progress.done 182: end 183: 184: @files << @quick_marshal_dir 185: 186: files 187: end
Build a single index for RubyGems 1.2 and newer
# File lib/rubygems/indexer.rb, line 192 192: def build_modern_index(index, file, name) 193: say "Generating #{name} index" 194: 195: Gem.time "Generated #{name} index" do 196: open(file, 'wb') do |io| 197: specs = index.map do |*spec| 198: # We have to splat here because latest_specs is an array, while the 199: # others are hashes. 200: spec = spec.flatten.last 201: platform = spec.original_platform 202: 203: # win32-api-1.0.4-x86-mswin32-60 204: unless String === platform then 205: alert_warning "Skipping invalid platform in gem: #{spec.full_name}" 206: next 207: end 208: 209: platform = Gem::Platform::RUBY if platform.nil? or platform.empty? 210: [spec.name, spec.version, platform] 211: end 212: 213: specs = compact_specs(specs) 214: Marshal.dump(specs, io) 215: end 216: end 217: end
Builds indicies for RubyGems 1.2 and newer. Handles full, latest, prerelease
# File lib/rubygems/indexer.rb, line 222 222: def build_modern_indicies 223: prerelease, released = Gem::Specification.partition { |s| 224: s.version.prerelease? 225: } 226: latest_specs = Gem::Specification.latest_specs 227: 228: build_modern_index(released.sort, @specs_index, 'specs') 229: build_modern_index(latest_specs.sort, @latest_specs_index, 'latest specs') 230: build_modern_index(prerelease.sort, @prerelease_specs_index, 231: 'prerelease specs') 232: 233: @files += [@specs_index, 234: "#{@specs_index}.gz", 235: @latest_specs_index, 236: "#{@latest_specs_index}.gz", 237: @prerelease_specs_index, 238: "#{@prerelease_specs_index}.gz"] 239: end
Builds an RSS feed for past two days gem releases according to the gem’s date.
# File lib/rubygems/indexer.rb, line 245 245: def build_rss 246: if @rss_host.nil? or @rss_gems_host.nil? then 247: if Gem.configuration.really_verbose then 248: alert_warning "no --rss-host or --rss-gems-host, RSS generation disabled" 249: end 250: return 251: end 252: 253: require 'cgi' 254: require 'rubygems/text' 255: 256: extend Gem::Text 257: 258: Gem.time 'Generated rss' do 259: open @rss_index, 'wb' do |io| 260: rss_host = CGI.escapeHTML @rss_host 261: rss_title = CGI.escapeHTML(@rss_title || 'gems') 262: 263: io.puts <?xml version="1.0"?><rss version="2.0"> <channel> <title>#{rss_title}</title> <link>http://#{rss_host}</link> <description>Recently released gems from http://#{rss_host}</description> <generator>RubyGems v#{Gem::VERSION}</generator> <docs>http://cyber.law.harvard.edu/rss/rss.html</docs> 264: 265: today = Gem::Specification::TODAY 266: yesterday = today - 86400 267: 268: index = Gem::Specification.select do |spec| 269: spec_date = spec.date 270: # TODO: remove this and make YAML based specs properly normalized 271: spec_date = Time.parse(spec_date.to_s) if Date === spec_date 272: 273: spec_date >= yesterday && spec_date <= today 274: end 275: 276: index.sort_by { |spec| [-spec.date.to_i, spec] }.each do |spec| 277: file_name = File.basename spec.cache_file 278: gem_path = CGI.escapeHTML "http://#{@rss_gems_host}/gems/#{file_name}" 279: size = File.stat(spec.loaded_from).size # rescue next 280: 281: description = spec.description || spec.summary || '' 282: authors = Array spec.authors 283: emails = Array spec.email 284: authors = emails.zip(authors).map do |email, author| 285: email += " (#{author})" if author and not author.empty? 286: end.join ', ' 287: 288: description = description.split(/\n\n+/).map do |chunk| 289: format_text chunk, 78 290: end 291: 292: description = description.join "\n\n" 293: 294: item = '' 295: 296: item << <item> <title>#{CGI.escapeHTML spec.full_name}</title> <description><pre>#{CGI.escapeHTML description.chomp}</pre> </description> <author>#{CGI.escapeHTML authors}</author> <guid>#{CGI.escapeHTML spec.full_name}</guid> <enclosure url=\"#{gem_path}\" length=\"#{size}\" type=\"application/octet-stream\" /> <pubDate>#{spec.date.rfc2822}</pubDate> 297: 298: item << <link>#{CGI.escapeHTML spec.homepage}</link> if spec.homepage 299: 300: item << </item> 301: 302: io.puts item 303: end 304: 305: io.puts </channel></rss> 306: end 307: end 308: 309: @files << @rss_index 310: end
Collect specifications from .gem files from the gem directory.
# File lib/rubygems/indexer.rb, line 379 379: def collect_specs(gems = gem_file_list) 380: Gem::Deprecate.skip_during do 381: index = Gem::SourceIndex.new 382: 383: map_gems_to_specs(gems).each do |spec| 384: index.add_spec spec, spec.original_name 385: end 386: 387: index 388: end 389: end
Compacts Marshal output for the specs index data source by using identical objects as much as possible.
# File lib/rubygems/indexer.rb, line 417 417: def compact_specs(specs) 418: names = {} 419: versions = {} 420: platforms = {} 421: 422: specs.map do |(name, version, platform)| 423: names[name] = name unless names.include? name 424: versions[version] = version unless versions.include? version 425: platforms[platform] = platform unless platforms.include? platform 426: 427: [names[name], versions[version], platforms[platform]] 428: end 429: end
Compress filename with extension.
# File lib/rubygems/indexer.rb, line 434 434: def compress(filename, extension) 435: data = Gem.read_binary filename 436: 437: zipped = Gem.deflate data 438: 439: open "#{filename}.#{extension}", 'wb' do |io| 440: io.write zipped 441: end 442: end
Compresses indicies on disk
# File lib/rubygems/indexer.rb, line 396 396: def compress_indicies 397: say "Compressing indicies" 398: 399: Gem.time 'Compressed indicies' do 400: if @build_legacy then 401: compress @marshal_index, 'Z' 402: paranoid @marshal_index, 'Z' 403: end 404: 405: if @build_modern then 406: gzip @specs_index 407: gzip @latest_specs_index 408: gzip @prerelease_specs_index 409: end 410: end 411: end
List of gem file names to index.
# File lib/rubygems/indexer.rb, line 447 447: def gem_file_list 448: Dir[File.join(@dest_directory, "gems", '*.gem')] 449: end
Builds and installs indicies.
# File lib/rubygems/indexer.rb, line 454 454: def generate_index 455: make_temp_directories 456: build_indicies 457: install_indicies 458: rescue SignalException 459: ensure 460: FileUtils.rm_rf @directory 461: end
Zlib::GzipWriter wrapper that gzips filename on disk.
# File lib/rubygems/indexer.rb, line 466 466: def gzip(filename) 467: Zlib::GzipWriter.open "#{filename}.gz" do |io| 468: io.write Gem.read_binary(filename) 469: end 470: end
Install generated indicies into the destination directory.
# File lib/rubygems/indexer.rb, line 475 475: def install_indicies 476: verbose = Gem.configuration.really_verbose 477: 478: say "Moving index into production dir #{@dest_directory}" if verbose 479: 480: files = @files 481: files.delete @quick_marshal_dir if files.include? @quick_dir 482: 483: if files.include? @quick_marshal_dir and not files.include? @quick_dir then 484: files.delete @quick_marshal_dir 485: 486: dst_name = File.join(@dest_directory, @quick_marshal_dir_base) 487: 488: FileUtils.mkdir_p File.dirname(dst_name), :verbose => verbose 489: FileUtils.rm_rf dst_name, :verbose => verbose 490: FileUtils.mv(@quick_marshal_dir, dst_name, 491: :verbose => verbose, :force => true) 492: end 493: 494: files = files.map do |path| 495: path.sub(/^#{Regexp.escape @directory}\/?/, '') # HACK? 496: end 497: 498: files.each do |file| 499: src_name = File.join @directory, file 500: dst_name = File.join @dest_directory, file 501: 502: FileUtils.rm_rf dst_name, :verbose => verbose 503: FileUtils.mv(src_name, @dest_directory, 504: :verbose => verbose, :force => true) 505: end 506: end
Make directories for index generation
# File lib/rubygems/indexer.rb, line 511 511: def make_temp_directories 512: FileUtils.rm_rf @directory 513: FileUtils.mkdir_p @directory, :mode => 0700 514: FileUtils.mkdir_p @quick_marshal_dir 515: end
# File lib/rubygems/indexer.rb, line 339 339: def map_gems_to_specs gems 340: gems.map { |gemfile| 341: if File.size(gemfile) == 0 then 342: alert_warning "Skipping zero-length gem: #{gemfile}" 343: next 344: end 345: 346: begin 347: spec = Gem::Format.from_file_by_path(gemfile).spec 348: spec.loaded_from = gemfile 349: 350: # HACK: fuck this shit - borks all tests that use pl1 351: # if File.basename(gemfile, ".gem") != spec.original_name then 352: # exp = spec.full_name 353: # exp << " (#{spec.original_name})" if 354: # spec.original_name != spec.full_name 355: # msg = "Skipping misnamed gem: #{gemfile} should be named #{exp}" 356: # alert_warning msg 357: # next 358: # end 359: 360: abbreviate spec 361: sanitize spec 362: 363: spec 364: rescue SignalException => e 365: alert_error "Received signal, exiting" 366: raise 367: rescue Exception => e 368: msg = ["Unable to process #{gemfile}", 369: "#{e.message} (#{e.class})", 370: "\t#{e.backtrace.join "\n\t"}"].join("\n") 371: alert_error msg 372: end 373: }.compact 374: end
Ensure path and path with extension are identical.
# File lib/rubygems/indexer.rb, line 520 520: def paranoid(path, extension) 521: data = Gem.read_binary path 522: compressed_data = Gem.read_binary "#{path}.#{extension}" 523: 524: unless data == Gem.inflate(compressed_data) then 525: raise "Compressed file #{compressed_path} does not match uncompressed file #{path}" 526: end 527: end
Sanitize the descriptive fields in the spec. Sometimes non-ASCII characters will garble the site index. Non-ASCII characters will be replaced by their XML entity equivalent.
# File lib/rubygems/indexer.rb, line 534 534: def sanitize(spec) 535: spec.summary = sanitize_string(spec.summary) 536: spec.description = sanitize_string(spec.description) 537: spec.post_install_message = sanitize_string(spec.post_install_message) 538: spec.authors = spec.authors.collect { |a| sanitize_string(a) } 539: 540: spec 541: end
Sanitize a single string.
# File lib/rubygems/indexer.rb, line 546 546: def sanitize_string(string) 547: return string unless string 548: 549: # HACK the #to_s is in here because RSpec has an Array of Arrays of 550: # Strings for authors. Need a way to disallow bad values on gemspec 551: # generation. (Probably won't happen.) 552: string = string.to_s 553: 554: begin 555: Builder::XChar.encode string 556: rescue NameError, NoMethodError 557: string.to_xs 558: end 559: end
Perform an in-place update of the repository from newly added gems. Only works for modern indicies, and sets # to false when run.
# File lib/rubygems/indexer.rb, line 565 565: def update_index 566: @build_legacy = false 567: 568: make_temp_directories 569: 570: specs_mtime = File.stat(@dest_specs_index).mtime 571: newest_mtime = Time.at 0 572: 573: updated_gems = gem_file_list.select do |gem| 574: gem_mtime = File.stat(gem).mtime 575: newest_mtime = gem_mtime if gem_mtime > newest_mtime 576: gem_mtime >= specs_mtime 577: end 578: 579: if updated_gems.empty? then 580: say 'No new gems' 581: terminate_interaction 0 582: end 583: 584: specs = map_gems_to_specs updated_gems 585: prerelease, released = specs.partition { |s| s.version.prerelease? } 586: 587: files = build_marshal_gemspecs 588: 589: Gem.time 'Updated indexes' do 590: update_specs_index released, @dest_specs_index, @specs_index 591: update_specs_index released, @dest_latest_specs_index, @latest_specs_index 592: update_specs_index(prerelease, 593: @dest_prerelease_specs_index, 594: @prerelease_specs_index) 595: end 596: 597: compress_indicies 598: 599: verbose = Gem.configuration.really_verbose 600: 601: say "Updating production dir #{@dest_directory}" if verbose 602: 603: files << @specs_index 604: files << "#{@specs_index}.gz" 605: files << @latest_specs_index 606: files << "#{@latest_specs_index}.gz" 607: files << @prerelease_specs_index 608: files << "#{@prerelease_specs_index}.gz" 609: 610: files = files.map do |path| 611: path.sub(/^#{Regexp.escape @directory}\/?/, '') # HACK? 612: end 613: 614: files.each do |file| 615: src_name = File.join @directory, file 616: dst_name = File.join @dest_directory, file # REFACTOR: duped above 617: 618: FileUtils.mv src_name, dst_name, :verbose => verbose, 619: :force => true 620: 621: File.utime newest_mtime, newest_mtime, dst_name 622: end 623: end
Combines specs in index and source then writes out a new copy to dest. For a latest index, does not ensure the new file is minimal.
# File lib/rubygems/indexer.rb, line 629 629: def update_specs_index(index, source, dest) 630: specs_index = Marshal.load Gem.read_binary(source) 631: 632: index.each do |spec| 633: platform = spec.original_platform 634: platform = Gem::Platform::RUBY if platform.nil? or platform.empty? 635: specs_index << [spec.name, spec.version, platform] 636: end 637: 638: specs_index = compact_specs specs_index.uniq.sort 639: 640: open dest, 'wb' do |io| 641: Marshal.dump specs_index, io 642: end 643: end
Disabled; run with --debug to generate this.
Generated with the Darkfish Rdoc Generator 1.1.6.