[Rubygems-developers] [PATCH] New .gem package format
Mauricio Fernández
batsman.geo at yahoo.com
Tue Jul 13 09:08:59 EDT 2004
I've merged the package.rb from rpa-base with some minor modifications
(mostly the ability to operate on IO objects directly instead of
filenames).
The new format is based on tar + gzip; it is structured as follows:
* the .gem is an uncompressed tar containing 2 files, metadata.gz
and data.tar.gz
* metadata.gz is a compressed YAML-serialization of the spec
* data.tar.gz is a compressed tar holding the actual data
TODO:
* how to handle the global MD5 for the .gem? Cheapest way, adding a top-level
entry ("digest" for instance) padded with zeros, later sub the text directly
* replace all the raise "foo" with exceptions appropriate for
RubyGems
* file format versioning: again, the easiest way is a top-level entry
I broke 2 assertions in test_format.rb in the process, and added 20
tests with 453 assertions in test_package.rb (mostly testing the tar
routines), which pass.
PS: I used sw=4 in my code, feel free to reformat to 2 spaces for
RubyGems
diff -ruN rubygems.orig/lib/rubygems/builder.rb rubygems/lib/rubygems/builder.rb
--- rubygems.orig/lib/rubygems/builder.rb 2004-07-13 13:13:23.000000000 +0200
+++ rubygems/lib/rubygems/builder.rb 2004-07-13 14:22:37.000000000 +0200
@@ -1,3 +1,6 @@
+require "rubygems/package"
+require "yaml"
+
module Gem
##
@@ -7,70 +10,6 @@
class Builder
include UserInteraction
- require 'stringio'
-
- ##
- # Builder::FileContents is the file contents
- #
- class FileContents < StringIO
- def add_ruby_header
- self.puts <<-EOS
-MD5SUM = "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"
-if $0 == __FILE__
- require 'optparse'
-
- options = {}
- ARGV.options do |opts|
- opts.on_tail("--help", "show this message") {puts opts; exit}
- opts.on('--dir=DIRNAME', "Installation directory for the Gem") {|options[:directory]|}
- opts.on('--force', "Force Gem to intall, bypassing dependency checks") {|options[:force]|}
- opts.on('--gen-rdoc', "Generate RDoc documentation for the Gem") {|options[:gen_rdoc]|}
- opts.parse!
- end
-
- require 'rubygems'
- Gem::manage_gems
- @directory = options[:directory] || Gem.dir
- @force = options[:force]
-
- gem = Gem::Installer.new(__FILE__).install(@force, @directory)
- if options[:gen_rdoc]
- Gem::DocManager.new(gem).generate_rdoc
- end
-end
-
-__END__
- EOS
- end
-
- ##
- # Reads the files listed in the specification and encodes
- # them into the provided file (IO)
- #
- # file:: [IO] the file to write the encoded data into
- #
- def write_files_to(files)
- require 'zlib'
- file_header = []
- files.each do |file_name|
- next if File.directory? file_name
- file_header << { "path" => file_name,
- "size" => File.size(file_name),
- "mode" => File.stat(file_name).mode & 0777
- }
- end
- self.puts file_header.to_yaml
- file_header.each do |entry|
- data = [Zlib::Deflate.deflate(File.open(entry['path'], "rb") {|f| f.read})].pack("m")
- self.puts "---"
- self.puts data
- end
- end
- def md5
- MD5.md5(string)
- end
- end
-
##
# Constructs a builder instance for the provided specification
#
@@ -86,21 +25,20 @@
def build
@spec.mark_version
@spec.validate
- require 'yaml'
- require 'md5'
- require 'stringio'
file_name = @spec.full_name+".gem"
- file_contents = FileContents.new
-
- file_contents.add_ruby_header
- file_contents.puts(@spec.to_yaml)
- file_contents.write_files_to(@spec.files)
- say success
- md5 = file_contents.md5
- File.open(file_name, "w") do |file|
- file.write(file_contents.string.gsub(/MD5SUM =.*$/, "MD5SUM = \"#{md5.to_s}\""))
+
+ Package.open(file_name, "w") do |pkg|
+ pkg.metadata = @spec.to_yaml
+ @spec.files.each do |file|
+ next if File.directory? file
+ pkg.add_file_simple(file, File.stat(file_name).mode & 0777,
+ File.size(file)) do |os|
+ os.write File.open(file, "rb"){|f|f.read}
+ end
+ end
end
+ say success
file_name
end
diff -ruN rubygems.orig/lib/rubygems/format.rb rubygems/lib/rubygems/format.rb
--- rubygems.orig/lib/rubygems/format.rb 2004-07-13 13:13:23.000000000 +0200
+++ rubygems/lib/rubygems/format.rb 2004-07-13 14:18:44.000000000 +0200
@@ -1,3 +1,5 @@
+require 'rubygems/package'
+
module Gem
##
@@ -38,9 +40,8 @@
raise exception
end
require 'fileutils'
- File.open(file_path, 'r') do |file|
- from_io(file, file_path)
- end
+ f = File.open(file_path, 'rb')
+ from_io(f, file_path)
end
##
@@ -51,102 +52,16 @@
#
def self.from_io(io, gem_path="(io)")
format = self.new(gem_path)
- skip_ruby(io)
- format.spec = read_spec(io)
- format.file_entries = []
- read_files_from_gem(io) do |entry, file_data|
- format.file_entries << [entry, file_data]
- end
- format
- end
-
- private
- ##
- # Skips the Ruby self-install header. After calling this method, the
- # IO index will be set after the Ruby code.
- #
- # file:: [IO] The IO to process (skip the Ruby code)
- #
- def self.skip_ruby(file)
- end_seen = false
- loop {
- line = file.gets
- if(line == nil || line.chomp == "__END__") then
- end_seen = true
- break
- end
- }
- if(end_seen == false) then
- raise FormatException.new("Failed to find end of ruby script while reading gem")
- end
- end
-
- ##
- # Reads the specification YAML from the supplied IO and constructs
- # a Gem::Specification from it. After calling this method, the
- # IO index will be set after the specification header.
- #
- # file:: [IO] The IO to process
- #
- def self.read_spec(file)
- require 'yaml'
- yaml = ''
- begin
- read_until_dashes(file) do |line|
- yaml << line
+ Package.open_from_io(io) do |pkg|
+ format.spec = pkg.metadata
+ format.file_entries = []
+ pkg.each do |entry|
+ format.file_entries << [{"size", entry.size, "mode", entry.mode,
+ "path", entry.full_name}, entry.read]
end
- YAML.load(yaml)
- rescue YAML::Error => e
- raise FormatException.new("Failed to parse gem specification out of gem file")
- rescue ArgumentError => e
- raise FormatException.new("Failed to parse gem specification out of gem file")
- end
- end
-
- ##
- # Reads lines from the supplied IO until a end-of-yaml (---) is
- # reached
- #
- # file:: [IO] The IO to process
- # block:: [String] The read line
- #
- def self.read_until_dashes(file)
- while((line = file.gets) && line.chomp.strip != "---") do
- yield line
end
+ format
end
-
- ##
- # Reads the embedded file data from a gem file, yielding an entry
- # containing metadata about the file and the file contents themselves
- # for each file that's archived in the gem.
- # NOTE: Many of these methods should be extracted into some kind of
- # Gem file read/writer
- #
- # gem_file:: [IO] The IO to process
- #
- def self.read_files_from_gem(gem_file)
- require 'zlib'
- require 'yaml'
- errstr = "Error reading files from gem"
- header_yaml = ''
- begin
- self.read_until_dashes(gem_file) do |line|
- header_yaml << line
- end
- header = YAML.load(header_yaml)
- raise FormatException.new(errstr) unless header
- header.each do |entry|
- file_data = ''
- self.read_until_dashes(gem_file) do |line|
- file_data << line
- end
- yield [entry, Zlib::Inflate.inflate(file_data.strip.unpack("m")[0])]
- end
- rescue Exception,Zlib::DataError => e
- raise FormatException.new(errstr)
- end
- end
end
end
diff -ruN rubygems.orig/lib/rubygems/package.rb rubygems/lib/rubygems/package.rb
--- rubygems.orig/lib/rubygems/package.rb 1970-01-01 01:00:00.000000000 +0100
+++ rubygems/lib/rubygems/package.rb 2004-07-13 14:17:35.000000000 +0200
@@ -0,0 +1,711 @@
+#
+# Copyright (C) 2004 Mauricio Julio Fernández Pradier
+# See LICENSE.txt for additional licensing information.
+#
+
+require 'yaml'
+require 'yaml/syck'
+require 'fileutils'
+
+module Gem
+
+# Wrapper for FileUtils meant to provide logging and additional operations if
+# needed.
+class FileOperations
+ require 'fileutils'
+ extend FileUtils
+ class << self
+ # additional methods not implemented in FileUtils
+ end
+ def initialize(logger = nil)
+ @logger = logger
+ end
+
+ def method_missing(meth, *args, &block)
+ case
+ when FileUtils.respond_to?(meth)
+ @logger.log "#{meth}: #{args}" if @logger
+ FileUtils.send meth, *args, &block
+ when FileOperations.respond_to?(meth)
+ @logger.log "#{meth}: #{args}" if @logger
+ FileOperations.send meth, *args, &block
+ else
+ super
+ end
+ end
+end
+
+
+module Package
+
+class NonSeekableIO < StandardError; end
+class ArgumentError < ::ArgumentError; end
+class ClosedIO < StandardError; end
+class BadCheckSum < StandardError; end
+class TooLongFileName < StandardError; end
+
+module FSyncDir
+ private
+ def fsync_dir(dirname)
+ # make sure this hits the disc
+ begin
+ dir = open(dirname, "r")
+ dir.fsync
+ rescue # ignore IOError if it's an unpatched (old) Ruby
+ ensure
+ dir.close if dir rescue nil
+ end
+ end
+end
+
+class TarHeader
+ FIELDS = [:name, :mode, :uid, :gid, :size, :mtime, :checksum, :typeflag,
+ :linkname, :magic, :version, :uname, :gname, :devmajor,
+ :devminor, :prefix]
+ FIELDS.each {|x| attr_reader x}
+
+ def self.new_from_stream(stream)
+ data = stream.read(512)
+ fields = data.unpack( "A100" + # record name
+ "A8A8A8" + # mode, uid, gid
+ "A12A12" + # size, mtime
+ "A8A" + # checksum, typeflag
+ "A100" + # linkname
+ "A6A2" + # magic, version
+ "A32" + # uname
+ "A32" + # gname
+ "A8A8" + # devmajor, devminor
+ "A155" # prefix
+ )
+ name = fields.shift
+ mode = fields.shift.oct
+ uid = fields.shift.oct
+ gid = fields.shift.oct
+ size = fields.shift.oct
+ mtime = fields.shift.oct
+ checksum = fields.shift.oct
+ typeflag = fields.shift
+ linkname = fields.shift
+ magic = fields.shift
+ version = fields.shift.oct
+ uname = fields.shift
+ gname = fields.shift
+ devmajor = fields.shift.oct
+ devminor = fields.shift.oct
+ prefix = fields.shift
+
+ empty = (data == "\0" * 512)
+
+ new(:name=>name, :mode=>mode, :uid=>uid, :gid=>gid, :size=>size,
+ :mtime=>mtime, :checksum=>checksum, :typeflag=>typeflag, :magic=>magic,
+ :version=>version, :uname=>uname, :gname=>gname, :devmajor=>devmajor,
+ :devminor=>devminor, :prefix=>prefix, :empty => empty )
+ end
+
+ def initialize(vals)
+ unless vals[:name] && vals[:size] && vals[:prefix] && vals[:mode]
+ raise Package::ArgumentError
+ end
+ vals[:uid] ||= 0
+ vals[:gid] ||= 0
+ vals[:mtime] ||= 0
+ vals[:checksum] ||= ""
+ vals[:typeflag] ||= "0"
+ vals[:magic] ||= "ustar"
+ vals[:version] ||= "00"
+ vals[:uname] ||= "wheel"
+ vals[:gname] ||= "wheel"
+ vals[:devmajor] ||= 0
+ vals[:devminor] ||= 0
+ FIELDS.each {|x| instance_variable_set "@#{x.to_s}", vals[x]}
+ @empty = vals[:empty]
+ end
+
+ def empty?
+ @empty
+ end
+
+ def to_s
+ update_checksum
+ header(checksum)
+ end
+
+ def update_checksum
+ h = header(" " * 8)
+ @checksum = oct(calculate_checksum(h), 6)
+ end
+
+ private
+ def oct(num, len)
+ "%0#{len}o" % num
+ end
+
+ def calculate_checksum(hdr)
+ hdr.unpack("C*").inject{|a,b| a+b}
+ end
+
+ def header(chksum)
+# struct tarfile_entry_posix {
+# char name[100]; # ASCII + (Z unless filled)
+# char mode[8]; # 0 padded, octal, null
+# char uid[8]; # ditto
+# char gid[8]; # ditto
+# char size[12]; # 0 padded, octal, null
+# char mtime[12]; # 0 padded, octal, null
+# char checksum[8]; # 0 padded, octal, null, space
+# char typeflag[1]; # file: "0" dir: "5"
+# char linkname[100]; # ASCII + (Z unless filled)
+# char magic[6]; # "ustar\0"
+# char version[2]; # "00"
+# char uname[32]; # ASCIIZ
+# char gname[32]; # ASCIIZ
+# char devmajor[8]; # 0 padded, octal, null
+# char devminor[8]; # o padded, octal, null
+# char prefix[155]; # ASCII + (Z unless filled)
+# };
+ arr = [name, oct(mode, 7), oct(uid, 7), oct(gid, 7), oct(size, 11),
+ oct(mtime, 11), chksum, " ", typeflag, linkname, magic, version,
+ uname, gname, oct(devmajor, 7), oct(devminor, 7), prefix]
+ str = arr.pack("a100a8a8a8a12a12" + # name, mode, uid, gid, size, mtime
+ "a7aaa100a6a2" + # chksum, typeflag, linkname, magic, version
+ "a32a32a8a8a155") # uname, gname, devmajor, devminor, prefix
+ str + "\0" * ((512 - str.size) % 512)
+ end
+end
+
+class TarWriter
+ class FileOverflow < StandardError; end
+ class BlockNeeded < StandardError; end
+
+ class BoundedStream
+ attr_reader :limit, :written
+ def initialize(io, limit)
+ @io = io
+ @limit = limit
+ @written = 0
+ end
+
+ def write(data)
+ if data.size + @written > @limit
+ raise FileOverflow,
+ "You tried to feed more data than fits in the file."
+ end
+ @io.write data
+ @written += data.size
+ data.size
+ end
+ end
+ class RestrictedStream
+ def initialize(anIO)
+ @io = anIO
+ end
+
+ def write(data)
+ @io.write data
+ end
+ end
+
+ def self.new(anIO)
+ writer = super(anIO)
+ return writer unless block_given?
+ begin
+ yield writer
+ ensure
+ writer.close
+ end
+ nil
+ end
+
+ def initialize(anIO)
+ @io = anIO
+ @closed = false
+ end
+
+ def add_file_simple(name, mode, size)
+ raise BlockNeeded unless block_given?
+ raise ClosedIO if @closed
+ name, prefix = split_name(name)
+ header = TarHeader.new(:name => name, :mode => mode,
+ :size => size, :prefix => prefix).to_s
+ @io.write header
+ os = BoundedStream.new(@io, size)
+ yield os
+ #FIXME: what if an exception is raised in the block?
+ min_padding = size - os.written
+ @io.write("\0" * min_padding)
+ remainder = (512 - (size % 512)) % 512
+ @io.write("\0" * remainder)
+ end
+
+ def add_file(name, mode)
+ raise BlockNeeded unless block_given?
+ raise ClosedIO if @closed
+ raise NonSeekableIO unless @io.respond_to? :pos=
+ name, prefix = split_name(name)
+ init_pos = @io.pos
+ @io.write "\0" * 512 # placeholder for the header
+ yield RestrictedStream.new(@io)
+ #FIXME: what if an exception is raised in the block?
+ #FIXME: what if an exception is raised in the block?
+ size = @io.pos - init_pos - 512
+ remainder = (512 - (size % 512)) % 512
+ @io.write("\0" * remainder)
+ final_pos = @io.pos
+ @io.pos = init_pos
+ header = TarHeader.new(:name => name, :mode => mode,
+ :size => size, :prefix => prefix).to_s
+ @io.write header
+ @io.pos = final_pos
+ end
+
+ def mkdir(name, mode)
+ raise ClosedIO if @closed
+ name, prefix = split_name(name)
+ header = TarHeader.new(:name => name, :mode => mode, :typeflag => "5",
+ :size => 0, :prefix => prefix).to_s
+ @io.write header
+ nil
+ end
+
+ def flush
+ raise ClosedIO if @closed
+ @io.flush if @io.respond_to? :flush
+ end
+
+ def close
+ #raise ClosedIO if @closed
+ return if @closed
+ @io.write "\0" * 1024
+ @closed = true
+ end
+
+ private
+ def split_name name
+ raise TooLongFileName if name.size > 256
+ if name.size <= 100
+ prefix = ""
+ else
+ parts = name.split(/\//)
+ newname = parts.pop
+ nxt = ""
+ loop do
+ nxt = parts.pop
+ break if newname.size + 1 + nxt.size > 100
+ newname = nxt + "/" + newname
+ end
+ prefix = (parts + [nxt]).join "/"
+ name = newname
+ raise TooLongFileName if name.size > 100 || prefix.size > 155
+ end
+ return name, prefix
+ end
+end
+
+class TarReader
+ include Gem::Package
+ class UnexpectedEOF < StandardError; end
+ module InvalidEntry
+ def read(len=nil); raise ClosedIO; end
+ def getc; raise ClosedIO; end
+ def rewind; raise ClosedIO; end
+ end
+ class Entry
+ TarHeader::FIELDS.each{|x| attr_reader x}
+
+ def initialize(header, anIO)
+ @io = anIO
+ @name = header.name
+ @mode = header.mode
+ @uid = header.uid
+ @gid = header.gid
+ @size = header.size
+ @mtime = header.mtime
+ @checksum = header.checksum
+ @typeflag = header.typeflag
+ @linkname = header.linkname
+ @magic = header.magic
+ @version = header.version
+ @uname = header.uname
+ @gname = header.gname
+ @devmajor = header.devmajor
+ @devminor = header.devminor
+ @prefix = header.prefix
+ @read = 0
+ @orig_pos = @io.pos
+ end
+
+ def read(len = nil)
+ return nil if @read >= @size
+ len ||= @size - @read
+ max_read = [len, @size - @read].min
+ ret = @io.read(max_read)
+ @read += ret.size
+ ret
+ end
+
+ def getc
+ return nil if @read >= @size
+ ret = @io.getc
+ @read += 1 if ret
+ ret
+ end
+
+ def is_directory?
+ @typeflag == "5"
+ end
+
+ def is_file?
+ @typeflag == "0"
+ end
+
+ def eof?
+ @read >= @size
+ end
+
+ def pos
+ @read
+ end
+
+ def rewind
+ raise NonSeekableIO unless @io.respond_to? :pos=
+ @io.pos = @orig_pos
+ @read = 0
+ end
+
+ alias_method :is_directory, :is_directory?
+ alias_method :is_file, :is_file
+
+ def bytes_read
+ @read
+ end
+
+ def full_name
+ if @prefix != ""
+ File.join(@prefix, @name)
+ else
+ @name
+ end
+ end
+
+ def close
+ invalidate
+ end
+
+ private
+ def invalidate
+ extend InvalidEntry
+ end
+ end
+
+ def self.new(anIO)
+ reader = super(anIO)
+ return reader unless block_given?
+ begin
+ yield reader
+ ensure
+ reader.close
+ end
+ nil
+ end
+
+ def initialize(anIO)
+ @io = anIO
+ @init_pos = anIO.pos
+ end
+
+ def each(&block)
+ each_entry(&block)
+ end
+
+ # do not call this during a #each or #each_entry iteration
+ def rewind
+ if @init_pos == 0
+ raise NonSeekableIO unless @io.respond_to? :rewind
+ @io.rewind
+ else
+ raise NonSeekableIO unless @io.respond_to? :pos=
+ @io.pos = @init_pos
+ end
+ end
+
+ def each_entry
+ loop do
+ return if @io.eof?
+ header = TarHeader.new_from_stream(@io)
+ return if header.empty?
+ entry = Entry.new header, @io
+ size = entry.size
+ yield entry
+ skip = (512 - (size % 512)) % 512
+ if @io.respond_to? :seek
+ # avoid reading...
+ @io.seek(size - entry.bytes_read, IO::SEEK_CUR)
+ else
+ pending = size - entry.bytes_read
+ while pending > 0
+ bread = @io.read([pending, 4096].min).size
+ raise UnexpectedEOF if @io.eof?
+ pending -= bread
+ end
+ end
+ @io.read(skip) # discard trailing zeros
+ # make sure nobody can use #read, #getc or #rewind anymore
+ entry.close
+ end
+ end
+
+ def close
+ end
+end
+
+class TarInput
+ include FSyncDir
+ include Enumerable
+ attr_reader :metadata
+ require 'zlib'
+ require 'digest/md5'
+ class << self; private :new end
+
+ def initialize(io)
+ @io = io
+ @tarreader = TarReader.new @io
+ has_meta = false
+ @tarreader.each do |entry|
+ case entry.full_name
+ when "metadata"
+ @metadata = YAML.load(entry.read) rescue nil
+ has_meta = true
+ break
+ when "metadata.gz"
+ begin
+ gzis = Zlib::GzipReader.new entry
+ # YAML wants an instance of IO
+ @metadata = YAML.load(gzis) rescue nil
+ has_meta = true
+ ensure
+ gzis.close
+ end
+ end
+ end
+ @tarreader.rewind
+ @fileops = FileOperations.new
+ raise RuntimeError, "No metadata found!" unless has_meta
+ end
+
+ def self.open(filename, &block)
+ open_from_io(File.open(filename, "rb"), &block)
+ end
+
+ def self.open_from_io(io, &block)
+ raise "Want a block" unless block_given?
+ begin
+ is = new(io)
+ yield is
+ ensure
+ is.close if is
+ end
+ end
+
+ def each(&block)
+ @tarreader.each do |entry|
+ next unless entry.full_name == "data.tar.gz"
+ begin
+ is = Zlib::GzipReader.new entry
+ TarReader.new(is) do |inner|
+ inner.each(&block)
+ end
+ ensure
+ is.finish
+ end
+ end
+ @tarreader.rewind
+ end
+
+ def extract_entry(destdir, entry, expected_md5sum = nil)
+ if entry.is_directory?
+ dest = File.join(destdir, entry.full_name)
+ if file_class.dir? dest
+ @fileops.chmod entry.mode, dest
+ else
+ @fileops.mkdir_p(dest, :mode => entry.mode)
+ end
+ fsync_dir dest
+ fsync_dir File.join(dest, "..")
+ return
+ end
+ # it's a file
+ md5 = Digest::MD5.new if expected_md5sum
+ destdir = File.join(destdir, File.dirname(entry.full_name))
+ @fileops.mkdir_p(destdir, :mode => 0755)
+ destfile = File.join(destdir, File.basename(entry.full_name))
+ @fileops.chmod 0600, destfile rescue nil # Errno::ENOENT
+ file_class.open(destfile, "wb", entry.mode) do |os|
+ loop do
+ data = entry.read(4096)
+ break unless data
+ md5 << data if expected_md5sum
+ os.write(data)
+ end
+ os.fsync
+ end
+ @fileops.chmod(entry.mode, destfile)
+ fsync_dir File.dirname(destfile)
+ fsync_dir File.join(File.dirname(destfile), "..")
+ if expected_md5sum && expected_md5sum != md5.hexdigest
+ raise BadCheckSum
+ end
+ end
+
+ def close
+ @io.close
+ @tarreader.close
+ end
+
+ private
+
+ def file_class
+ File
+ end
+end
+
+class TarOutput
+ require 'zlib'
+ require 'yaml'
+
+ class << self; private :new end
+
+ class << self
+
+ end
+
+ def initialize(io)
+ @io = io
+ @external = TarWriter.new @io
+ end
+
+ def external_handle
+ @external
+ end
+
+ def self.open(filename, &block)
+ io = File.open(filename, "wb")
+ open_from_io(io, &block)
+ nil
+ end
+
+ def self.open_from_io(io, &block)
+ outputter = new(io)
+ metadata = nil
+ set_meta = lambda{|x| metadata = x}
+ raise "Want a block" unless block_given?
+ begin
+ outputter.external_handle.add_file("data.tar.gz", 0644) do |inner|
+ begin
+ os = Zlib::GzipWriter.new inner
+ TarWriter.new(os) do |inner_tar_stream|
+ klass = class <<inner_tar_stream; self end
+ klass.send(:define_method, :metadata=, &set_meta)
+ block.call inner_tar_stream
+ end
+ ensure
+ os.flush
+ os.finish
+ #os.close
+ end
+ end
+ outputter.external_handle.add_file("metadata.gz", 0644) do |os|
+ begin
+ gzos = Zlib::GzipWriter.new os
+ gzos.write metadata
+ ensure
+ gzos.flush
+ gzos.finish
+ end
+ end
+ ensure
+ outputter.close
+ end
+ nil
+ end
+
+ def close
+ @external.close
+ @io.close
+ end
+end
+end # module Package
+
+
+module Package
+ #FIXME: refactor the following 2 methods
+ def self.open(dest, mode = "r", &block)
+ raise "Block needed" unless block_given?
+
+ case mode
+ when "r"
+ TarInput.open(dest, &block)
+ when "w"
+ TarOutput.open(dest, &block)
+ else
+ raise "Unknown Package open mode"
+ end
+ end
+
+ def self.open_from_io(io, mode = "r", &block)
+ raise "Block needed" unless block_given?
+
+ case mode
+ when "r"
+ TarInput.open_from_io(io, &block)
+ when "w"
+ TarOutput.open_from_io(io, &block)
+ else
+ raise "Unknown Package open mode"
+ end
+ end
+
+ def self.pack(src, destname)
+ TarOutput.open(destname) do |outp|
+ dir_class.chdir(src) do
+ outp.metadata = (file_class.read("RPA/metadata") rescue nil)
+ find_class.find('.') do |entry|
+ case
+ when file_class.file?(entry)
+ entry.sub!(%r{\./}, "")
+ next if entry =~ /\ARPA\//
+ stat = File.stat(entry)
+ outp.add_file_simple(entry, stat.mode, stat.size) do |os|
+ file_class.open(entry, "rb") do |f|
+ os.write(f.read(4096)) until f.eof?
+ end
+ end
+ when file_class.dir?(entry)
+ entry.sub!(%r{\./}, "")
+ next if entry == "RPA"
+ outp.mkdir(entry, file_class.stat(entry).mode)
+ else
+ raise "Don't know how to pack this yet!"
+ end
+ end
+ end
+ end
+ end
+
+ class << self
+ def file_class
+ File
+ end
+
+ def dir_class
+ Dir
+ end
+
+ def find_class
+ require 'find'
+ Find
+ end
+ end
+end
+
+end
+
diff -ruN rubygems.orig/test/test_package.rb rubygems/test/test_package.rb
--- rubygems.orig/test/test_package.rb 1970-01-01 01:00:00.000000000 +0100
+++ rubygems/test/test_package.rb 2004-07-13 14:37:09.000000000 +0200
@@ -0,0 +1,592 @@
+
+require 'rubygems'
+Gem::manage_gems
+require 'rubygems/package'
+require 'test/unit'
+require 'stringio'
+
+
+class File
+ # straight from setup.rb
+ def File.dir?(path)
+ # for corrupted windows stat()
+ File.directory?((path[-1,1] == '/') ? path : path + '/')
+ end
+
+ def File.read_b(name)
+ File.open(name, "rb"){|f| f.read}
+ end
+end
+
+module TarTester
+ private
+ def assert_headers_equal(h1, h2)
+ fields = %w[name 100 mode 8 uid 8 gid 8 size 12 mtime 12 checksum 8
+ typeflag 1 linkname 100 magic 6 version 2 uname 32
+ gname 32 devmajor 8 devminor 8 prefix 155]
+ offset = 0
+ until fields.empty?
+ name = fields.shift
+ length = fields.shift.to_i
+ if name == "checksum"
+ chksum_off = offset
+ offset += length
+ next
+ end
+ assert_equal(h1[offset, length], h2[offset, length],
+ "Field #{name} of the tar header differs.")
+ offset += length
+ end
+ assert_equal(h1[chksum_off, 8], h2[chksum_off, 8])
+ end
+
+ def tar_file_header(fname, dname, mode, length)
+ h = header("0", fname, dname, length, mode)
+ checksum = calc_checksum(h)
+ header("0", fname, dname, length, mode, checksum)
+ end
+
+ def tar_dir_header(name, prefix, mode)
+ h = header("5", name, prefix, 0, mode)
+ checksum = calc_checksum(h)
+ header("5", name, prefix, 0, mode, checksum)
+ end
+
+ def header(type, fname, dname, length, mode, checksum = nil)
+# struct tarfile_entry_posix {
+# char name[100]; # ASCII + (Z unless filled)
+# char mode[8]; # 0 padded, octal null
+# char uid[8]; # ditto
+# char gid[8]; # ditto
+# char size[12]; # 0 padded, octal, null
+# char mtime[12]; # 0 padded, octal, null
+# char checksum[8]; # 0 padded, octal, null and space
+# char typeflag[1]; # file: "0" dir: "5"
+# char linkname[100]; # ASCII + (Z unless filled)
+# char magic[6]; # "ustar\0"
+# char version[2]; # "00"
+# char uname[32]; # ASCIIZ
+# char gname[32]; # ASCIIZ
+# char devmajor[8]; # 0 padded, octal, null
+# char devminor[8]; # o padded, octal, null
+# char prefix[155]; # ASCII + (Z unless filled)
+# };
+
+ checksum ||= " " * 8
+ arr = [ASCIIZ(fname, 100), Z(to_oct(mode, 7)), Z(to_oct(0, 7)),
+ Z(to_oct(0,7)), Z(to_oct(length, 11)), Z(to_oct(0,11)),
+ checksum, type, "\0" * 100, "ustar\0", "00", ASCIIZ("wheel", 32),
+ ASCIIZ("wheel", 32), Z(to_oct(0,7)), Z(to_oct(0,7)),
+ ASCIIZ(dname, 155) ]
+ arr = arr.join("").split(//).map{|x| x[0]}
+ h = arr.pack("C100C8C8C8C12C12" + # name, mode, uid, gid, size, mtime
+ "C8CC100C6C2" + # checksum, typeflag, linkname, magic, version
+ "C32C32C8C8C155") # uname, gname, devmajor, devminor, prefix
+ ret = h + "\0" * (512 - h.size)
+ assert_equal(512, ret.size)
+ ret
+ end
+
+ def calc_checksum(header)
+ sum = header.unpack("C*").inject{|s,a| s + a}
+ SP(Z(to_oct(sum, 6)))
+ end
+
+ def to_oct(n, pad_size)
+ "%0#{pad_size}o" % n
+ end
+
+ def ASCIIZ(str, length)
+ str + "\0" * (length - str.length)
+ end
+
+ def SP(s)
+ s + " "
+ end
+
+ def Z(s)
+ s + "\0"
+ end
+
+ def SP_Z(s)
+ s + " \0"
+ end
+end
+
+class TC_TarHeader < Test::Unit::TestCase
+ include Gem::Package
+ include TarTester
+
+ def test_arguments_are_checked
+ e = Gem::Package::ArgumentError
+ assert_raises(e){TarHeader.new :name=>"", :size=>"", :mode=>"" }
+ assert_raises(e){TarHeader.new :name=>"", :size=>"", :prefix=>"" }
+ assert_raises(e){TarHeader.new :name=>"", :prefix=>"", :mode=>"" }
+ assert_raises(e){TarHeader.new :prefix=>"", :size=>"", :mode=>"" }
+ end
+
+ def test_basic_headers
+ assert_headers_equal(tar_file_header("bla", "", 012345, 10),
+ TarHeader.new(:name => "bla", :mode => 012345,
+ :size => 10, :prefix => "").to_s)
+ assert_headers_equal(tar_dir_header("bla", "", 012345),
+ TarHeader.new(:name => "bla", :mode => 012345,
+ :size => 0, :prefix => "",
+ :typeflag => "5" ).to_s)
+ end
+
+ def test_long_name_works
+ assert_headers_equal(tar_file_header("a" * 100, "", 012345, 10),
+ TarHeader.new(:name => "a" * 100, :mode => 012345,
+ :size => 10, :prefix => "").to_s)
+ assert_headers_equal(tar_file_header("a" * 100, "bb" * 60,
+ 012345, 10),
+ TarHeader.new(:name => "a" * 100, :mode => 012345,
+ :size => 10, :prefix => "bb" * 60).to_s)
+ end
+
+ def test_new_from_stream
+ header = tar_file_header("a" * 100, "", 012345, 10)
+ h = nil
+ header = StringIO.new header
+ assert_nothing_raised{ h = TarHeader.new_from_stream header }
+ assert_equal("a" * 100, h.name)
+ assert_equal(012345, h.mode)
+ assert_equal(10, h.size)
+ assert_equal("", h.prefix)
+ assert_equal("ustar", h.magic)
+ end
+end
+
+
+class TC_TarWriter < Test::Unit::TestCase
+ include Gem::Package
+ include TarTester
+ require 'stringio'
+ class DummyIO
+ attr_reader :data
+ def initialize
+ @data = ""
+ end
+ def write(dat)
+ data << dat
+ dat.size
+ end
+ def reset
+ @data = ""
+ end
+ end
+
+ def setup
+ @data = "a" * 10
+ @dummyos = DummyIO.new
+ @os = TarWriter.new(@dummyos)
+ end
+
+ def teardown
+ @os.close
+ end
+
+ def test_add_file_simple
+ @dummyos.reset
+ TarWriter.new(@dummyos) do |os|
+ os.add_file_simple("lib/foo/bar", 0644, 10) {|f| f.write "a" * 10 }
+ os.add_file_simple("lib/bar/baz", 0644, 100) {|f| f.write "fillme"}
+ end
+ assert_headers_equal(tar_file_header("lib/foo/bar", "", 0644, 10),
+ @dummyos.data[0,512])
+ assert_equal("a" * 10 + "\0" * 502, @dummyos.data[512,512])
+ assert_headers_equal(tar_file_header("lib/bar/baz", "", 0644, 100),
+ @dummyos.data[512*2,512])
+ assert_equal("fillme" + "\0" * 506, @dummyos.data[512*3,512])
+ assert_equal("\0" * 512, @dummyos.data[512*4, 512])
+ assert_equal("\0" * 512, @dummyos.data[512*5, 512])
+ end
+
+ def test_write_operations_fail_after_closed
+ @dummyos.reset
+ @os.add_file_simple("sadd", 0644, 20) { |f| }
+ @os.close
+ assert_raises(ClosedIO) { @os.flush }
+ assert_raises(ClosedIO) { @os.add_file("dfdsf", 0644){} }
+ assert_raises(ClosedIO) { @os.mkdir "sdfdsf", 0644 }
+ end
+
+ def test_file_name_is_split_correctly
+ # test insane file lengths, and
+ # a{100}/b{155}, etc
+ @dummyos.reset
+ names = ["a" * 155 + '/' + "b" * 100, "a" * 151 + "/" + ("qwer/" * 19) + "bla" ]
+ o_names = ["b" * 100, "qwer/" * 19 + "bla"]
+ o_prefixes = ["a" * 155, "a" * 151]
+ names.each {|name| @os.add_file_simple(name, 0644, 10) { } }
+ o_names.each_with_index do |nam, i|
+ assert_headers_equal(tar_file_header(nam, o_prefixes[i], 0644, 10),
+ @dummyos.data[2*i*512,512])
+ end
+ assert_raises(TooLongFileName) do
+ @os.add_file_simple(File.join("a" * 152, "b" * 10, "a" * 92), 0644,10) {}
+ end
+ assert_raises(TooLongFileName) do
+ @os.add_file_simple(File.join("a" * 162, "b" * 10), 0644,10) {}
+ end
+ assert_raises(TooLongFileName) do
+ @os.add_file_simple(File.join("a" * 10, "b" * 110), 0644,10) {}
+ end
+ end
+
+ def test_add_file
+ dummyos = StringIO.new
+ class << dummyos
+ def method_missing(meth, *a)
+ self.string.send(meth, *a)
+ end
+ end
+ os = TarWriter.new dummyos
+ content1 = ('a'..'z').to_a.join("") # 26
+ content2 = ('aa'..'zz').to_a.join("") # 1352
+ TarWriter.new(dummyos) do |os|
+ os.add_file("lib/foo/bar", 0644) {|f| f.write "a" * 10 }
+ os.add_file("lib/bar/baz", 0644) {|f| f.write content1 }
+ os.add_file("lib/bar/baz", 0644) {|f| f.write content2 }
+ os.add_file("lib/bar/baz", 0644) {|f| }
+ end
+ assert_headers_equal(tar_file_header("lib/foo/bar", "", 0644, 10),
+ dummyos[0,512])
+ assert_equal("a" * 10 + "\0" * 502, dummyos[512,512])
+ offset = 512 * 2
+ [content1, content2, ""].each do |data|
+ assert_headers_equal(tar_file_header("lib/bar/baz", "", 0644,
+ data.size),
+ dummyos[offset,512])
+ offset += 512
+ until !data || data == ""
+ chunk = data[0,512]
+ data[0,512] = ""
+ assert_equal(chunk + "\0" * (512-chunk.size),
+ dummyos[offset,512])
+ offset += 512
+ end
+ end
+ assert_equal("\0" * 1024, dummyos[offset,1024])
+ end
+
+ def test_add_file_tests_seekability
+ assert_raises(Gem::Package::NonSeekableIO) do
+ @os.add_file("libdfdsfd", 0644) {|f| }
+ end
+ end
+
+ def test_write_header
+ @dummyos.reset
+ @os.add_file_simple("lib/foo/bar", 0644, 0) { |f| }
+ @os.flush
+ assert_headers_equal(tar_file_header("lib/foo/bar", "", 0644, 0),
+ @dummyos.data[0,512])
+ @dummyos.reset
+ @os.mkdir("lib/foo", 0644)
+ assert_headers_equal(tar_dir_header("lib/foo", "", 0644),
+ @dummyos.data[0,512])
+ @os.mkdir("lib/bar", 0644)
+ assert_headers_equal(tar_dir_header("lib/bar", "", 0644),
+ @dummyos.data[512*1,512])
+ end
+
+ def test_write_data
+ @dummyos.reset
+ @os.add_file_simple("lib/foo/bar", 0644, 10) { |f| f.write @data }
+ @os.flush
+ assert_equal(@data + ("\0" * (512- at data.size)),
+ @dummyos.data[512,512])
+ end
+
+ def test_file_size_is_checked
+ @dummyos.reset
+ assert_raises(TarWriter::FileOverflow) do
+ @os.add_file_simple("lib/foo/bar", 0644, 10) {|f| f.write "1" * 100}
+ end
+ assert_nothing_raised do
+ @os.add_file_simple("lib/foo/bar", 0644, 10) {|f| }
+ end
+ end
+end
+
+
+class TC_TarReader < Test::Unit::TestCase
+ include Gem::Package
+ include TarTester
+ require 'stringio'
+
+ def setup
+ end
+
+ def teardown
+ end
+
+ def test_multiple_entries
+ str = tar_file_header("lib/foo", "", 010644, 10) + "\0" * 512
+ str += tar_file_header("bar", "baz", 0644, 0)
+ str += tar_dir_header("foo", "bar", 012345)
+ str += "\0" * 1024
+ names = %w[lib/foo bar foo]
+ prefixes = ["", "baz", "bar"]
+ modes = [010644, 0644, 012345]
+ sizes = [10, 0, 0]
+ isdir = [false, false, true]
+ isfile = [true, true, false]
+ TarReader.new(StringIO.new(str)) do |is|
+ i = 0
+ is.each_entry do |entry|
+ assert_kind_of(TarReader::Entry, entry)
+ assert_equal(names[i], entry.name)
+ assert_equal(prefixes[i], entry.prefix)
+ assert_equal(sizes[i], entry.size)
+ assert_equal(modes[i], entry.mode)
+ assert_equal(isdir[i], entry.is_directory?)
+ assert_equal(isfile[i], entry.is_file?)
+ if prefixes[i] != ""
+ assert_equal(File.join(prefixes[i], names[i]),
+ entry.full_name)
+ else
+ assert_equal(names[i], entry.name)
+ end
+ i += 1
+ end
+ assert_equal(names.size, i)
+ end
+ end
+
+ def test_rewind_entry_works
+ content = ('a'..'z').to_a.join(" ")
+ str = tar_file_header("lib/foo", "", 010644, content.size) + content +
+ "\0" * (512 - content.size)
+ str << "\0" * 1024
+ TarReader.new(StringIO.new(str)) do |is|
+ is.each_entry do |entry|
+ 3.times do
+ entry.rewind
+ assert_equal(content, entry.read)
+ assert_equal(content.size, entry.pos)
+ end
+ end
+ end
+ end
+
+ def test_rewind_works
+ content = ('a'..'z').to_a.join(" ")
+ str = tar_file_header("lib/foo", "", 010644, content.size) + content +
+ "\0" * (512 - content.size)
+ str << "\0" * 1024
+ TarReader.new(StringIO.new(str)) do |is|
+ 3.times do
+ is.rewind
+ i = 0
+ is.each_entry do |entry|
+ assert_equal(content, entry.read)
+ i += 1
+ end
+ assert_equal(1, i)
+ end
+ end
+ end
+
+ def test_read_works
+ contents = ('a'..'z').inject(""){|s,x| s << x * 100}
+ str = tar_file_header("lib/foo", "", 010644, contents.size) + contents
+ str += "\0" * (512 - (str.size % 512))
+ TarReader.new(StringIO.new(str)) do |is|
+ is.each_entry do |entry|
+ assert_kind_of(TarReader::Entry, entry)
+ data = entry.read(3000) # bigger than contents.size
+ assert_equal(contents, data)
+ assert_equal(true, entry.eof?)
+ end
+ end
+ TarReader.new(StringIO.new(str)) do |is|
+ is.each_entry do |entry|
+ assert_kind_of(TarReader::Entry, entry)
+ data = entry.read(100)
+ (entry.size - data.size).times {|i| data << entry.getc.chr }
+ assert_equal(contents, data)
+ assert_equal(nil, entry.read(10))
+ assert_equal(true, entry.eof?)
+ end
+ end
+ TarReader.new(StringIO.new(str)) do |is|
+ is.each_entry do |entry|
+ assert_kind_of(TarReader::Entry, entry)
+ data = entry.read
+ assert_equal(contents, data)
+ assert_equal(nil, entry.read(10))
+ assert_equal(nil, entry.read)
+ assert_equal(nil, entry.getc)
+ assert_equal(true, entry.eof?)
+ end
+ end
+ end
+
+ def test_eof_works
+ str = tar_file_header("bar", "baz", 0644, 0)
+ TarReader.new(StringIO.new(str)) do |is|
+ is.each_entry do |entry|
+ assert_kind_of(TarReader::Entry, entry)
+ data = entry.read
+ assert_equal(nil, data)
+ assert_equal(nil, entry.read(10))
+ assert_equal(nil, entry.read)
+ assert_equal(nil, entry.getc)
+ assert_equal(true, entry.eof?)
+ end
+ end
+ str = tar_dir_header("foo", "bar", 012345)
+ TarReader.new(StringIO.new(str)) do |is|
+ is.each_entry do |entry|
+ assert_kind_of(TarReader::Entry, entry)
+ data = entry.read
+ assert_equal(nil, data)
+ assert_equal(nil, entry.read(10))
+ assert_equal(nil, entry.read)
+ assert_equal(nil, entry.getc)
+ assert_equal(true, entry.eof?)
+ end
+ end
+ str = tar_dir_header("foo", "bar", 012345)
+ str += tar_file_header("bar", "baz", 0644, 0)
+ str += tar_file_header("bar", "baz", 0644, 0)
+ TarReader.new(StringIO.new(str)) do |is|
+ is.each_entry do |entry|
+ assert_kind_of(TarReader::Entry, entry)
+ data = entry.read
+ assert_equal(nil, data)
+ assert_equal(nil, entry.read(10))
+ assert_equal(nil, entry.read)
+ assert_equal(nil, entry.getc)
+ assert_equal(true, entry.eof?)
+ end
+ end
+ end
+
+ class TC_TarInput < Test::Unit::TestCase
+ include Gem::Package
+ include TarTester
+ require 'rbconfig'
+ require 'zlib'
+ def setup
+ FileUtils.mkdir_p "data__"
+ inner_tar = tar_file_header("bla", "", 0612, 10)
+ inner_tar += "0123456789" + "\0" * 502
+ inner_tar += tar_file_header("foo", "", 0636, 5)
+ inner_tar += "01234" + "\0" * 507
+ inner_tar += tar_dir_header("__dir__", "", 0600)
+ inner_tar += "\0" * 1024
+ str = StringIO.new ""
+ begin
+ os = Zlib::GzipWriter.new str
+ os.write inner_tar
+ ensure
+ os.finish
+ end
+ str.rewind
+ File.open("data__/bla.tar", "wb") do |f|
+ f.write tar_file_header("data.tar.gz", "", 0644, str.string.size)
+ f.write str.string
+ f.write "\0" * ((512 - (str.string.size % 512)) % 512 )
+ meta = "abcde".to_yaml
+ f.write tar_file_header("metadata", "", 0644, meta.size)
+ f.write meta + "\0" * (512 - meta.size)
+ f.write "\0" * 1024
+ end
+ @file = "data__/bla.tar"
+ @entry_names = %w{bla foo __dir__}
+ @entry_sizes = [10, 5, 0]
+ #FIXME: are these modes system dependent?
+ @entry_modes = [0100612, 0100636, 040600]
+ @entry_files = %w{data__/bla data__/foo}
+ @entry_contents = %w[0123456789 01234]
+ end
+
+ def teardown
+ FileUtils.rm_rf "data__"
+ end
+
+ def test_each_works
+ TarInput.open(@file) do |is|
+ i = 0
+ is.each_with_index do |entry, i|
+ assert_kind_of(TarReader::Entry, entry)
+ assert_equal(@entry_names[i], entry.name)
+ assert_equal(@entry_sizes[i], entry.size)
+ end
+ assert_equal(2, i)
+ assert_equal("abcde", is.metadata)
+ end
+ end
+
+ def test_extract_entry_works
+ TarInput.open(@file) do |is|
+ assert_equal("abcde", is.metadata)
+ i = 0
+ is.each_with_index do |entry, i|
+ is.extract_entry "data__", entry
+ name = File.join("data__", entry.name)
+ if entry.is_directory?
+ assert File.dir?(name)
+ else
+ assert File.file?(name)
+ assert_equal(@entry_sizes[i], File.stat(name).size)
+ #FIXME: win32? !!
+ end
+ unless ::Config::CONFIG["arch"] =~ /msdos|win32/i
+ assert_equal(@entry_modes[i], File.stat(name).mode)
+ end
+ end
+ assert_equal(2, i)
+ end
+ @entry_files.each_with_index do |x, i|
+ assert(File.file?(x))
+ assert_equal(@entry_contents[i], File.read_b(x))
+ end
+ end
+ end
+
+ class TC_TarOutput < Test::Unit::TestCase
+ include Gem::Package
+ include TarTester
+ require 'zlib'
+
+ def setup
+ FileUtils.mkdir_p "data__"
+ @file = "data__/bla2.tar"
+ end
+
+ def teardown
+ FileUtils.rm_rf "data__"
+ end
+
+ require 'zlib'
+ def test_file_looks_good
+ TarOutput.open(@file) do |os|
+ os.metadata = "bla".to_yaml
+ end
+ f = File.open(@file, "rb")
+ TarReader.new(f) do |is|
+ i = 0
+ is.each do |entry|
+ case i
+ when 0
+ assert_equal("data.tar.gz", entry.name)
+ when 1
+ assert_equal("metadata.gz", entry.name)
+ gzis = Zlib::GzipReader.new entry
+ assert_equal("bla".to_yaml, gzis.read)
+ gzis.close
+ end
+ i += 1
+ end
+ assert_equal(2, i)
+ end
+ ensure
+ f.close
+ end
+ end
+end
--
Running Debian GNU/Linux Sid (unstable)
batsman dot geo at yahoo dot com
MSDOS didn't get as bad as it is overnight -- it took over ten years
of careful development.
-- dmeggins at aix1.uottawa.ca
More information about the Rubygems-developers
mailing list