* lib/csv/csv.rb: Reworked CSV's parser and generator to be m17n. Data

  is now parsed in the Encoding it is in without need for translation.
* lib/csv/csv.rb:  Improved inspect() messages for better IRb support.
* lib/csv/csv.rb:  Fixed header writing bug reported by Dov Murik.
* lib/csv/csv.rb:  Use custom separators in parsing header Strings as
  suggested by Shmulik Regev.
* lib/csv/csv.rb:  Added a :write_headers option for outputting headers.
* lib/csv/csv.rb:  Handle open() calls in binary mode whenever we can to
  workaround a Windows issue where line-ending translation can cause an
  off-by-one error in seeking back to a non-zero starting position after
  auto-discovery for :row_sep as suggested by Robert Battle.
* lib/csv/csv.rb:  Improved the parser to fail faster when fed some forms
  of invalid CSV that can be detected without reading ahead.
* lib/csv/csv.rb:  Added a :field_size_limit option to control CSV's
  lookahead and prevent the parser from biting off more data than
  it can chew.
* lib/csv/csv.rb:  Added readers for CSV attributes:  col_sep(), row_sep(),
  quote_char(), field_size_limit(), converters(), unconverted_fields?(),
  headers(), return_headers?(), write_headers?(), header_converters(),
  skip_blanks?(), and force_quotes?().
* lib/csv/csv.rb:  Cleaned up code syntax to be more inline with
  Ruby 1.9 than 1.8.



git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@19441 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

11 files changed, 527 insertions, 20 deletions

diff --git a/test/csv/tc_csv_parsing.rb b/test/csv/tc_csv_parsing.rb
index 965af929f3..635ae90531 100644
--- a/test/csv/tc_csv_parsing.rb
+++ b/test/csv/tc_csv_parsing.rb
@@ -1,4 +1,5 @@
-#!/usr/local/bin/ruby -w
+#!/usr/bin/env ruby -w
+# encoding: UTF-8
 
 # tc_csv_parsing.rb
 #
@@ -7,6 +8,7 @@
 #  under the terms of Ruby's license.
 
 require "test/unit"
+require "timeout"
 
 require "csv"
 
@@ -17,6 +19,8 @@ require "csv"
 # separator <tt>$/</tt>.
 # 
 class TestCSVParsing < Test::Unit::TestCase
+  BIG_DATA = "123456789\n" * 1024
+  
   def test_mastering_regex_example
     ex = %Q{Ten Thousand,10000, 2710 ,,"10,000","It's ""10 Grand"", baby",10K}
     assert_equal( [ "Ten Thousand", "10000", " 2710 ", nil, "10,000",
@@ -158,7 +162,31 @@ class TestCSVParsing < Test::Unit::TestCase
         assert_send([csv.lineno, :<, 4])
       end
     rescue CSV::MalformedCSVError
-      assert_equal("Unclosed quoted field on line 4.", $!.message)
+      assert_equal("Illegal quoting on line 4.", $!.message)
+    end
+  end
+  
+  def test_the_parse_fails_fast_when_it_can_for_unquoted_fields
+    assert_parse_errors_out('valid,fields,bad start"' + BIG_DATA)
+  end
+  
+  def test_the_parse_fails_fast_when_it_can_for_unescaped_quotes
+    assert_parse_errors_out('valid,fields,"bad start"unescaped' + BIG_DATA)
+  end
+  
+  def test_field_size_limit_controls_lookahead
+    assert_parse_errors_out( 'valid,fields,"' + BIG_DATA + '"',
+                             :field_size_limit => 2048 )
+  end
+  
+  private
+  
+  def assert_parse_errors_out(*args)
+    assert_raise(CSV::MalformedCSVError) do
+      Timeout.timeout(0.2) do
+        CSV.parse(*args)
+        fail("Parse didn't error out")
+      end
     end
   end
 end
diff --git a/test/csv/tc_csv_writing.rb b/test/csv/tc_csv_writing.rb
index 4677284306..a1ce4de97e 100644
--- a/test/csv/tc_csv_writing.rb
+++ b/test/csv/tc_csv_writing.rb
@@ -1,4 +1,5 @@
-#!/usr/local/bin/ruby -w
+#!/usr/bin/env ruby -w
+# encoding: UTF-8
 
 # tc_csv_writing.rb
 #
diff --git a/test/csv/tc_data_converters.rb b/test/csv/tc_data_converters.rb
index 24c6b6b76f..acf27a6669 100644
--- a/test/csv/tc_data_converters.rb
+++ b/test/csv/tc_data_converters.rb
@@ -1,4 +1,5 @@
-#!/usr/local/bin/ruby -w
+#!/usr/bin/env ruby -w
+# encoding: UTF-8
 
 # tc_data_converters.rb
 #
diff --git a/test/csv/tc_encodings.rb b/test/csv/tc_encodings.rb
new file mode 100644
index 0000000000..c773ec9cb8
--- /dev/null
+++ b/test/csv/tc_encodings.rb
@@ -0,0 +1,255 @@
+#!/usr/bin/env ruby -w
+# encoding: UTF-8
+
+# tc_encodings.rb
+#
+#  Created by James Edward Gray II on 2008-09-13.
+#  Copyright 2008 James Edward Gray II. You can redistribute or modify this code
+#  under the terms of Ruby's license.
+
+require "test/unit"
+
+require "csv"
+
+class TestEncodings < Test::Unit::TestCase
+  def setup
+    @temp_csv_path = File.join(File.dirname(__FILE__), "temp.csv")
+  end
+  
+  def teardown
+    File.unlink(@temp_csv_path) if File.exist? @temp_csv_path
+  end
+  
+  ########################################
+  ### Hand Test Some Popular Encodings ###
+  ########################################
+  
+  def test_parses_utf8_encoding
+    assert_parses( [ %w[ one two … ],
+                     %w[ 1   …   3 ],
+                     %w[ …   5   6 ] ], "UTF-8" )
+  end
+  
+  def test_parses_latin1_encoding
+    assert_parses( [ %w[ one    two    Résumé ],
+                     %w[ 1      Résumé 3      ],
+                     %w[ Résumé 5      6      ] ], "ISO-8859-1" )
+  end
+  
+  def test_parses_utf16be_encoding
+    assert_parses( [ %w[ one two … ],
+                     %w[ 1   …   3 ],
+                     %w[ …   5   6 ] ], "UTF-16BE" )
+  end
+  
+  def test_parses_shift_jis_encoding
+    assert_parses( [ %w[ 一 二 三 ],
+                     %w[ 四 五 六 ],
+                     %w[ 七 八 九 ] ], "Shift_JIS" )
+  end
+  
+  ###########################################################
+  ### Try Simple Reading for All Non-dummy Ruby Encodings ###
+  ###########################################################
+  
+  def test_reading_with_most_encodings
+    each_encoding do |encoding|
+      begin
+        assert_parses( [ %w[ abc def ],
+                         %w[ ghi jkl ] ], encoding )
+      rescue Encoding::NoConverterError
+        fail("Failed to support #{encoding.name}.")
+      end
+    end
+  end
+  
+  def test_regular_expression_escaping
+    each_encoding do |encoding|
+      begin
+        assert_parses( [ %w[ abc def ],
+                         %w[ ghi jkl ] ], encoding, :col_sep => "|" )
+      rescue Encoding::NoConverterError
+        fail("Failed to properly escape #{encoding.name}.")
+      end
+    end
+  end
+  
+  #######################################################################
+  ### Stress Test ASCII Compatible and Non-ASCII Compatible Encodings ###
+  #######################################################################
+  
+  def test_auto_line_ending_detection
+    # arrange data to place a \r at the end of CSV's read ahead point
+    encode_for_tests([["a" * 509]], :row_sep => "\r\n") do |data|
+      assert_equal("\r\n".encode(data.encoding), CSV.new(data).row_sep)
+    end
+  end
+  
+  def test_csv_chars_are_transcoded
+    encode_for_tests([%w[abc def]]) do |data|
+      %w[col_sep row_sep quote_char].each do |csv_char|
+        assert_equal( "|".encode(data.encoding),
+                      CSV.new(data, csv_char.to_sym => "|").send(csv_char) )
+      end
+    end
+  end
+  
+  def test_parser_works_with_encoded_headers
+    encode_for_tests([%w[one two three], %w[1 2 3]]) do |data|
+      parsed = CSV.parse(data, :headers => true)
+      assert( parsed.headers.all? { |h| h.encoding == data.encoding },
+              "Wrong data encoding." )
+      parsed.each do |row|
+        assert( row.fields.all? { |f| f.encoding == data.encoding },
+                "Wrong data encoding." )
+      end
+    end
+  end
+  
+  def test_built_in_converters_transcode_to_utf_8_then_convert
+    encode_for_tests([%w[one two three], %w[1 2 3]]) do |data|
+      parsed = CSV.parse(data, :converters => :integer)
+      assert( parsed[0].all? { |f| f.encoding == data.encoding },
+              "Wrong data encoding." )
+      assert_equal([1, 2, 3], parsed[1])
+    end
+  end
+  
+  def test_built_in_header_converters_transcode_to_utf_8_then_convert
+    encode_for_tests([%w[one two three], %w[1 2 3]]) do |data|
+      parsed = CSV.parse( data, :headers           => true,
+                                :header_converters => :downcase )
+      assert( parsed.headers.all? { |h| h.encoding.name == "UTF-8" },
+              "Wrong data encoding." )
+      assert( parsed[0].fields.all? { |f| f.encoding == data.encoding },
+              "Wrong data encoding." )
+    end
+  end
+  
+  def test_open_allows_you_to_set_encodings
+    encode_for_tests([%w[abc def]]) do |data|
+      # read and write in encoding
+      File.open(@temp_csv_path, "wb:#{data.encoding.name}") { |f| f << data }
+      CSV.open(@temp_csv_path, "rb:#{data.encoding.name}") do |csv|
+        csv.each do |row|
+          assert( row.all? { |f| f.encoding == data.encoding },
+                  "Wrong data encoding." )
+        end
+      end
+      
+      # read and write with transcoding
+      File.open(@temp_csv_path, "wb:UTF-32BE:#{data.encoding.name}") do |f|
+        f << data
+      end
+      CSV.open(@temp_csv_path, "rb:UTF-32BE:#{data.encoding.name}") do |csv|
+        csv.each do |row|
+          assert( row.all? { |f| f.encoding == data.encoding },
+                  "Wrong data encoding." )
+        end
+      end
+    end
+  end
+  
+  def test_foreach_allows_you_to_set_encodings
+    encode_for_tests([%w[abc def]]) do |data|
+      # read and write in encoding
+      File.open(@temp_csv_path, "wb:#{data.encoding.name}") { |f| f << data }
+      CSV.foreach(@temp_csv_path, :encoding => data.encoding.name) do |row|
+        assert( row.all? { |f| f.encoding == data.encoding },
+                "Wrong data encoding." )
+      end
+      
+      # read and write with transcoding
+      File.open(@temp_csv_path, "wb:UTF-32BE:#{data.encoding.name}") do |f|
+        f << data
+      end
+      CSV.foreach( @temp_csv_path,
+                   :encoding => "UTF-32BE:#{data.encoding.name}" ) do |row|
+        assert( row.all? { |f| f.encoding == data.encoding },
+                "Wrong data encoding." )
+      end
+    end
+  end
+  
+  def test_read_allows_you_to_set_encodings
+    encode_for_tests([%w[abc def]]) do |data|
+      # read and write in encoding
+      File.open(@temp_csv_path, "wb:#{data.encoding.name}") { |f| f << data }
+      rows = CSV.read(@temp_csv_path, :encoding => data.encoding.name)
+      assert( rows.flatten.all? { |f| f.encoding == data.encoding },
+              "Wrong data encoding." )
+      
+      # read and write with transcoding
+      File.open(@temp_csv_path, "wb:UTF-32BE:#{data.encoding.name}") do |f|
+        f << data
+      end
+      rows = CSV.read( @temp_csv_path,
+                       :encoding => "UTF-32BE:#{data.encoding.name}" )
+      assert( rows.flatten.all? { |f| f.encoding == data.encoding },
+              "Wrong data encoding." )
+    end
+  end
+  
+  #################################
+  ### Write CSV in any Encoding ###
+  #################################
+  
+  def test_can_write_csv_in_any_encoding
+    each_encoding do |encoding|
+      # test generate_line with encoding hint
+      csv = %w[abc d|ef].map { |f| f.encode(encoding) }.
+                         to_csv(:col_sep => "|", :encoding => encoding.name)
+      assert_equal(encoding, csv.encoding)
+      
+      # test generate_line with encoding guessing from fields
+      csv = %w[abc d|ef].map { |f| f.encode(encoding) }.to_csv(:col_sep => "|")
+      assert_equal(encoding, csv.encoding)
+      
+      # writing to files
+      data = encode_ary([%w[abc d,ef], %w[123 456 ]], encoding)
+      CSV.open(@temp_csv_path, "wb:#{encoding.name}") do |csv|
+        data.each { |row| csv << row }
+      end
+      assert_equal(data, CSV.read(@temp_csv_path, :encoding => encoding.name))
+    end
+  end
+  
+  private
+  
+  def assert_parses(fields, encoding, options = { })
+    encoding = Encoding.find(encoding) unless encoding.is_a? Encoding
+    fields   = encode_ary(fields, encoding)
+    parsed   = CSV.parse(ary_to_data(fields, options), options)
+    assert_equal(fields, parsed)
+    assert( parsed.flatten.all? { |field| field.encoding == encoding },
+            "Fields were transcoded." )
+  end
+  
+  def encode_ary(ary, encoding)
+    ary.map { |row| row.map { |field| field.encode(encoding) } }
+  end
+  
+  def ary_to_data(ary, options = { })
+    encoding   = ary.flatten.first.encoding
+    quote_char = (options[:quote_char] || '"').encode(encoding)
+    col_sep    = (options[:col_sep]    || ",").encode(encoding)
+    row_sep    = (options[:row_sep]    || "\n").encode(encoding)
+    ary.map { |row|
+      row.map { |field|
+        [quote_char, field.encode(encoding), quote_char].join
+      }.join(col_sep) + row_sep
+    }.join.encode(encoding)
+  end
+  
+  def encode_for_tests(data, options = { })
+    yield ary_to_data(encode_ary(data, "UTF-8"),    options)
+    yield ary_to_data(encode_ary(data, "UTF-16BE"), options)
+  end
+  
+  def each_encoding
+    Encoding.list.each do |encoding|
+      next if encoding.dummy?  # skip "dummy" encodings
+      yield encoding
+    end
+  end
+end
diff --git a/test/csv/tc_features.rb b/test/csv/tc_features.rb
index ae5a8a451b..ad6732a1ec 100644
--- a/test/csv/tc_features.rb
+++ b/test/csv/tc_features.rb
@@ -1,4 +1,5 @@
-#!/usr/local/bin/ruby -w
+#!/usr/bin/env ruby -w
+# encoding: UTF-8
 
 # tc_features.rb
 #
@@ -67,18 +68,25 @@ class TestCSVFeatures < Test::Unit::TestCase
     end
   end
   
+  def test_csv_char_readers
+    %w[col_sep row_sep quote_char].each do |reader|
+      csv = CSV.new("abc,def", reader.to_sym => "|")
+      assert_equal("|", csv.send(reader))
+    end
+  end
+  
   def test_row_sep_auto_discovery
     ["\r\n", "\n", "\r"].each do |line_end|
       data       = "1,2,3#{line_end}4,5#{line_end}"
-      discovered = CSV.new(data).instance_eval { @row_sep }
+      discovered = CSV.new(data).row_sep
       assert_equal(line_end, discovered)
     end
     
-    assert_equal("\n", CSV.new("\n\r\n\r").instance_eval { @row_sep })
+    assert_equal("\n", CSV.new("\n\r\n\r").row_sep)
     
-    assert_equal($/, CSV.new("").instance_eval { @row_sep })
+    assert_equal($/, CSV.new("").row_sep)
     
-    assert_equal($/, CSV.new(STDERR).instance_eval { @row_sep })
+    assert_equal($/, CSV.new(STDERR).row_sep)
   end
   
   def test_lineno
@@ -117,6 +125,51 @@ class TestCSVFeatures < Test::Unit::TestCase
     assert_equal(3, count)
   end
   
+  def test_csv_behavior_readers
+    %w[ unconverted_fields return_headers write_headers
+        skip_blanks        force_quotes ].each do |behavior|
+      assert( !CSV.new("abc,def").send("#{behavior}?"),
+              "Behavior defaulted to on." )
+      csv = CSV.new("abc,def", behavior.to_sym => true)
+      assert(csv.send("#{behavior}?"), "Behavior change now registered.")
+    end
+  end
+  
+  def test_converters_reader
+    # no change
+    assert_equal( [:integer],
+                  CSV.new("abc,def", :converters => [:integer]).converters )
+    
+    # just one
+    assert_equal( [:integer],
+                  CSV.new("abc,def", :converters => :integer).converters )
+    
+    # expanded
+    assert_equal( [:integer, :float],
+                  CSV.new("abc,def", :converters => :numeric).converters )
+    
+    # custom
+    csv = CSV.new("abc,def", :converters => [:integer, lambda {  }])
+    assert_equal(2, csv.converters.size)
+    assert_equal(:integer, csv.converters.first)
+    assert_instance_of(Proc, csv.converters.last)
+  end
+  
+  def test_header_converters_reader
+    # no change
+    hc = :header_converters
+    assert_equal([:downcase], CSV.new("abc,def", hc => [:downcase]).send(hc))
+    
+    # just one
+    assert_equal([:downcase], CSV.new("abc,def", hc => :downcase).send(hc))
+    
+    # custom
+    csv = CSV.new("abc,def", hc => [:symbol, lambda {  }])
+    assert_equal(2, csv.send(hc).size)
+    assert_equal(:symbol, csv.send(hc).first)
+    assert_instance_of(Proc, csv.send(hc).last)
+  end
+  
   # reported by Kev Jackson
   def test_failing_to_escape_col_sep_bug_fix
     assert_nothing_raised(Exception) { CSV.new(String.new, :col_sep => "|") }
@@ -149,7 +202,7 @@ class TestCSVFeatures < Test::Unit::TestCase
                  )
                )
     end
-    assert_equal("\r\n", zipped.instance_eval { @row_sep })
+    assert_equal("\r\n", zipped.row_sep)
   end
   
   def test_gzip_writer_bug_fix
@@ -168,6 +221,41 @@ class TestCSVFeatures < Test::Unit::TestCase
     File.unlink(file)
   end
   
+  def test_inspect_is_smart_about_io_types
+    str = CSV.new("string,data").inspect
+    assert(str.include?("io_type:StringIO"), "IO type not detected.")
+
+    str = CSV.new($stderr).inspect
+    assert(str.include?("io_type:$stderr"), "IO type not detected.")
+
+    path = File.join(File.dirname(__FILE__), "temp.csv")
+    File.open(path, "w") { |csv| csv << "one,two,three\n1,2,3\n" }
+    str  = CSV.open(path) { |csv| csv.inspect }
+    assert(str.include?("io_type:File"), "IO type not detected.")
+    File.unlink(path)
+  end
+  
+  def test_inspect_shows_key_attributes
+    str = @csv.inspect
+    %w[lineno col_sep row_sep quote_char].each do |attr_name|
+      assert_match(/\b#{attr_name}:[^\s>]+/, str)
+    end
+  end
+  
+  def test_inspect_shows_headers_when_available
+    CSV.new("one,two,three\n1,2,3\n", :headers => true) do |csv|
+      assert(csv.inspect.include?("headers:true"), "Header hint not shown.")
+      csv.shift  # load headers
+      assert_match(/headers:\[[^\]]+\]/, csv.inspect)
+    end
+  end
+  
+  def test_inspect_is_ascii_8bit_encoded
+    CSV.new("one,two,three\n1,2,3\n".encode("UTF-16BE")) do |csv|
+      assert_equal("ASCII-8BIT", csv.inspect.encoding.name)
+    end
+  end
+  
   def test_version
     assert_not_nil(CSV::VERSION)
     assert_instance_of(String, CSV::VERSION)
diff --git a/test/csv/tc_headers.rb b/test/csv/tc_headers.rb
index 74e2f54ad4..e0f544dadf 100644
--- a/test/csv/tc_headers.rb
+++ b/test/csv/tc_headers.rb
@@ -1,4 +1,5 @@
-#!/usr/local/bin/ruby -w
+#!/usr/bin/env ruby -w
+# encoding: UTF-8
 
 # tc_headers.rb
 #
@@ -129,6 +130,21 @@ class TestCSVHeaders < Test::Unit::TestCase
     assert(!row.field_row?)
   end
   
+  def test_csv_header_string_inherits_separators
+    # parse with custom col_sep
+    csv = nil
+    assert_nothing_raised(Exception) do 
+      csv = CSV.parse( @data.tr(",", "|"), :col_sep => "|",
+                                           :headers => "my|new|headers" )
+    end
+
+    # verify headers were recognized
+    row = csv[0]
+    assert_not_nil(row)
+    assert_instance_of(CSV::Row, row)
+    assert_equal([%w{my first}, %w{new second}, %w{headers third}], row.to_a)
+  end
+  
   def test_return_headers
     # activate headers and request they are returned
     csv = nil
@@ -250,6 +266,17 @@ class TestCSVHeaders < Test::Unit::TestCase
     end
   end
   
+  def test_headers_reader
+    # no headers
+    assert_nil(CSV.new(@data).headers)
+    
+    # headers
+    csv = CSV.new(@data, :headers => true)
+    assert_equal(true, csv.headers)                    # before headers are read
+    csv.shift                                          # set headers
+    assert_equal(%w[first second third], csv.headers)  # after headers are read
+  end
+  
   def test_blank_row_bug_fix
     @data += "\n#{@data}"  # add a blank row
     
diff --git a/test/csv/tc_interface.rb b/test/csv/tc_interface.rb
index e8cc920f9d..9cacc28b0a 100644
--- a/test/csv/tc_interface.rb
+++ b/test/csv/tc_interface.rb
@@ -1,4 +1,5 @@
-#!/usr/local/bin/ruby -w
+#!/usr/bin/env ruby -w
+# encoding: UTF-8
 
 # tc_interface.rb
 #
@@ -42,8 +43,9 @@ class TestCSVInterface < Test::Unit::TestCase
     csv.close
     assert(csv.closed?)
     
-    ret = CSV.open(@path) do |csv|
-      assert_instance_of(CSV, csv)
+    ret = CSV.open(@path) do |new_csv|
+      csv = new_csv
+      assert_instance_of(CSV, new_csv)
       "Return value."
     end
     assert(csv.closed?)
@@ -161,7 +163,6 @@ class TestCSVInterface < Test::Unit::TestCase
 
     lines = [{:a => 1, :b => 2, :c => 3}, {:a => 4, :b => 5, :c => 6}]
     CSV.open( @path, "w", :headers           => true,
-                          :converters        => :all,
                           :header_converters => :symbol ) do |csv|
       csv << lines.first.keys
       lines.each { |line| csv << line }
@@ -173,6 +174,74 @@ class TestCSVInterface < Test::Unit::TestCase
     end
   end
   
+  def test_write_hash_with_headers_array
+    File.unlink(@path)
+
+    lines = [{:a => 1, :b => 2, :c => 3}, {:a => 4, :b => 5, :c => 6}]
+    CSV.open(@path, "w", :headers => [:b, :a, :c]) do |csv|
+      lines.each { |line| csv << line }
+    end
+
+    # test writing fields in the correct order
+    File.open(@path, "r") do |f|
+      assert_equal("2,1,3", f.gets.strip)
+      assert_equal("5,4,6", f.gets.strip)
+    end
+
+    # test reading CSV with headers
+    CSV.open( @path, "r", :headers    => [:b, :a, :c],
+                          :converters => :all ) do |csv|
+      csv.each { |line| assert_equal(lines.shift, line.to_hash) }
+    end
+  end
+
+  def test_write_hash_with_headers_string
+    File.unlink(@path)
+
+    lines = [{"a" => 1, "b" => 2, "c" => 3}, {"a" => 4, "b" => 5, "c" => 6}]
+    CSV.open(@path, "w", :headers => "b|a|c", :col_sep => "|") do |csv|
+      lines.each { |line| csv << line }
+    end
+
+    # test writing fields in the correct order
+    File.open(@path, "r") do |f|
+      assert_equal("2|1|3", f.gets.strip)
+      assert_equal("5|4|6", f.gets.strip)
+    end
+
+    # test reading CSV with headers
+    CSV.open( @path, "r", :headers    => "b|a|c",
+                          :col_sep    => "|",
+                          :converters => :all ) do |csv|
+      csv.each { |line| assert_equal(lines.shift, line.to_hash) }
+    end
+  end
+  
+  def test_write_headers
+    File.unlink(@path)
+
+    lines = [{"a" => 1, "b" => 2, "c" => 3}, {"a" => 4, "b" => 5, "c" => 6}]
+    CSV.open( @path, "w", :headers       => "b|a|c",
+                          :write_headers => true,
+                          :col_sep       => "|" ) do |csv|
+      lines.each { |line| csv << line }
+    end
+
+    # test writing fields in the correct order
+    File.open(@path, "r") do |f|
+      assert_equal("b|a|c", f.gets.strip)
+      assert_equal("2|1|3", f.gets.strip)
+      assert_equal("5|4|6", f.gets.strip)
+    end
+
+    # test reading CSV with headers
+    CSV.open( @path, "r", :headers    => true,
+                          :col_sep    => "|",
+                          :converters => :all ) do |csv|
+      csv.each { |line| assert_equal(lines.shift, line.to_hash) }
+    end
+  end
+  
   def test_append  # aliased add_row() and puts()
     File.unlink(@path)
     
@@ -230,6 +299,6 @@ class TestCSVInterface < Test::Unit::TestCase
     
     # shortcuts
     assert_equal(STDOUT, CSV.instance.instance_eval { @io })
-    assert_equal(STDOUT, CSV { |csv| csv.instance_eval { @io } })
+    assert_equal(STDOUT, CSV { |new_csv| new_csv.instance_eval { @io } })
   end
 end
diff --git a/test/csv/tc_row.rb b/test/csv/tc_row.rb
index a9b7f042b2..3fa3784bba 100644
--- a/test/csv/tc_row.rb
+++ b/test/csv/tc_row.rb
@@ -1,4 +1,5 @@
-#!/usr/local/bin/ruby -w
+#!/usr/bin/env ruby -w
+# encoding: UTF-8
 
 # tc_row.rb
 #
@@ -286,4 +287,24 @@ class TestCSVRow < Test::Unit::TestCase
     
     assert_equal([@row.headers.size, @row.fields.size].max, @row.size)
   end
+  
+  def test_inspect_shows_header_field_pairs
+    str = @row.inspect
+    @row.each do |header, field|
+      assert( str.include?("#{header.inspect}:#{field.inspect}"),
+              "Header field pair not found." )
+    end
+  end
+  
+  def test_inspect_is_ascii_8bit_encoded
+    assert_equal("ASCII-8BIT", @row.inspect.encoding.name)
+  end
+  
+  def test_inspect_shows_symbol_headers_as_bare_attributes
+    str = CSV::Row.new(@row.headers.map { |h| h.to_sym }, @row.fields).inspect
+    @row.each do |header, field|
+      assert( str.include?("#{header}:#{field.inspect}"),
+              "Header field pair not found." )
+    end
+  end
 end
diff --git a/test/csv/tc_serialization.rb b/test/csv/tc_serialization.rb
index d9c37fde21..c8273bdb3c 100644
--- a/test/csv/tc_serialization.rb
+++ b/test/csv/tc_serialization.rb
@@ -1,4 +1,5 @@
-#!/usr/local/bin/ruby -w
+#!/usr/bin/env ruby -w
+# encoding: UTF-8
 
 # tc_serialization.rb
 #
diff --git a/test/csv/tc_table.rb b/test/csv/tc_table.rb
index 028274d97f..1e572d979c 100644
--- a/test/csv/tc_table.rb
+++ b/test/csv/tc_table.rb
@@ -1,4 +1,5 @@
-#!/usr/local/bin/ruby -w
+#!/usr/bin/env ruby -w
+# encoding: UTF-8
 
 # tc_table.rb
 #
@@ -389,4 +390,17 @@ class TestCSVTable < Test::Unit::TestCase
     
     assert_equal(@rows.size, @table.size)
   end
+  
+  def test_inspect_shows_current_mode
+    str = @table.inspect
+    assert(str.include?("mode:#{@table.mode}"), "Mode not shown.")
+    
+    @table.by_col!
+    str = @table.inspect
+    assert(str.include?("mode:#{@table.mode}"), "Mode not shown.")
+  end
+  
+  def test_inspect_is_us_ascii_encoded
+    assert_equal("US-ASCII", @table.inspect.encoding.name)
+  end
 end
diff --git a/test/csv/ts_all.rb b/test/csv/ts_all.rb
index c930523757..d380ab5318 100644
--- a/test/csv/ts_all.rb
+++ b/test/csv/ts_all.rb
@@ -1,4 +1,5 @@
-#!/usr/local/bin/ruby -w
+#!/usr/bin/env ruby -w
+# encoding: UTF-8
 
 # ts_all.rb
 #
@@ -17,3 +18,4 @@ require "tc_row"
 require "tc_table"
 require "tc_headers"
 require "tc_serialization"
+require "tc_encodings"