diff options
| -rw-r--r-- | lib/csv/parser.rb | 85 | ||||
| -rw-r--r-- | test/csv/parse/test_inputs_scanner.rb | 37 |
2 files changed, 34 insertions, 88 deletions
diff --git a/lib/csv/parser.rb b/lib/csv/parser.rb index 78ff363d69..7e943acf21 100644 --- a/lib/csv/parser.rb +++ b/lib/csv/parser.rb @@ -167,7 +167,6 @@ class CSV end def keep_start - adjust_last_keep @keeps.push([@scanner.pos, nil]) end @@ -198,17 +197,7 @@ class CSV end def keep_drop - _, buffer = @keeps.pop - return unless buffer - - last_keep = @keeps.last - return unless last_keep - - if last_keep[1] - last_keep[1] << buffer - else - last_keep[1] = buffer - end + @keeps.pop end def rest @@ -216,30 +205,24 @@ class CSV end private - def adjust_last_keep - keep = @keeps.last - return if keep.nil? - - keep_start = keep[0] - return if @scanner.pos == keep_start - - string = @scanner.string - keep_data = string.byteslice(keep_start, @scanner.pos - keep_start) - if keep_data - keep_buffer = keep[1] - if keep_buffer - keep_buffer << keep_data - else - keep[1] = keep_data.dup - end - end - keep[0] = 0 - end - def read_chunk return false if @last_scanner - adjust_last_keep + unless @keeps.empty? + keep = @keeps.last + keep_start = keep[0] + string = @scanner.string + keep_data = string.byteslice(keep_start, @scanner.pos - keep_start) + if keep_data + keep_buffer = keep[1] + if keep_buffer + keep_buffer << keep_data + else + keep[1] = keep_data.dup + end + end + keep[0] = 0 + end input = @inputs.first case input @@ -746,26 +729,28 @@ class CSV sample[0, 128].index(@quote_character) end - class UnoptimizedStringIO # :nodoc: - def initialize(string) - @io = StringIO.new(string, "rb:#{string.encoding}") - end + SCANNER_TEST = (ENV["CSV_PARSER_SCANNER_TEST"] == "yes") + if SCANNER_TEST + class UnoptimizedStringIO + def initialize(string) + @io = StringIO.new(string, "rb:#{string.encoding}") + end - def gets(*args) - @io.gets(*args) - end + def gets(*args) + @io.gets(*args) + end - def each_line(*args, &block) - @io.each_line(*args, &block) - end + def each_line(*args, &block) + @io.each_line(*args, &block) + end - def eof? - @io.eof? + def eof? + @io.eof? + end end - end - SCANNER_TEST = (ENV["CSV_PARSER_SCANNER_TEST"] == "yes") - if SCANNER_TEST + SCANNER_TEST_CHUNK_SIZE = + Integer((ENV["CSV_PARSER_SCANNER_TEST_CHUNK_SIZE"] || "1"), 10) def build_scanner inputs = @samples.collect do |sample| UnoptimizedStringIO.new(sample) @@ -775,12 +760,10 @@ class CSV else inputs << @input end - chunk_size = - Integer((ENV["CSV_PARSER_SCANNER_TEST_CHUNK_SIZE"] || "1"), 10) InputsScanner.new(inputs, @encoding, @row_separator, - chunk_size: chunk_size) + chunk_size: SCANNER_TEST_CHUNK_SIZE) end else def build_scanner diff --git a/test/csv/parse/test_inputs_scanner.rb b/test/csv/parse/test_inputs_scanner.rb deleted file mode 100644 index dd0a64cc45..0000000000 --- a/test/csv/parse/test_inputs_scanner.rb +++ /dev/null @@ -1,37 +0,0 @@ -require_relative "../helper" - -class TestCSVParseInputsScanner < Test::Unit::TestCase - include Helper - - def test_keep_over_chunks_nested_back - input = CSV::Parser::UnoptimizedStringIO.new("abcdefghijklmnl") - scanner = CSV::Parser::InputsScanner.new([input], - Encoding::UTF_8, - nil, - chunk_size: 2) - scanner.keep_start - assert_equal("abc", scanner.scan_all(/[a-c]+/)) - scanner.keep_start - assert_equal("def", scanner.scan_all(/[d-f]+/)) - scanner.keep_back - scanner.keep_back - assert_equal("abcdefg", scanner.scan_all(/[a-g]+/)) - end - - - def test_keep_over_chunks_nested_drop_back - input = CSV::Parser::UnoptimizedStringIO.new("abcdefghijklmnl") - scanner = CSV::Parser::InputsScanner.new([input], - Encoding::UTF_8, - nil, - chunk_size: 3) - scanner.keep_start - assert_equal("ab", scanner.scan(/../)) - scanner.keep_start - assert_equal("c", scanner.scan(/./)) - assert_equal("d", scanner.scan(/./)) - scanner.keep_drop - scanner.keep_back - assert_equal("abcdefg", scanner.scan_all(/[a-g]+/)) - end -end |