diff options
| -rw-r--r-- | ChangeLog | 6 | ||||
| -rw-r--r-- | enc/trans/newline.trans | 26 | ||||
| -rw-r--r-- | test/ruby/test_econv.rb | 18 | ||||
| -rw-r--r-- | test/ruby/test_io_m17n.rb | 2 |
4 files changed, 37 insertions, 15 deletions
@@ -1,3 +1,9 @@ +Mon Nov 23 04:12:00 2009 Tanaka Akira <akr@fsij.org> + + * enc/trans/newline.trans (fun_so_universal_newline): generate \n + after \r\n detection instead of just after \r. + [ruby-list:45988] [ruby-core:25881] [ruby-core:26788] + Sat Nov 21 18:48:35 2009 Tanaka Akira <akr@fsij.org> * vm_eval.c (rb_search_method_entry): show flags and klass value in diff --git a/enc/trans/newline.trans b/enc/trans/newline.trans index d62034aa49..a200ec00a7 100644 --- a/enc/trans/newline.trans +++ b/enc/trans/newline.trans @@ -48,26 +48,26 @@ fun_so_universal_newline(void *statep, const unsigned char *s, size_t l, unsigne if (s[0] == '\n') { if (STATE == NORMAL) { NEWLINES_MET |= MET_LF; - o[0] = '\n'; - len = 1; } else { /* JUST_AFTER_CR */ NEWLINES_MET |= MET_CRLF; - len = 0; } + o[0] = '\n'; + len = 1; STATE = NORMAL; } else { - if (STATE == JUST_AFTER_CR) - NEWLINES_MET |= MET_CR; - if (s[0] == '\r') { + len = 0; + if (STATE == JUST_AFTER_CR) { o[0] = '\n'; len = 1; + NEWLINES_MET |= MET_CR; + } + if (s[0] == '\r') { STATE = JUST_AFTER_CR; } else { - o[0] = s[0]; - len = 1; + o[len++] = s[0]; STATE = NORMAL; } } @@ -79,10 +79,14 @@ static ssize_t universal_newline_finish(void *statep, unsigned char *o, size_t osize) { unsigned char *sp = statep; - if (STATE == JUST_AFTER_CR) + int len = 0; + if (STATE == JUST_AFTER_CR) { + o[0] = '\n'; + len = 1; NEWLINES_MET |= MET_CR; + } STATE = NORMAL; - return 0; + return len; } static const rb_transcoder @@ -91,7 +95,7 @@ rb_universal_newline = { TRANSCODE_TABLE_INFO, 1, /* input_unit_length */ 1, /* max_input */ - 1, /* max_output */ + 2, /* max_output */ asciicompat_converter, /* asciicompat_type */ 2, universal_newline_init, universal_newline_init, /* state_size, state_init, state_fini */ NULL, NULL, NULL, fun_so_universal_newline, diff --git a/test/ruby/test_econv.rb b/test/ruby/test_econv.rb index 5fce12da84..11f5bad2ad 100644 --- a/test/ruby/test_econv.rb +++ b/test/ruby/test_econv.rb @@ -399,7 +399,7 @@ class TestEncodingConverter < Test::Unit::TestCase src << "abc\r\ndef"; check_ec("abc\ndef", "", :source_buffer_empty, *a) src << "ghi\njkl"; check_ec("abc\ndefghi\njkl", "", :source_buffer_empty, *a) src << "mno\rpqr"; check_ec("abc\ndefghi\njklmno\npqr", "", :source_buffer_empty, *a) - src << "stu\r"; check_ec("abc\ndefghi\njklmno\npqrstu\n", "", :source_buffer_empty, *a) + src << "stu\r"; check_ec("abc\ndefghi\njklmno\npqrstu", "", :source_buffer_empty, *a) src << "\nvwx"; check_ec("abc\ndefghi\njklmno\npqrstu\nvwx", "", :source_buffer_empty, *a) src << "\nyz"; check_ec("abc\ndefghi\njklmno\npqrstu\nvwx\nyz", "", :source_buffer_empty, *a) end @@ -410,11 +410,25 @@ class TestEncodingConverter < Test::Unit::TestCase src << "abc\r\ndef"; check_ec("abc\ndef", "", :source_buffer_empty, *a) src << "ghi\njkl"; check_ec("abc\ndefghi\njkl", "", :source_buffer_empty, *a) src << "mno\rpqr"; check_ec("abc\ndefghi\njklmno\npqr", "", :source_buffer_empty, *a) - src << "stu\r"; check_ec("abc\ndefghi\njklmno\npqrstu\n", "", :source_buffer_empty, *a) + src << "stu\r"; check_ec("abc\ndefghi\njklmno\npqrstu", "", :source_buffer_empty, *a) src << "\nvwx"; check_ec("abc\ndefghi\njklmno\npqrstu\nvwx", "", :source_buffer_empty, *a) src << "\nyz"; check_ec("abc\ndefghi\njklmno\npqrstu\nvwx\nyz", "", :source_buffer_empty, *a) end + def test_universal_newline3 + ec = Encoding::Converter.new("", "", universal_newline: true) + a = ["", src="", ec, nil, 50, :partial_input=>true] + src << "abc\r\ndef"; check_ec("abc\ndef", "", :source_buffer_empty, *a) + src << "ghi\njkl"; check_ec("abc\ndefghi\njkl", "", :source_buffer_empty, *a) + src << "mno\rpqr"; check_ec("abc\ndefghi\njklmno\npqr", "", :source_buffer_empty, *a) + src << "stu\r"; check_ec("abc\ndefghi\njklmno\npqrstu", "", :source_buffer_empty, *a) + src << "\nvwx"; check_ec("abc\ndefghi\njklmno\npqrstu\nvwx", "", :source_buffer_empty, *a) + src << "\nyz"; check_ec("abc\ndefghi\njklmno\npqrstu\nvwx\nyz", "", :source_buffer_empty, *a) + src << "\r"; check_ec("abc\ndefghi\njklmno\npqrstu\nvwx\nyz", "", :source_buffer_empty, *a) + a[-1] = nil + src << ""; check_ec("abc\ndefghi\njklmno\npqrstu\nvwx\nyz\n", "", :finished, *a) + end + def test_crlf_newline ec = Encoding::Converter.new("UTF-8", "EUC-JP", crlf_newline: true) assert_econv("abc\r\ndef", :finished, 50, ec, "abc\ndef", "") diff --git a/test/ruby/test_io_m17n.rb b/test/ruby/test_io_m17n.rb index 4d1a62381b..deacd0b36d 100644 --- a/test/ruby/test_io_m17n.rb +++ b/test/ruby/test_io_m17n.rb @@ -1511,7 +1511,6 @@ EOT assert_equal("a", f.getc) assert_equal("\n", f.getc) f.binmode - assert_equal("\n", f.getc) assert_equal("b", f.getc) assert_equal("\r", f.getc) assert_equal("\n", f.getc) @@ -1531,7 +1530,6 @@ EOT assert_equal("a", f.getc) assert_equal("\n", f.getc) f.binmode - assert_equal("\n", f.getc) assert_equal("b", f.getc) assert_equal("\r", f.getc) assert_equal("\n", f.getc) |