add test_ill_formed_utf_8_replace to test recommended number of \uFFFD

Add test method test_ill_formed_utf_8_replace to test/ruby/test_transcode.rb
to check for the recommended number of \uFFFD replacement characters.
This is the first part, using ill-formed prefixes, with suffixes up to
the length of the original UTF-8 structure (including overlongs and
the full 31-bit space.)

For more details, see Unicode 9.0.0, Section 3.9, Best Practices for Using U+FFFD.


git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@59026 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

1 files changed, 74 insertions, 0 deletions

diff --git a/test/ruby/test_transcode.rb b/test/ruby/test_transcode.rb
index b65a8c2b98..1561468b54 100644
--- a/test/ruby/test_transcode.rb
+++ b/test/ruby/test_transcode.rb
@@ -998,6 +998,80 @@ class TestTranscode < Test::Unit::TestCase
     check_both_ways("\u00A0", "\xFF", 'CP855') # non-breaking space
   end
 
+  def test_ill_formed_utf_8_replace
+    fffd1 = "\uFFFD".encode 'UTF-16BE'
+    fffd2 = "\uFFFD\uFFFD".encode 'UTF-16BE'
+    fffd3 = "\uFFFD\uFFFD\uFFFD".encode 'UTF-16BE'
+    fffd4 = "\uFFFD\uFFFD\uFFFD\uFFFD".encode 'UTF-16BE'
+    fffd5 = "\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD".encode 'UTF-16BE'
+    fffd6 = "\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD".encode 'UTF-16BE'
+
+    assert_equal fffd2, "\xC0\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd2, "\xC0\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd2, "\xC1\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd2, "\xC1\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd2, "\xE0\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd2, "\xE0\x9F".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd3, "\xE0\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd3, "\xE0\x9F\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd2, "\xED\xA0".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd2, "\xED\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd3, "\xED\xA0\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd3, "\xED\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd2, "\xF0\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd2, "\xF0\x8F".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd3, "\xF0\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd3, "\xF0\x8F\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd4, "\xF0\x80\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd4, "\xF0\x8F\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd2, "\xF4\x90".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd2, "\xF4\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd3, "\xF4\x90\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd3, "\xF4\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd4, "\xF4\x90\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd4, "\xF4\xBF\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd2, "\xF5\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd2, "\xF7\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd3, "\xF5\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd3, "\xF7\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd4, "\xF5\x80\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd4, "\xF7\xBF\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd1, "\xF8".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd1, "\xFB".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd2, "\xF8\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd2, "\xFB\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd3, "\xF8\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd3, "\xFB\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd4, "\xF8\x80\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd4, "\xFB\xBF\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd5, "\xF8\x80\x80\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd5, "\xFB\xBF\xBF\xBF\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd1, "\xFC".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd1, "\xFD".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd2, "\xFC\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd2, "\xFD\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd3, "\xFC\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd3, "\xFD\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd4, "\xFC\x80\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd4, "\xFD\xBF\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd5, "\xFC\x80\x80\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd5, "\xFD\xBF\xBF\xBF\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd6, "\xFC\x80\x80\x80\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd6, "\xFD\xBF\xBF\xBF\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd1, "\xFE".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd1, "\xFF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd2, "\xFE\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd2, "\xFF\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd3, "\xFE\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd3, "\xFF\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd4, "\xFE\x80\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd4, "\xFF\xBF\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd5, "\xFE\x80\x80\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd5, "\xFF\xBF\xBF\xBF\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd6, "\xFE\x80\x80\x80\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd6, "\xFF\xBF\xBF\xBF\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
+  end
+
   def check_utf_16_both_ways(utf8, raw)
     copy = raw.dup
     0.step(copy.length-1, 2) { |i| copy[i+1], copy[i] = copy[i], copy[i+1] }