From aec1ac6e5127c19d8db89f464e42cca439333974 Mon Sep 17 00:00:00 2001 From: duerst Date: Tue, 26 Jul 2016 06:54:18 +0000 Subject: * enc/windows_1251.c, test/ruby/enc/test_case_comprehensive.rb: Implement non-ASCII case conversion for Windows-1251, by Shunsuke Sato. git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@55750 b2dd03c8-39d4-4d8f-98ff-823fe69b080e --- ChangeLog | 7 ++- enc/windows_1253.c | 74 +++++++++++++++++++++++++++++--- test/ruby/enc/test_case_comprehensive.rb | 2 +- 3 files changed, 74 insertions(+), 9 deletions(-) diff --git a/ChangeLog b/ChangeLog index 1a50c3da41..039f930687 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,8 +1,13 @@ -Tue Jul 26 15:30:37 2016 Martin Duerst +Tue Jul 26 15:54:17 2016 Martin Duerst * enc/windows_1251.c, test/ruby/enc/test_case_comprehensive.rb: Implement non-ASCII case conversion for Windows-1251, by Shunsuke Sato. +Tue Jul 26 15:30:37 2016 Martin Duerst + + * enc/windows_1253.c, test/ruby/enc/test_case_comprehensive.rb: + Implement non-ASCII case conversion for Windows-1253, by Takumi Koyama. + Tue Jul 26 13:04:59 2016 Martin Duerst * test/ruby/enc/test_case_comprehensive.rb: Add explicit skip test for diff --git a/enc/windows_1253.c b/enc/windows_1253.c index cd2987f588..9be59430b4 100644 --- a/enc/windows_1253.c +++ b/enc/windows_1253.c @@ -62,9 +62,9 @@ static const UChar EncCP1253_ToLowerCaseTable[256] = { '\210', '\211', '\212', '\213', '\214', '\215', '\216', '\217', '\220', '\221', '\222', '\223', '\224', '\225', '\226', '\227', '\230', '\231', '\232', '\233', '\234', '\235', '\236', '\237', - '\240', '\241', '\242', '\243', '\244', '\245', '\246', '\247', + '\240', '\241', '\334', '\243', '\244', '\245', '\246', '\247', '\250', '\251', '\252', '\253', '\254', '\255', '\256', '\257', - '\260', '\261', '\262', '\263', '\264', '\265', '\334', '\267', + '\260', '\261', '\262', '\263', '\264', '\354', '\334', '\267', '\335', '\336', '\337', '\273', '\374', '\275', '\375', '\376', '\300', '\341', '\342', '\343', '\344', '\345', '\346', '\347', '\350', '\351', '\352', '\353', '\354', '\355', '\356', '\357', @@ -97,9 +97,9 @@ static const unsigned short EncCP1253_CtypeTable[256] = { 0x0008, 0x0008, 0x0008, 0x0008, 0x0008, 0x0008, 0x0008, 0x0008, 0x0008, 0x0008, 0x0008, 0x0008, 0x0008, 0x0008, 0x0008, 0x0008, 0x0008, 0x0008, 0x0008, 0x0008, 0x0008, 0x0008, 0x0008, 0x0008, - 0x0284, 0x01a0, 0x01a0, 0x00a0, 0x0000, 0x0000, 0x00a0, 0x00a0, + 0x0284, 0x01a0, 0x34a2, 0x00a0, 0x0000, 0x0000, 0x00a0, 0x00a0, 0x00a0, 0x00a0, 0x0000, 0x01a0, 0x00a0, 0x01a0, 0x0000, 0x01a0, - 0x00a0, 0x00a0, 0x10a0, 0x10a0, 0x00a0, 0x00a0, 0x34a2, 0x01a0, + 0x00a0, 0x00a0, 0x10a0, 0x10a0, 0x00a0, 0x30e2, 0x34a2, 0x01a0, 0x34a2, 0x34a2, 0x34a2, 0x01a0, 0x34a2, 0x10a0, 0x34a2, 0x34a2, 0x30e2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, @@ -198,8 +198,8 @@ apply_all_case_fold(OnigCaseFoldType flag, OnigEncoding enc ARG_UNUSED) { return onigenc_apply_all_case_fold_with_map( - numberof(CaseFoldMap), CaseFoldMap, 0, - flag, f, arg); + numberof(CaseFoldMap), CaseFoldMap, 0, + flag, f, arg); } static int @@ -213,6 +213,66 @@ get_case_fold_codes_by_str(OnigCaseFoldType flag, flag, p, end, items); } +static int +case_map (OnigCaseFoldType* flagP, const OnigUChar** pp, + const OnigUChar* end, OnigUChar* to, OnigUChar* to_end, + const struct OnigEncodingTypeST* enc) +{ + OnigCodePoint code; + OnigUChar *to_start = to; + OnigCaseFoldType flags = *flagP; + + while (*pp=0xDD && code<=0xDF) + code -= 0x25; + else if (code==0xFC) + code = 0xBC; + else if (code==0xFD || code==0xFE) + code -= 0x3F; + else + code -= 0x20; + } + *to++ = code; + if (flags&ONIGENC_CASE_TITLECASE) /* switch from titlecase to lowercase for capitalize */ + flags ^= (ONIGENC_CASE_UPCASE|ONIGENC_CASE_DOWNCASE|ONIGENC_CASE_TITLECASE); + } + *flagP = flags; + return (int)(to-to_start); +} OnigEncodingDefine(windows_1253, Windows_1253) = { onigenc_single_byte_mbc_enc_len, @@ -233,6 +293,6 @@ OnigEncodingDefine(windows_1253, Windows_1253) = { onigenc_always_true_is_allowed_reverse_match, 0, ONIGENC_FLAG_NONE, - onigenc_single_byte_ascii_only_case_map, + case_map, }; ENC_ALIAS("CP1253", "Windows-1253") diff --git a/test/ruby/enc/test_case_comprehensive.rb b/test/ruby/enc/test_case_comprehensive.rb index dabe727423..1cbb7eb02f 100644 --- a/test/ruby/enc/test_case_comprehensive.rb +++ b/test/ruby/enc/test_case_comprehensive.rb @@ -289,7 +289,7 @@ TestComprehensiveCaseFold.data_files_available? and class TestComprehensiveCase generate_ascii_only_case_mapping_tests 'Windows-1250' generate_case_mapping_tests 'Windows-1251' generate_case_mapping_tests 'Windows-1252' - generate_ascii_only_case_mapping_tests 'Windows-1253' + generate_case_mapping_tests 'Windows-1253' generate_ascii_only_case_mapping_tests 'Windows-1254' generate_case_mapping_tests 'Windows-1255' generate_ascii_only_case_mapping_tests 'Windows-1256' -- cgit v1.2.3