diff options
Diffstat (limited to 'lib/unicode_normalize')
| -rw-r--r-- | lib/unicode_normalize/normalize.rb | 30 | ||||
| -rw-r--r-- | lib/unicode_normalize/tables.rb | 186 |
2 files changed, 204 insertions, 12 deletions
diff --git a/lib/unicode_normalize/normalize.rb b/lib/unicode_normalize/normalize.rb index 1caf2cc8c8..0447df8de7 100644 --- a/lib/unicode_normalize/normalize.rb +++ b/lib/unicode_normalize/normalize.rb @@ -82,16 +82,22 @@ module UnicodeNormalize # :nodoc: ## Canonical Ordering def self.canonical_ordering_one(string) - sorting = string.each_char.collect { |c| [c, CLASS_TABLE[c]] } - (sorting.length-2).downto(0) do |i| # almost, but not exactly bubble sort - (0..i).each do |j| - later_class = sorting[j+1].last - if 0<later_class and later_class<sorting[j].last - sorting[j], sorting[j+1] = sorting[j+1], sorting[j] - end + result = '' + unordered = [] + chars = string.chars + n = chars.size + chars.each_with_index do |char, i| + ccc = CLASS_TABLE[char] + if ccc == 0 + unordered.sort!.each { result << chars[it % n] } + unordered.clear + result << char + else + unordered << ccc * n + i end end - return sorting.collect(&:first).join('') + unordered.sort!.each { result << chars[it % n] } + result end ## Normalization Forms for Patterns (not whole Strings) @@ -105,16 +111,22 @@ module UnicodeNormalize # :nodoc: start = nfd_string[0] last_class = CLASS_TABLE[start]-1 accents = '' + result = '' nfd_string[1..-1].each_char do |accent| accent_class = CLASS_TABLE[accent] if last_class<accent_class and composite = COMPOSITION_TABLE[start+accent] start = composite + elsif accent_class == 0 + result << start << accents + start = accent + accents = '' + last_class = -1 else accents << accent last_class = accent_class end end - hangul_comp_one(start+accents) + hangul_comp_one(result+start+accents) end def self.normalize(string, form = :nfc) diff --git a/lib/unicode_normalize/tables.rb b/lib/unicode_normalize/tables.rb index 7448fad13f..dd5d3499b8 100644 --- a/lib/unicode_normalize/tables.rb +++ b/lib/unicode_normalize/tables.rb @@ -1,6 +1,9 @@ # coding: us-ascii # frozen_string_literal: true +Encoding::UNICODE_VERSION == "17.0.0" or + raise "Unicode version mismatch: 17.0.0 expected but #{Encoding::UNICODE_VERSION}" + # automatically generated by template/unicode_norm_gen.tmpl module UnicodeNormalize # :nodoc: @@ -29,7 +32,7 @@ module UnicodeNormalize # :nodoc: "\u0825-\u0827" \ "\u0829-\u082D" \ "\u0859-\u085B" \ - "\u0898-\u089F" \ + "\u0897-\u089F" \ "\u08CA-\u08E1" \ "\u08E3-\u08FF" \ "\u093C" \ @@ -96,7 +99,8 @@ module UnicodeNormalize # :nodoc: "\u1A75-\u1A7C" \ "\u1A7F" \ "\u1AB0-\u1ABD" \ - "\u1ABF-\u1ACE" \ + "\u1ABF-\u1ADD" \ + "\u1AE0-\u1AEB" \ "\u1B34\u1B35" \ "\u1B44" \ "\u1B6B-\u1B73" \ @@ -149,7 +153,9 @@ module UnicodeNormalize # :nodoc: "\u{10A3F}" \ "\u{10AE5}\u{10AE6}" \ "\u{10D24}-\u{10D27}" \ + "\u{10D69}-\u{10D6D}" \ "\u{10EAB}\u{10EAC}" \ + "\u{10EFA}\u{10EFB}" \ "\u{10EFD}-\u{10EFF}" \ "\u{10F46}-\u{10F50}" \ "\u{10F82}-\u{10F85}" \ @@ -171,6 +177,12 @@ module UnicodeNormalize # :nodoc: "\u{11357}" \ "\u{11366}-\u{1136C}" \ "\u{11370}-\u{11374}" \ + "\u{113B8}" \ + "\u{113BB}" \ + "\u{113C2}" \ + "\u{113C5}" \ + "\u{113C7}-\u{113C9}" \ + "\u{113CE}-\u{113D0}" \ "\u{11442}" \ "\u{11446}" \ "\u{1145E}" \ @@ -196,8 +208,11 @@ module UnicodeNormalize # :nodoc: "\u{11D44}\u{11D45}" \ "\u{11D97}" \ "\u{11F41}\u{11F42}" \ + "\u{1611E}-\u{16129}" \ + "\u{1612F}" \ "\u{16AF0}-\u{16AF4}" \ "\u{16B30}-\u{16B36}" \ + "\u{16D67}\u{16D68}" \ "\u{16FF0}\u{16FF1}" \ "\u{1BC9E}" \ "\u{1D165}-\u{1D169}" \ @@ -216,6 +231,11 @@ module UnicodeNormalize # :nodoc: "\u{1E2AE}" \ "\u{1E2EC}-\u{1E2EF}" \ "\u{1E4EC}-\u{1E4EF}" \ + "\u{1E5EE}\u{1E5EF}" \ + "\u{1E6E3}" \ + "\u{1E6E6}" \ + "\u{1E6EE}\u{1E6EF}" \ + "\u{1E6F5}" \ "\u{1E8D0}-\u{1E8D6}" \ "\u{1E944}-\u{1E94A}" \ "]" @@ -441,15 +461,25 @@ module UnicodeNormalize # :nodoc: "\uFB40\uFB41" \ "\uFB43\uFB44" \ "\uFB46-\uFB4E" \ + "\u{105C9}" \ + "\u{105E4}" \ "\u{1109A}" \ "\u{1109C}" \ "\u{110AB}" \ "\u{1112E}\u{1112F}" \ "\u{1134B}\u{1134C}" \ + "\u{11383}" \ + "\u{11385}" \ + "\u{1138E}" \ + "\u{11391}" \ + "\u{113C5}" \ + "\u{113C7}\u{113C8}" \ "\u{114BB}\u{114BC}" \ "\u{114BE}" \ "\u{115BA}\u{115BB}" \ "\u{11938}" \ + "\u{16121}-\u{16128}" \ + "\u{16D68}-\u{16D6A}" \ "\u{1D15E}-\u{1D164}" \ "\u{1D1BB}-\u{1D1C0}" \ "\u{2F800}-\u{2FA1D}" \ @@ -613,14 +643,25 @@ module UnicodeNormalize # :nodoc: "\u30DB" \ "\u30EF-\u30F2" \ "\u30FD" \ + "\u{105D2}" \ + "\u{105DA}" \ "\u{11099}" \ "\u{1109B}" \ "\u{110A5}" \ "\u{11131}\u{11132}" \ "\u{11347}" \ + "\u{11382}" \ + "\u{11384}" \ + "\u{1138B}" \ + "\u{11390}" \ + "\u{113C2}" \ "\u{114B9}" \ "\u{115B8}\u{115B9}" \ "\u{11935}" \ + "\u{1611E}" \ + "\u{16129}" \ + "\u{16D63}" \ + "\u{16D67}" \ "]?#{accents}+" \ "|#{'' # precomposed Hangul syllables }" \ @@ -891,6 +932,10 @@ module UnicodeNormalize # :nodoc: "\u30F4" \ "\u30F7-\u30FA" \ "\u30FD\u30FE" \ + "\u{105C9}" \ + "\u{105D2}" \ + "\u{105DA}" \ + "\u{105E4}" \ "\u{11099}-\u{1109C}" \ "\u{110A5}" \ "\u{110AB}" \ @@ -898,12 +943,23 @@ module UnicodeNormalize # :nodoc: "\u{11131}\u{11132}" \ "\u{11347}" \ "\u{1134B}\u{1134C}" \ + "\u{11382}-\u{11385}" \ + "\u{1138B}" \ + "\u{1138E}" \ + "\u{11390}\u{11391}" \ + "\u{113C2}" \ + "\u{113C5}" \ + "\u{113C7}\u{113C8}" \ "\u{114B9}" \ "\u{114BB}\u{114BC}" \ "\u{114BE}" \ "\u{115B8}-\u{115BB}" \ "\u{11935}" \ "\u{11938}" \ + "\u{1611E}" \ + "\u{16121}-\u{16129}" \ + "\u{16D63}" \ + "\u{16D67}-\u{16D6A}" \ "]?#{accents}+" \ "|#{'' # Hangul syllables with separate trailer }" \ @@ -1410,7 +1466,7 @@ module UnicodeNormalize # :nodoc: "\u3280-\u33FF" \ "\uA69C\uA69D" \ "\uA770" \ - "\uA7F2-\uA7F4" \ + "\uA7F1-\uA7F4" \ "\uA7F8\uA7F9" \ "\uAB5C-\uAB5F" \ "\uAB69" \ @@ -1440,6 +1496,7 @@ module UnicodeNormalize # :nodoc: "\u{10781}-\u{10785}" \ "\u{10787}-\u{107B0}" \ "\u{107B2}-\u{107BA}" \ + "\u{1CCD6}-\u{1CCF9}" \ "\u{1D400}-\u{1D454}" \ "\u{1D456}-\u{1D49C}" \ "\u{1D49E}\u{1D49F}" \ @@ -1789,6 +1846,7 @@ module UnicodeNormalize # :nodoc: "\u0859"=>220, "\u085A"=>220, "\u085B"=>220, + "\u0897"=>230, "\u0898"=>230, "\u0899"=>220, "\u089A"=>220, @@ -1967,6 +2025,33 @@ module UnicodeNormalize # :nodoc: "\u1ACC"=>230, "\u1ACD"=>230, "\u1ACE"=>230, + "\u1ACF"=>230, + "\u1AD0"=>230, + "\u1AD1"=>230, + "\u1AD2"=>230, + "\u1AD3"=>230, + "\u1AD4"=>230, + "\u1AD5"=>230, + "\u1AD6"=>230, + "\u1AD7"=>230, + "\u1AD8"=>230, + "\u1AD9"=>230, + "\u1ADA"=>230, + "\u1ADB"=>230, + "\u1ADC"=>230, + "\u1ADD"=>220, + "\u1AE0"=>230, + "\u1AE1"=>230, + "\u1AE2"=>230, + "\u1AE3"=>230, + "\u1AE4"=>230, + "\u1AE5"=>230, + "\u1AE6"=>220, + "\u1AE7"=>230, + "\u1AE8"=>230, + "\u1AE9"=>230, + "\u1AEA"=>230, + "\u1AEB"=>234, "\u1B34"=>7, "\u1B44"=>9, "\u1B6B"=>230, @@ -2234,8 +2319,15 @@ module UnicodeNormalize # :nodoc: "\u{10D25}"=>230, "\u{10D26}"=>230, "\u{10D27}"=>230, + "\u{10D69}"=>230, + "\u{10D6A}"=>230, + "\u{10D6B}"=>230, + "\u{10D6C}"=>230, + "\u{10D6D}"=>230, "\u{10EAB}"=>230, "\u{10EAC}"=>230, + "\u{10EFA}"=>220, + "\u{10EFB}"=>220, "\u{10EFD}"=>220, "\u{10EFE}"=>220, "\u{10EFF}"=>220, @@ -2286,6 +2378,9 @@ module UnicodeNormalize # :nodoc: "\u{11372}"=>230, "\u{11373}"=>230, "\u{11374}"=>230, + "\u{113CE}"=>9, + "\u{113CF}"=>9, + "\u{113D0}"=>9, "\u{11442}"=>9, "\u{11446}"=>7, "\u{1145E}"=>230, @@ -2313,6 +2408,7 @@ module UnicodeNormalize # :nodoc: "\u{11D97}"=>9, "\u{11F41}"=>9, "\u{11F42}"=>9, + "\u{1612F}"=>9, "\u{16AF0}"=>1, "\u{16AF1}"=>1, "\u{16AF2}"=>1, @@ -2416,6 +2512,13 @@ module UnicodeNormalize # :nodoc: "\u{1E4ED}"=>232, "\u{1E4EE}"=>220, "\u{1E4EF}"=>230, + "\u{1E5EE}"=>230, + "\u{1E5EF}"=>220, + "\u{1E6E3}"=>230, + "\u{1E6E6}"=>230, + "\u{1E6EE}"=>230, + "\u{1E6EF}"=>230, + "\u{1E6F5}"=>230, "\u{1E8D0}"=>220, "\u{1E8D1}"=>220, "\u{1E8D2}"=>220, @@ -3928,6 +4031,8 @@ module UnicodeNormalize # :nodoc: "\uFB4C"=>"\u05D1\u05BF", "\uFB4D"=>"\u05DB\u05BF", "\uFB4E"=>"\u05E4\u05BF", + "\u{105C9}"=>"\u{105D2}\u0307", + "\u{105E4}"=>"\u{105DA}\u0307", "\u{1109A}"=>"\u{11099}\u{110BA}", "\u{1109C}"=>"\u{1109B}\u{110BA}", "\u{110AB}"=>"\u{110A5}\u{110BA}", @@ -3935,12 +4040,30 @@ module UnicodeNormalize # :nodoc: "\u{1112F}"=>"\u{11132}\u{11127}", "\u{1134B}"=>"\u{11347}\u{1133E}", "\u{1134C}"=>"\u{11347}\u{11357}", + "\u{11383}"=>"\u{11382}\u{113C9}", + "\u{11385}"=>"\u{11384}\u{113BB}", + "\u{1138E}"=>"\u{1138B}\u{113C2}", + "\u{11391}"=>"\u{11390}\u{113C9}", + "\u{113C5}"=>"\u{113C2}\u{113C2}", + "\u{113C7}"=>"\u{113C2}\u{113B8}", + "\u{113C8}"=>"\u{113C2}\u{113C9}", "\u{114BB}"=>"\u{114B9}\u{114BA}", "\u{114BC}"=>"\u{114B9}\u{114B0}", "\u{114BE}"=>"\u{114B9}\u{114BD}", "\u{115BA}"=>"\u{115B8}\u{115AF}", "\u{115BB}"=>"\u{115B9}\u{115AF}", "\u{11938}"=>"\u{11935}\u{11930}", + "\u{16121}"=>"\u{1611E}\u{1611E}", + "\u{16122}"=>"\u{1611E}\u{16129}", + "\u{16123}"=>"\u{1611E}\u{1611F}", + "\u{16124}"=>"\u{16129}\u{1611F}", + "\u{16125}"=>"\u{1611E}\u{16120}", + "\u{16126}"=>"\u{1611E}\u{1611E}\u{1611F}", + "\u{16127}"=>"\u{1611E}\u{16129}\u{1611F}", + "\u{16128}"=>"\u{1611E}\u{1611E}\u{16120}", + "\u{16D68}"=>"\u{16D67}\u{16D67}", + "\u{16D69}"=>"\u{16D63}\u{16D67}", + "\u{16D6A}"=>"\u{16D63}\u{16D67}\u{16D67}", "\u{1D15E}"=>"\u{1D157}\u{1D165}", "\u{1D15F}"=>"\u{1D158}\u{1D165}", "\u{1D160}"=>"\u{1D158}\u{1D165}\u{1D16E}", @@ -5839,6 +5962,7 @@ module UnicodeNormalize # :nodoc: "\uA69C"=>"\u044A", "\uA69D"=>"\u044C", "\uA770"=>"\uA76F", + "\uA7F1"=>"S", "\uA7F2"=>"C", "\uA7F3"=>"F", "\uA7F4"=>"Q", @@ -6950,6 +7074,42 @@ module UnicodeNormalize # :nodoc: "\u{107B8}"=>"\u01C2", "\u{107B9}"=>"\u{1DF0A}", "\u{107BA}"=>"\u{1DF1E}", + "\u{1CCD6}"=>"A", + "\u{1CCD7}"=>"B", + "\u{1CCD8}"=>"C", + "\u{1CCD9}"=>"D", + "\u{1CCDA}"=>"E", + "\u{1CCDB}"=>"F", + "\u{1CCDC}"=>"G", + "\u{1CCDD}"=>"H", + "\u{1CCDE}"=>"I", + "\u{1CCDF}"=>"J", + "\u{1CCE0}"=>"K", + "\u{1CCE1}"=>"L", + "\u{1CCE2}"=>"M", + "\u{1CCE3}"=>"N", + "\u{1CCE4}"=>"O", + "\u{1CCE5}"=>"P", + "\u{1CCE6}"=>"Q", + "\u{1CCE7}"=>"R", + "\u{1CCE8}"=>"S", + "\u{1CCE9}"=>"T", + "\u{1CCEA}"=>"U", + "\u{1CCEB}"=>"V", + "\u{1CCEC}"=>"W", + "\u{1CCED}"=>"X", + "\u{1CCEE}"=>"Y", + "\u{1CCEF}"=>"Z", + "\u{1CCF0}"=>"0", + "\u{1CCF1}"=>"1", + "\u{1CCF2}"=>"2", + "\u{1CCF3}"=>"3", + "\u{1CCF4}"=>"4", + "\u{1CCF5}"=>"5", + "\u{1CCF6}"=>"6", + "\u{1CCF7}"=>"7", + "\u{1CCF8}"=>"8", + "\u{1CCF9}"=>"9", "\u{1D400}"=>"A", "\u{1D401}"=>"B", "\u{1D402}"=>"C", @@ -9242,6 +9402,8 @@ module UnicodeNormalize # :nodoc: "\u30F1\u3099"=>"\u30F9", "\u30F2\u3099"=>"\u30FA", "\u30FD\u3099"=>"\u30FE", + "\u{105D2}\u0307"=>"\u{105C9}", + "\u{105DA}\u0307"=>"\u{105E4}", "\u{11099}\u{110BA}"=>"\u{1109A}", "\u{1109B}\u{110BA}"=>"\u{1109C}", "\u{110A5}\u{110BA}"=>"\u{110AB}", @@ -9249,11 +9411,29 @@ module UnicodeNormalize # :nodoc: "\u{11132}\u{11127}"=>"\u{1112F}", "\u{11347}\u{1133E}"=>"\u{1134B}", "\u{11347}\u{11357}"=>"\u{1134C}", + "\u{11382}\u{113C9}"=>"\u{11383}", + "\u{11384}\u{113BB}"=>"\u{11385}", + "\u{1138B}\u{113C2}"=>"\u{1138E}", + "\u{11390}\u{113C9}"=>"\u{11391}", + "\u{113C2}\u{113C2}"=>"\u{113C5}", + "\u{113C2}\u{113B8}"=>"\u{113C7}", + "\u{113C2}\u{113C9}"=>"\u{113C8}", "\u{114B9}\u{114BA}"=>"\u{114BB}", "\u{114B9}\u{114B0}"=>"\u{114BC}", "\u{114B9}\u{114BD}"=>"\u{114BE}", "\u{115B8}\u{115AF}"=>"\u{115BA}", "\u{115B9}\u{115AF}"=>"\u{115BB}", "\u{11935}\u{11930}"=>"\u{11938}", + "\u{1611E}\u{1611E}"=>"\u{16121}", + "\u{1611E}\u{16129}"=>"\u{16122}", + "\u{1611E}\u{1611F}"=>"\u{16123}", + "\u{16129}\u{1611F}"=>"\u{16124}", + "\u{1611E}\u{16120}"=>"\u{16125}", + "\u{16121}\u{1611F}"=>"\u{16126}", + "\u{16122}\u{1611F}"=>"\u{16127}", + "\u{16121}\u{16120}"=>"\u{16128}", + "\u{16D67}\u{16D67}"=>"\u{16D68}", + "\u{16D63}\u{16D67}"=>"\u{16D69}", + "\u{16D69}\u{16D67}"=>"\u{16D6A}", }.freeze end |