summaryrefslogtreecommitdiff
path: root/lib/unicode_normalize
diff options
context:
space:
mode:
authorsvn <svn@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>2014-10-19 00:38:53 +0000
committersvn <svn@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>2014-10-19 00:38:53 +0000
commitd64dc54e0ce890204da96b4575d8f9557f480b8a (patch)
treea6befe3cbbaa02df6cad0f5772c57af48d4d76b7 /lib/unicode_normalize
parent165b9120c5f80d2bffbca427c499a76ecd91f55b (diff)
* properties.
git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@48007 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
Diffstat (limited to 'lib/unicode_normalize')
-rw-r--r--lib/unicode_normalize/normalize.rb352
1 files changed, 176 insertions, 176 deletions
diff --git a/lib/unicode_normalize/normalize.rb b/lib/unicode_normalize/normalize.rb
index 4f0fdbc..db1aaac 100644
--- a/lib/unicode_normalize/normalize.rb
+++ b/lib/unicode_normalize/normalize.rb
@@ -1,176 +1,176 @@
-# coding: utf-8
-
-# Copyright 2010-2013 Ayumu Nojima (野島 歩) and Martin J. Dürst (duerst@it.aoyama.ac.jp)
-# available under the same licence as Ruby itself
-# (see http://www.ruby-lang.org/en/LICENSE.txt)
-
-require_relative 'normalize_tables'
-
-
-module Normalize
- ## Constant for max hash capacity to avoid DoS attack
- MAX_HASH_LENGTH = 18000 # enough for all test cases, otherwise tests get slow
-
- ## Regular Expressions and Hash Constants
- REGEXP_D = Regexp.compile(REGEXP_D_STRING, Regexp::EXTENDED)
- REGEXP_C = Regexp.compile(REGEXP_C_STRING, Regexp::EXTENDED)
- REGEXP_K = Regexp.compile(REGEXP_K_STRING, Regexp::EXTENDED)
- NF_HASH_D = Hash.new do |hash, key|
- hash.delete hash.first[0] if hash.length>MAX_HASH_LENGTH # prevent DoS attack
- hash[key] = Normalize.nfd_one(key)
- end
- NF_HASH_C = Hash.new do |hash, key|
- hash.delete hash.first[0] if hash.length>MAX_HASH_LENGTH # prevent DoS attack
- hash[key] = Normalize.nfc_one(key)
- end
- NF_HASH_K = Hash.new do |hash, key|
- hash.delete hash.first[0] if hash.length>MAX_HASH_LENGTH # prevent DoS attack
- hash[key] = Normalize.nfkd_one(key)
- end
-
- ## Constants For Hangul
- SBASE = 0xAC00
- LBASE = 0x1100
- VBASE = 0x1161
- TBASE = 0x11A7
- LCOUNT = 19
- VCOUNT = 21
- TCOUNT = 28
- NCOUNT = VCOUNT * TCOUNT
- SCOUNT = LCOUNT * NCOUNT
-
- # Unicode-based encodings (except UTF-8)
- UNICODE_ENCODINGS = [Encoding::UTF_16BE, Encoding::UTF_16LE, Encoding::UTF_32BE, Encoding::UTF_32LE,
- Encoding::GB18030, Encoding::UCS_2BE, Encoding::UCS_4BE]
-
- ## Hangul Algorithm
- def Normalize.hangul_decomp_one(target)
- sIndex = target.ord - SBASE
- return target if sIndex < 0 || sIndex >= SCOUNT
- l = LBASE + sIndex / NCOUNT
- v = VBASE + (sIndex % NCOUNT) / TCOUNT
- t = TBASE + sIndex % TCOUNT
- (t==TBASE ? [l, v] : [l, v, t]).pack('U*') + target[1..-1]
- end
-
- def Normalize.hangul_comp_one(string)
- length = string.length
- if length>1 and 0 <= (lead =string[0].ord-LBASE) and lead < LCOUNT and
- 0 <= (vowel=string[1].ord-VBASE) and vowel < VCOUNT
- lead_vowel = SBASE + (lead * VCOUNT + vowel) * TCOUNT
- if length>2 and 0 <= (trail=string[2].ord-TBASE) and trail < TCOUNT
- (lead_vowel + trail).chr(Encoding::UTF_8) + string[3..-1]
- else
- lead_vowel.chr(Encoding::UTF_8) + string[2..-1]
- end
- else
- string
- end
- end
-
- ## Canonical Ordering
- def Normalize.canonical_ordering_one(string)
- sorting = string.each_char.collect { |c| [c, CLASS_TABLE[c]] }
- (sorting.length-2).downto(0) do |i| # bubble sort
- (0..i).each do |j|
- later_class = sorting[j+1].last
- if 0<later_class and later_class<sorting[j].last
- sorting[j], sorting[j+1] = sorting[j+1], sorting[j]
- end
- end
- end
- return sorting.collect(&:first).join
- end
-
- ## Normalization Forms for Patterns (not whole Strings)
- def Normalize.nfd_one(string)
- string = string.dup
- (0...string.length).each do |position|
- if decomposition = DECOMPOSITION_TABLE[string[position]]
- string[position] = decomposition
- end
- end
- canonical_ordering_one(hangul_decomp_one(string))
- end
-
- def Normalize.nfkd_one(string)
- string = string.dup
- position = 0
- while position < string.length
- if decomposition = KOMPATIBLE_TABLE[string[position]]
- string[position] = decomposition
- else
- position += 1
- end
- end
- string
- end
-
- def Normalize.nfc_one (string)
- nfd_string = nfd_one string
- start = nfd_string[0]
- last_class = CLASS_TABLE[start]-1
- accents = ''
- nfd_string[1..-1].each_char do |accent|
- accent_class = CLASS_TABLE[accent]
- if last_class<accent_class and composite = COMPOSITION_TABLE[start+accent]
- start = composite
- else
- accents += accent
- last_class = accent_class
- end
- end
- hangul_comp_one(start+accents)
- end
-
- def Normalize.normalize(string, form = :nfc)
- encoding = string.encoding
- if encoding == Encoding::UTF_8
- case form
- when :nfc then
- string.gsub REGEXP_C, NF_HASH_C
- when :nfd then
- string.gsub REGEXP_D, NF_HASH_D
- when :nfkc then
- string.gsub(REGEXP_K, NF_HASH_K).gsub REGEXP_C, NF_HASH_C
- when :nfkd then
- string.gsub(REGEXP_K, NF_HASH_K).gsub REGEXP_D, NF_HASH_D
- else
- raise ArgumentError, "Invalid normalization form #{form}."
- end
- elsif UNICODE_ENCODINGS.include? encoding
- normalize(string.encode(Encoding::UTF_8), form).encode(encoding)
- else
- raise Encoding::CompatibilityError, "Unicode Normalization not appropriate for #{encoding}"
- end
- end
-
- def Normalize.normalized?(string, form = :nfc)
- encoding = string.encoding
- if encoding == Encoding::UTF_8
- case form
- when :nfc then
- string.scan REGEXP_C do |match|
- return false if NF_HASH_C[match] != match
- end
- true
- when :nfd then
- string.scan REGEXP_D do |match|
- return false if NF_HASH_D[match] != match
- end
- true
- when :nfkc then
- normalized?(string, :nfc) and string !~ REGEXP_K
- when :nfkd then
- normalized?(string, :nfd) and string !~ REGEXP_K
- else
- raise ArgumentError, "Invalid normalization form #{form}."
- end
- elsif UNICODE_ENCODINGS.include? encoding
- normalized? string.encode(Encoding::UTF_8), form
- else
- raise Encoding::CompatibilityError, "Unicode Normalization not appropriate for #{encoding}"
- end
- end
-
-end # module
+# coding: utf-8
+
+# Copyright 2010-2013 Ayumu Nojima (野島 歩) and Martin J. Dürst (duerst@it.aoyama.ac.jp)
+# available under the same licence as Ruby itself
+# (see http://www.ruby-lang.org/en/LICENSE.txt)
+
+require_relative 'normalize_tables'
+
+
+module Normalize
+ ## Constant for max hash capacity to avoid DoS attack
+ MAX_HASH_LENGTH = 18000 # enough for all test cases, otherwise tests get slow
+
+ ## Regular Expressions and Hash Constants
+ REGEXP_D = Regexp.compile(REGEXP_D_STRING, Regexp::EXTENDED)
+ REGEXP_C = Regexp.compile(REGEXP_C_STRING, Regexp::EXTENDED)
+ REGEXP_K = Regexp.compile(REGEXP_K_STRING, Regexp::EXTENDED)
+ NF_HASH_D = Hash.new do |hash, key|
+ hash.delete hash.first[0] if hash.length>MAX_HASH_LENGTH # prevent DoS attack
+ hash[key] = Normalize.nfd_one(key)
+ end
+ NF_HASH_C = Hash.new do |hash, key|
+ hash.delete hash.first[0] if hash.length>MAX_HASH_LENGTH # prevent DoS attack
+ hash[key] = Normalize.nfc_one(key)
+ end
+ NF_HASH_K = Hash.new do |hash, key|
+ hash.delete hash.first[0] if hash.length>MAX_HASH_LENGTH # prevent DoS attack
+ hash[key] = Normalize.nfkd_one(key)
+ end
+
+ ## Constants For Hangul
+ SBASE = 0xAC00
+ LBASE = 0x1100
+ VBASE = 0x1161
+ TBASE = 0x11A7
+ LCOUNT = 19
+ VCOUNT = 21
+ TCOUNT = 28
+ NCOUNT = VCOUNT * TCOUNT
+ SCOUNT = LCOUNT * NCOUNT
+
+ # Unicode-based encodings (except UTF-8)
+ UNICODE_ENCODINGS = [Encoding::UTF_16BE, Encoding::UTF_16LE, Encoding::UTF_32BE, Encoding::UTF_32LE,
+ Encoding::GB18030, Encoding::UCS_2BE, Encoding::UCS_4BE]
+
+ ## Hangul Algorithm
+ def Normalize.hangul_decomp_one(target)
+ sIndex = target.ord - SBASE
+ return target if sIndex < 0 || sIndex >= SCOUNT
+ l = LBASE + sIndex / NCOUNT
+ v = VBASE + (sIndex % NCOUNT) / TCOUNT
+ t = TBASE + sIndex % TCOUNT
+ (t==TBASE ? [l, v] : [l, v, t]).pack('U*') + target[1..-1]
+ end
+
+ def Normalize.hangul_comp_one(string)
+ length = string.length
+ if length>1 and 0 <= (lead =string[0].ord-LBASE) and lead < LCOUNT and
+ 0 <= (vowel=string[1].ord-VBASE) and vowel < VCOUNT
+ lead_vowel = SBASE + (lead * VCOUNT + vowel) * TCOUNT
+ if length>2 and 0 <= (trail=string[2].ord-TBASE) and trail < TCOUNT
+ (lead_vowel + trail).chr(Encoding::UTF_8) + string[3..-1]
+ else
+ lead_vowel.chr(Encoding::UTF_8) + string[2..-1]
+ end
+ else
+ string
+ end
+ end
+
+ ## Canonical Ordering
+ def Normalize.canonical_ordering_one(string)
+ sorting = string.each_char.collect { |c| [c, CLASS_TABLE[c]] }
+ (sorting.length-2).downto(0) do |i| # bubble sort
+ (0..i).each do |j|
+ later_class = sorting[j+1].last
+ if 0<later_class and later_class<sorting[j].last
+ sorting[j], sorting[j+1] = sorting[j+1], sorting[j]
+ end
+ end
+ end
+ return sorting.collect(&:first).join
+ end
+
+ ## Normalization Forms for Patterns (not whole Strings)
+ def Normalize.nfd_one(string)
+ string = string.dup
+ (0...string.length).each do |position|
+ if decomposition = DECOMPOSITION_TABLE[string[position]]
+ string[position] = decomposition
+ end
+ end
+ canonical_ordering_one(hangul_decomp_one(string))
+ end
+
+ def Normalize.nfkd_one(string)
+ string = string.dup
+ position = 0
+ while position < string.length
+ if decomposition = KOMPATIBLE_TABLE[string[position]]
+ string[position] = decomposition
+ else
+ position += 1
+ end
+ end
+ string
+ end
+
+ def Normalize.nfc_one (string)
+ nfd_string = nfd_one string
+ start = nfd_string[0]
+ last_class = CLASS_TABLE[start]-1
+ accents = ''
+ nfd_string[1..-1].each_char do |accent|
+ accent_class = CLASS_TABLE[accent]
+ if last_class<accent_class and composite = COMPOSITION_TABLE[start+accent]
+ start = composite
+ else
+ accents += accent
+ last_class = accent_class
+ end
+ end
+ hangul_comp_one(start+accents)
+ end
+
+ def Normalize.normalize(string, form = :nfc)
+ encoding = string.encoding
+ if encoding == Encoding::UTF_8
+ case form
+ when :nfc then
+ string.gsub REGEXP_C, NF_HASH_C
+ when :nfd then
+ string.gsub REGEXP_D, NF_HASH_D
+ when :nfkc then
+ string.gsub(REGEXP_K, NF_HASH_K).gsub REGEXP_C, NF_HASH_C
+ when :nfkd then
+ string.gsub(REGEXP_K, NF_HASH_K).gsub REGEXP_D, NF_HASH_D
+ else
+ raise ArgumentError, "Invalid normalization form #{form}."
+ end
+ elsif UNICODE_ENCODINGS.include? encoding
+ normalize(string.encode(Encoding::UTF_8), form).encode(encoding)
+ else
+ raise Encoding::CompatibilityError, "Unicode Normalization not appropriate for #{encoding}"
+ end
+ end
+
+ def Normalize.normalized?(string, form = :nfc)
+ encoding = string.encoding
+ if encoding == Encoding::UTF_8
+ case form
+ when :nfc then
+ string.scan REGEXP_C do |match|
+ return false if NF_HASH_C[match] != match
+ end
+ true
+ when :nfd then
+ string.scan REGEXP_D do |match|
+ return false if NF_HASH_D[match] != match
+ end
+ true
+ when :nfkc then
+ normalized?(string, :nfc) and string !~ REGEXP_K
+ when :nfkd then
+ normalized?(string, :nfd) and string !~ REGEXP_K
+ else
+ raise ArgumentError, "Invalid normalization form #{form}."
+ end
+ elsif UNICODE_ENCODINGS.include? encoding
+ normalized? string.encode(Encoding::UTF_8), form
+ else
+ raise Encoding::CompatibilityError, "Unicode Normalization not appropriate for #{encoding}"
+ end
+ end
+
+end # module