summaryrefslogtreecommitdiff
path: root/enc/utf_8.c
diff options
context:
space:
mode:
Diffstat (limited to 'enc/utf_8.c')
-rw-r--r--enc/utf_8.c74
1 files changed, 36 insertions, 38 deletions
diff --git a/enc/utf_8.c b/enc/utf_8.c
index 3b9387c613..ae7c98469d 100644
--- a/enc/utf_8.c
+++ b/enc/utf_8.c
@@ -28,15 +28,22 @@
*/
#include "regenc.h"
+#ifdef RUBY
+# include "encindex.h"
+#endif
+
+#ifndef ENCINDEX_UTF_8
+# define ENCINDEX_UTF_8 0
+#endif
#define USE_INVALID_CODE_SCHEME
#ifdef USE_INVALID_CODE_SCHEME
/* virtual codepoint values for invalid encoding byte 0xfe and 0xff */
-#define INVALID_CODE_FE 0xfffffffe
-#define INVALID_CODE_FF 0xffffffff
-#define VALID_CODE_LIMIT 0x7fffffff
+# define INVALID_CODE_FE 0xfffffffe
+# define INVALID_CODE_FF 0xffffffff
#endif
+#define VALID_CODE_LIMIT 0x0010ffff
#define utf8_islead(c) ((UChar )((c) & 0xc0) != 0x80)
@@ -84,7 +91,7 @@ static const signed char trans[][0x100] = {
/* c */ F, F, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
/* d */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
/* e */ 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3,
- /* f */ 5, 6, 6, 6, 7, F, F, F, F, F, F, F, F, F, F, F
+ /* f */ 5, 6, 6, 6, 7, F, F, F, F, F, F, F, F, F, F, F
},
{ /* S1 0 1 2 3 4 5 6 7 8 9 a b c d e f */
/* 0 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
@@ -102,7 +109,7 @@ static const signed char trans[][0x100] = {
/* c */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
/* d */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
/* e */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* f */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F
+ /* f */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F
},
{ /* S2 0 1 2 3 4 5 6 7 8 9 a b c d e f */
/* 0 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
@@ -120,7 +127,7 @@ static const signed char trans[][0x100] = {
/* c */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
/* d */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
/* e */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* f */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F
+ /* f */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F
},
{ /* S3 0 1 2 3 4 5 6 7 8 9 a b c d e f */
/* 0 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
@@ -138,7 +145,7 @@ static const signed char trans[][0x100] = {
/* c */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
/* d */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
/* e */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* f */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F
+ /* f */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F
},
{ /* S4 0 1 2 3 4 5 6 7 8 9 a b c d e f */
/* 0 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
@@ -156,7 +163,7 @@ static const signed char trans[][0x100] = {
/* c */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
/* d */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
/* e */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* f */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F
+ /* f */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F
},
{ /* S5 0 1 2 3 4 5 6 7 8 9 a b c d e f */
/* 0 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
@@ -174,7 +181,7 @@ static const signed char trans[][0x100] = {
/* c */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
/* d */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
/* e */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* f */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F
+ /* f */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F
},
{ /* S6 0 1 2 3 4 5 6 7 8 9 a b c d e f */
/* 0 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
@@ -192,7 +199,7 @@ static const signed char trans[][0x100] = {
/* c */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
/* d */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
/* e */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* f */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F
+ /* f */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F
},
{ /* S7 0 1 2 3 4 5 6 7 8 9 a b c d e f */
/* 0 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
@@ -210,7 +217,7 @@ static const signed char trans[][0x100] = {
/* c */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
/* d */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
/* e */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F,
- /* f */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F
+ /* f */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F
},
};
#undef A
@@ -248,9 +255,7 @@ is_mbc_newline(const UChar* p, const UChar* end, OnigEncoding enc)
if (*p == 0x0a) return 1;
#ifdef USE_UNICODE_ALL_LINE_TERMINATORS
-#ifndef USE_CRNL_AS_LINE_TERMINATOR
- if (*p == 0x0d) return 1;
-#endif
+ if (*p == 0x0b || *p == 0x0c || *p == 0x0d) return 1;
if (p + 1 < end) {
if (*(p+1) == 0x85 && *p == 0xc2) /* U+0085 */
return 1;
@@ -272,7 +277,7 @@ mbc_to_code(const UChar* p, const UChar* end, OnigEncoding enc)
int c, len;
OnigCodePoint n;
- len = enclen(enc, p, end);
+ len = mbc_enc_len(p, end, enc);
c = *p++;
if (len > 1) {
len--;
@@ -299,9 +304,7 @@ code_to_mbclen(OnigCodePoint code, OnigEncoding enc ARG_UNUSED)
if ((code & 0xffffff80) == 0) return 1;
else if ((code & 0xfffff800) == 0) return 2;
else if ((code & 0xffff0000) == 0) return 3;
- else if ((code & 0xffe00000) == 0) return 4;
- else if ((code & 0xfc000000) == 0) return 5;
- else if ((code & 0x80000000) == 0) return 6;
+ else if (code <= VALID_CODE_LIMIT) return 4;
#ifdef USE_INVALID_CODE_SCHEME
else if (code == INVALID_CODE_FE) return 1;
else if (code == INVALID_CODE_FF) return 1;
@@ -330,24 +333,11 @@ code_to_mbc(OnigCodePoint code, UChar *buf, OnigEncoding enc ARG_UNUSED)
*p++ = (UChar )(((code>>12) & 0x0f) | 0xe0);
*p++ = UTF8_TRAILS(code, 6);
}
- else if ((code & 0xffe00000) == 0) {
+ else if (code <= VALID_CODE_LIMIT) {
*p++ = (UChar )(((code>>18) & 0x07) | 0xf0);
*p++ = UTF8_TRAILS(code, 12);
*p++ = UTF8_TRAILS(code, 6);
}
- else if ((code & 0xfc000000) == 0) {
- *p++ = (UChar )(((code>>24) & 0x03) | 0xf8);
- *p++ = UTF8_TRAILS(code, 18);
- *p++ = UTF8_TRAILS(code, 12);
- *p++ = UTF8_TRAILS(code, 6);
- }
- else if ((code & 0x80000000) == 0) {
- *p++ = (UChar )(((code>>30) & 0x01) | 0xfc);
- *p++ = UTF8_TRAILS(code, 24);
- *p++ = UTF8_TRAILS(code, 18);
- *p++ = UTF8_TRAILS(code, 12);
- *p++ = UTF8_TRAILS(code, 6);
- }
#ifdef USE_INVALID_CODE_SCHEME
else if (code == INVALID_CODE_FE) {
*p = 0xfe;
@@ -363,13 +353,13 @@ code_to_mbc(OnigCodePoint code, UChar *buf, OnigEncoding enc ARG_UNUSED)
}
*p++ = UTF8_TRAIL0(code);
- return p - buf;
+ return (int )(p - buf);
}
}
static int
mbc_case_fold(OnigCaseFoldType flag, const UChar** pp,
- const UChar* end, UChar* fold, OnigEncoding enc)
+ const UChar* end, UChar* fold, OnigEncoding enc)
{
const UChar* p = *pp;
@@ -397,7 +387,7 @@ mbc_case_fold(OnigCaseFoldType flag, const UChar** pp,
static int
get_ctype_code_range(OnigCtype ctype, OnigCodePoint *sb_out,
- const OnigCodePoint* ranges[], OnigEncoding enc ARG_UNUSED)
+ const OnigCodePoint* ranges[], OnigEncoding enc ARG_UNUSED)
{
*sb_out = 0x80;
return onigenc_unicode_ctype_code_range(ctype, ranges);
@@ -427,7 +417,7 @@ get_case_fold_codes_by_str(OnigCaseFoldType flag,
OnigEncodingDefine(utf_8, UTF_8) = {
mbc_enc_len,
"UTF-8", /* name */
- 6, /* max byte length */
+ 4, /* max byte length */
1, /* min byte length */
is_mbc_newline,
mbc_to_code,
@@ -440,7 +430,14 @@ OnigEncodingDefine(utf_8, UTF_8) = {
onigenc_unicode_is_code_ctype,
get_ctype_code_range,
left_adjust_char_head,
- onigenc_always_true_is_allowed_reverse_match
+ onigenc_always_true_is_allowed_reverse_match,
+#ifdef USE_CASE_MAP_API
+ onigenc_unicode_case_map,
+#else
+ NULL,
+#endif
+ ENCINDEX_UTF_8,
+ ONIGENC_FLAG_UNICODE,
};
ENC_ALIAS("CP65001", "UTF-8")
@@ -449,7 +446,8 @@ ENC_ALIAS("CP65001", "UTF-8")
* Link: http://developer.apple.com/documentation/MacOSX/Conceptual/BPFileSystem/BPFileSystem.html
* Link: http://developer.apple.com/qa/qa2001/qa1235.html
* Link: http://developer.apple.com/jp/qa/qa2001/qa1235.html
+ * Link: http://www.gnu.org/software/emacs/NEWS.23.2
*/
ENC_REPLICATE("UTF8-MAC", "UTF-8")
ENC_ALIAS("UTF-8-MAC", "UTF8-MAC")
-
+ENC_ALIAS("UTF-8-HFS", "UTF8-MAC") /* Emacs 23.2 */
generated by cgit v1.2.3 (git 2.25.1) at 2026-06-06 14:19:56 +0000