1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
|
#include "prism/util/pm_strpbrk.h"
/**
* Add an invalid multibyte character error to the parser.
*/
static inline void
pm_strpbrk_invalid_multibyte_character(pm_parser_t *parser, const uint8_t *start, const uint8_t *end) {
pm_diagnostic_list_append_format(&parser->error_list, start, end, PM_ERR_INVALID_MULTIBYTE_CHARACTER, *start);
}
/**
* This is the default path.
*/
static inline const uint8_t *
pm_strpbrk_utf8(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, size_t maximum, bool validate) {
size_t index = 0;
while (index < maximum) {
if (strchr((const char *) charset, source[index]) != NULL) {
return source + index;
}
if (source[index] < 0x80) {
index++;
} else {
size_t width = pm_encoding_utf_8_char_width(source + index, (ptrdiff_t) (maximum - index));
if (width > 0) {
index += width;
} else if (!validate) {
index++;
} else {
// At this point we know we have an invalid multibyte character.
// We'll walk forward as far as we can until we find the next
// valid character so that we don't spam the user with a ton of
// the same kind of error.
const size_t start = index;
do {
index++;
} while (index < maximum && pm_encoding_utf_8_char_width(source + index, (ptrdiff_t) (maximum - index)) == 0);
pm_strpbrk_invalid_multibyte_character(parser, source + start, source + index);
}
}
}
return NULL;
}
/**
* This is the path when the encoding is ASCII-8BIT.
*/
static inline const uint8_t *
pm_strpbrk_ascii_8bit(const uint8_t *source, const uint8_t *charset, size_t maximum) {
size_t index = 0;
while (index < maximum) {
if (strchr((const char *) charset, source[index]) != NULL) {
return source + index;
}
index++;
}
return NULL;
}
/**
* This is the slow path that does care about the encoding.
*/
static inline const uint8_t *
pm_strpbrk_multi_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, size_t maximum, bool validate) {
size_t index = 0;
while (index < maximum) {
if (strchr((const char *) charset, source[index]) != NULL) {
return source + index;
}
if (source[index] < 0x80) {
index++;
} else {
size_t width = parser->encoding->char_width(source + index, (ptrdiff_t) (maximum - index));
if (width > 0) {
index += width;
} else if (!validate) {
index++;
} else {
// At this point we know we have an invalid multibyte character.
// We'll walk forward as far as we can until we find the next
// valid character so that we don't spam the user with a ton of
// the same kind of error.
const size_t start = index;
do {
index++;
} while (index < maximum && parser->encoding->char_width(source + index, (ptrdiff_t) (maximum - index)) == 0);
pm_strpbrk_invalid_multibyte_character(parser, source + start, source + index);
}
}
}
return NULL;
}
/**
* This is the fast path that does not care about the encoding because we know
* the encoding only supports single-byte characters.
*/
static inline const uint8_t *
pm_strpbrk_single_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, size_t maximum, bool validate) {
size_t index = 0;
while (index < maximum) {
if (strchr((const char *) charset, source[index]) != NULL) {
return source + index;
}
if (source[index] < 0x80 || !validate) {
index++;
} else {
size_t width = parser->encoding->char_width(source + index, (ptrdiff_t) (maximum - index));
if (width > 0) {
index += width;
} else {
// At this point we know we have an invalid multibyte character.
// We'll walk forward as far as we can until we find the next
// valid character so that we don't spam the user with a ton of
// the same kind of error.
const size_t start = index;
do {
index++;
} while (index < maximum && parser->encoding->char_width(source + index, (ptrdiff_t) (maximum - index)) == 0);
pm_strpbrk_invalid_multibyte_character(parser, source + start, source + index);
}
}
}
return NULL;
}
/**
* Here we have rolled our own version of strpbrk. The standard library strpbrk
* has undefined behavior when the source string is not null-terminated. We want
* to support strings that are not null-terminated because pm_parse does not
* have the contract that the string is null-terminated. (This is desirable
* because it means the extension can call pm_parse with the result of a call to
* mmap).
*
* The standard library strpbrk also does not support passing a maximum length
* to search. We want to support this for the reason mentioned above, but we
* also don't want it to stop on null bytes. Ruby actually allows null bytes
* within strings, comments, regular expressions, etc. So we need to be able to
* skip past them.
*
* Finally, we want to support encodings wherein the charset could contain
* characters that are trailing bytes of multi-byte characters. For example, in
* Shift_JIS, the backslash character can be a trailing byte. In that case we
* need to take a slower path and iterate one multi-byte character at a time.
*/
const uint8_t *
pm_strpbrk(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, ptrdiff_t length, bool validate) {
if (length <= 0) {
return NULL;
} else if (!parser->encoding_changed) {
return pm_strpbrk_utf8(parser, source, charset, (size_t) length, validate);
} else if (parser->encoding == PM_ENCODING_ASCII_8BIT_ENTRY) {
return pm_strpbrk_ascii_8bit(source, charset, (size_t) length);
} else if (parser->encoding->multibyte) {
return pm_strpbrk_multi_byte(parser, source, charset, (size_t) length, validate);
} else {
return pm_strpbrk_single_byte(parser, source, charset, (size_t) length, validate);
}
}
|
