diff options
| -rw-r--r-- | ChangeLog | 87 | ||||
| -rw-r--r-- | encoding.c | 4 | ||||
| -rw-r--r-- | string.c | 154 | ||||
| -rw-r--r-- | version.h | 2 |
4 files changed, 207 insertions, 40 deletions
@@ -1,3 +1,90 @@ +Sat Oct 1 00:06:03 2016 Naohisa Goto <ngotogenome@gmail.com> + + * string.c (str_buf_cat): Fix capa size for embed string. + Fix bug in r55547. [Bug #12536] + +Sat Oct 1 00:06:03 2016 Naohisa Goto <ngotogenome@gmail.com> + + * string.c (rb_str_change_terminator_length): New function to change + termlen and resize heap for the terminator. This is split from + rb_str_fill_terminator (str_fill_term) because filling terminator + and changing terminator length are different things. [Bug #12536] + + * internal.h: declaration for rb_str_change_terminator_length. + + * string.c (str_fill_term): Simplify only to zero-fill the terminator. + For non-shared strings, it assumes that (capa + termlen) bytes of + heap is allocated. This partially reverts r55557. + + * encoding.c (rb_enc_associate_index): rb_str_change_terminator_length + is used, and it should be called whenever the termlen is changed. + + * string.c (str_capacity): New static function to return capacity + of a string with the given termlen, because the termlen may + sometimes be different from TERM_LEN(str) especially during + changing termlen or filling terminator with specific termlen. + + * string.c (rb_str_capacity): Use str_capacity. + +Sat Oct 1 00:06:03 2016 Naohisa Goto <ngotogenome@gmail.com> + + * string.c: Partially reverts r55547 and r55555. + ChangeLog about the reverted changes are also deleted in this file. + [Bug #12536] [ruby-dev:49699] [ruby-dev:49702] + +Sat Oct 1 00:06:03 2016 Naohisa Goto <ngotogenome@gmail.com> + + * string.c (str_fill_term): When termlen increases, re-allocation + of memory for termlen should always be needed. + In this fix, if possible, decrease capa instead of realloc. + [Bug #12536] [ruby-dev:49699] + +Sat Oct 1 00:06:03 2016 Naohisa Goto <ngotogenome@gmail.com> + + * string.c: Specify termlen as far as possible. + Additional fix for [Bug #12536] [ruby-dev:49699]. + + * string.c (str_new_static): Specify termlen from the given encoding + when creating a new String object is needed. + + * string.c (rb_tainted_str_new_with_enc): New function to create a + tainted String object with the given encoding. This means that + the termlen is correctly specified. Currently static function. + The function name might be renamed to rb_tainted_enc_str_new + or rb_enc_tainted_str_new. + + * string.c (rb_external_str_new_with_enc): Use encoding by using the + above rb_tainted_str_new_with_enc(). + +Sat Oct 1 00:06:03 2016 Naohisa Goto <ngotogenome@gmail.com> + + * string.c (rb_str_subseq, str_substr): When RSTRING_EMBED_LEN_MAX + is used, TERM_LEN(str) should be considered with it because + embedded strings are also processed by TERM_FILL. + Additional fix for [Bug #12536] [ruby-dev:49699]. + +Sat Oct 1 00:06:03 2016 Naohisa Goto <ngotogenome@gmail.com> + + * string.c: Fix memory corruptions when using UTF-16/32 strings. + [Bug #12536] [ruby-dev:49699] + + * string.c (rb_str_new_with_class): Use TERM_LEN of the "obj". + + * string.c (rb_str_plus, rb_str_justify): Use str_new0 which is aware + of termlen. + + * string.c (str_shared_replace): Copy +termlen bytes instead of +1. + + * string.c (rb_str_times): termlen should not be included in capa. + + * string.c (RESIZE_CAPA_TERM): When using RSTRING_EMBED_LEN_MAX, + termlen should be counted with it because embedded strings are + also processed by TERM_FILL. + + * string.c (rb_str_capacity, str_shared_replace, str_buf_cat): ditto. + + * string.c (rb_str_drop_bytes, rb_str_setbyte, str_byte_substr): ditto. + Sat Oct 1 00:00:13 2016 Eric Wong <e@80x24.org> * ext/openssl/ossl_ssl.c (ossl_ssl_write_internal): diff --git a/encoding.c b/encoding.c index 1c6719c910..07b32f5f83 100644 --- a/encoding.c +++ b/encoding.c @@ -821,8 +821,8 @@ rb_enc_associate_index(VALUE obj, int idx) } termlen = rb_enc_mbminlen(enc); oldtermlen = rb_enc_mbminlen(rb_enc_from_index(oldidx)); - if (oldtermlen < termlen && RB_TYPE_P(obj, T_STRING)) { - rb_str_fill_terminator(obj, termlen); + if (oldtermlen != termlen && RB_TYPE_P(obj, T_STRING)) { + rb_str_change_terminator_length(obj, oldtermlen, termlen); } enc_set_index(obj, idx); return obj; @@ -102,7 +102,7 @@ VALUE rb_cSymbol; } while (0) #define RESIZE_CAPA_TERM(str,capacity,termlen) do {\ if (STR_EMBED_P(str)) {\ - if ((capacity) > RSTRING_EMBED_LEN_MAX) {\ + if ((capacity) > (RSTRING_EMBED_LEN_MAX + 1 - (termlen))) {\ char *const tmp = ALLOC_N(char, (capacity)+termlen);\ const long tlen = RSTRING_LEN(str);\ memcpy(tmp, RSTRING_PTR(str), tlen);\ @@ -138,6 +138,16 @@ VALUE rb_cSymbol; #define SHARABLE_SUBSTRING_P(beg, len, end) 1 #endif +static void str_make_independent_expand(VALUE str, long len, long expand, const int termlen); + +static inline void +str_make_independent(VALUE str) +{ + long len = RSTRING_LEN(str); + int termlen = TERM_LEN(str); + str_make_independent_expand((str), len, 0L, termlen); +} + static rb_encoding * get_actual_encoding(const int encidx, VALUE str) { @@ -524,11 +534,11 @@ str_mod_check(VALUE s, const char *p, long len) } } -size_t -rb_str_capacity(VALUE str) +static size_t +str_capacity(VALUE str, const int termlen) { if (STR_EMBED_P(str)) { - return RSTRING_EMBED_LEN_MAX; + return (RSTRING_EMBED_LEN_MAX + 1 - termlen); } else if (FL_TEST(str, STR_SHARED|STR_NOFREE)) { return RSTRING(str)->as.heap.len; @@ -538,6 +548,12 @@ rb_str_capacity(VALUE str) } } +size_t +rb_str_capacity(VALUE str) +{ + return str_capacity(str, TERM_LEN(str)); +} + static inline void must_not_null(const char *ptr) { @@ -576,7 +592,7 @@ str_new0(VALUE klass, const char *ptr, long len, int termlen) } str = str_alloc(klass); - if (len > RSTRING_EMBED_LEN_MAX) { + if (len > (RSTRING_EMBED_LEN_MAX + 1 - termlen)) { RSTRING(str)->as.heap.aux.capa = len; RSTRING(str)->as.heap.ptr = ALLOC_N(char, len + termlen); STR_SET_NOEMBED(str); @@ -675,7 +691,8 @@ str_new_static(VALUE klass, const char *ptr, long len, int encindex) } if (!ptr) { - str = str_new(klass, ptr, len); + rb_encoding *enc = rb_enc_get_from_index(encindex); + str = str_new0(klass, ptr, len, rb_enc_mbminlen(enc)); } else { if (RUBY_DTRACE_STRING_CREATE_ENABLED()) { @@ -725,6 +742,15 @@ rb_tainted_str_new(const char *ptr, long len) return str; } +static VALUE +rb_tainted_str_new_with_enc(const char *ptr, long len, rb_encoding *enc) +{ + VALUE str = rb_enc_str_new(ptr, len, enc); + + OBJ_TAINT(str); + return str; +} + VALUE rb_tainted_str_new_cstr(const char *ptr) { @@ -817,7 +843,7 @@ rb_external_str_new_with_enc(const char *ptr, long len, rb_encoding *eenc) { VALUE str; - str = rb_tainted_str_new(ptr, len); + str = rb_tainted_str_new_with_enc(ptr, len, eenc); return rb_external_str_with_enc(str, eenc); } @@ -980,7 +1006,7 @@ rb_str_new_frozen(VALUE orig) VALUE rb_str_new_with_class(VALUE obj, const char *ptr, long len) { - return str_new(rb_obj_class(obj), ptr, len); + return str_new0(rb_obj_class(obj), ptr, len, TERM_LEN(obj)); } static VALUE @@ -1081,15 +1107,17 @@ rb_str_shared_replace(VALUE str, VALUE str2) { rb_encoding *enc; int cr; + int termlen; if (str == str2) return; enc = STR_ENC_GET(str2); cr = ENC_CODERANGE(str2); str_discard(str); OBJ_INFECT(str, str2); + termlen = rb_enc_mbminlen(enc); - if (RSTRING_LEN(str2) <= RSTRING_EMBED_LEN_MAX) { + if (RSTRING_LEN(str2) <= (RSTRING_EMBED_LEN_MAX + 1 - termlen)) { STR_SET_EMBED(str); - memcpy(RSTRING_PTR(str), RSTRING_PTR(str2), RSTRING_LEN(str2)+1); + memcpy(RSTRING_PTR(str), RSTRING_PTR(str2), RSTRING_LEN(str2)+termlen); STR_SET_EMBED_LEN(str, RSTRING_LEN(str2)); rb_enc_associate(str, enc); ENC_CODERANGE_SET(str, cr); @@ -1465,16 +1493,18 @@ rb_str_plus(VALUE str1, VALUE str2) rb_encoding *enc; char *ptr1, *ptr2, *ptr3; long len1, len2; + int termlen; StringValue(str2); enc = rb_enc_check(str1, str2); RSTRING_GETMEM(str1, ptr1, len1); RSTRING_GETMEM(str2, ptr2, len2); - str3 = rb_str_new(0, len1+len2); + termlen = rb_enc_mbminlen(enc); + str3 = str_new0(rb_cString, 0, len1+len2, termlen); ptr3 = RSTRING_PTR(str3); memcpy(ptr3, ptr1, len1); memcpy(ptr3+len1, ptr2, len2); - TERM_FILL(&ptr3[len1+len2], rb_enc_mbminlen(enc)); + TERM_FILL(&ptr3[len1+len2], termlen); FL_SET_RAW(str3, OBJ_TAINTED_RAW(str1) | OBJ_TAINTED_RAW(str2)); ENCODING_CODERANGE_SET(str3, rb_enc_to_index(enc), @@ -1561,15 +1591,32 @@ rb_str_format_m(VALUE str, VALUE arg) } static inline void -str_modifiable(VALUE str) +rb_check_lockedtmp(VALUE str) { if (FL_TEST(str, STR_TMPLOCK)) { rb_raise(rb_eRuntimeError, "can't modify string; temporarily locked"); } +} + +static inline void +str_modifiable(VALUE str) +{ + rb_check_lockedtmp(str); rb_check_frozen(str); } static inline int +str_dependent_p(VALUE str) +{ + if (STR_EMBED_P(str) || !FL_TEST(str, STR_SHARED|STR_NOFREE)) { + return 0; + } + else { + return 1; + } +} + +static inline int str_independent(VALUE str) { str_modifiable(str); @@ -1582,16 +1629,15 @@ str_independent(VALUE str) } static void -str_make_independent_expand(VALUE str, long expand) +str_make_independent_expand(VALUE str, long len, long expand, const int termlen) { char *ptr; - long len = RSTRING_LEN(str); - const int termlen = TERM_LEN(str); + const char *oldptr; long capa = len + expand; if (len > capa) len = capa; - if (capa + termlen - 1 <= RSTRING_EMBED_LEN_MAX && !STR_EMBED_P(str)) { + if (!STR_EMBED_P(str) && capa + termlen <= RSTRING_EMBED_LEN_MAX + 1) { ptr = RSTRING(str)->as.heap.ptr; STR_SET_EMBED(str); memcpy(RSTRING(str)->as.ary, ptr, len); @@ -1600,9 +1646,10 @@ str_make_independent_expand(VALUE str, long expand) return; } - ptr = ALLOC_N(char, capa + termlen); - if (RSTRING_PTR(str)) { - memcpy(ptr, RSTRING_PTR(str), len); + ptr = ALLOC_N(char, (size_t)capa + termlen); + oldptr = RSTRING_PTR(str); + if (oldptr) { + memcpy(ptr, oldptr, len); } STR_SET_NOEMBED(str); FL_UNSET(str, STR_SHARED|STR_NOFREE); @@ -1612,8 +1659,6 @@ str_make_independent_expand(VALUE str, long expand) RSTRING(str)->as.heap.aux.capa = capa; } -#define str_make_independent(str) str_make_independent_expand((str), 0L) - void rb_str_modify(VALUE str) { @@ -1625,16 +1670,17 @@ rb_str_modify(VALUE str) void rb_str_modify_expand(VALUE str, long expand) { + int termlen = TERM_LEN(str); + long len = RSTRING_LEN(str); + if (expand < 0) { rb_raise(rb_eArgError, "negative expanding string size"); } if (!str_independent(str)) { - str_make_independent_expand(str, expand); + str_make_independent_expand(str, len, expand, termlen); } else if (expand > 0) { - long len = RSTRING_LEN(str); long capa = len + expand; - int termlen = TERM_LEN(str); if (expand >= LONG_MAX - len - termlen) { rb_raise(rb_eArgError, "string size too big"); } @@ -1643,7 +1689,7 @@ rb_str_modify_expand(VALUE str, long expand) RSTRING(str)->as.heap.aux.capa = capa; } else if (capa + termlen > RSTRING_EMBED_LEN_MAX + 1) { - str_make_independent_expand(str, expand); + str_make_independent_expand(str, len, expand, termlen); } } ENC_CODERANGE_CLEAR(str); @@ -1721,9 +1767,12 @@ str_null_char(const char *s, long len, const int minlen, rb_encoding *enc) static char * str_fill_term(VALUE str, char *s, long len, int oldtermlen, int termlen) { - long capa = rb_str_capacity(str) + 1; + long capa = str_capacity(str, termlen); - if (capa < len + termlen) { + /* This function assumes that (capa + termlen) bytes of memory + * is allocated, like many other functions in this file. + */ + if (capa < len) { rb_str_modify_expand(str, termlen); } else if (!str_independent(str)) { @@ -1735,6 +1784,34 @@ str_fill_term(VALUE str, char *s, long len, int oldtermlen, int termlen) return s; } +void +rb_str_change_terminator_length(VALUE str, const int oldtermlen, const int termlen) +{ + long capa = str_capacity(str, oldtermlen); + long len = RSTRING_LEN(str); + + if (capa < len + termlen - oldtermlen) { + rb_check_lockedtmp(str); + str_make_independent_expand(str, len, 0L, termlen); + } + else if (str_dependent_p(str)) { + if (termlen > oldtermlen) + str_make_independent_expand(str, len, 0L, termlen); + } + else { + if (!STR_EMBED_P(str)) { + /* modify capa instead of realloc */ + assert(!FL_TEST((str), STR_SHARED)); + RSTRING(str)->as.heap.aux.capa = capa - (termlen - oldtermlen); + } + if (termlen > oldtermlen) { + TERM_FILL(RSTRING_PTR(str) + len, termlen); + } + } + + return; +} + char * rb_string_value_cstr(volatile VALUE *ptr) { @@ -1935,7 +2012,7 @@ rb_str_subseq(VALUE str, long beg, long len) { VALUE str2; - if (RSTRING_EMBED_LEN_MAX < len && SHARABLE_SUBSTRING_P(beg, len, RSTRING_LEN(str))) { + if ((RSTRING_EMBED_LEN_MAX + 1 - TERM_LEN(str)) < len && SHARABLE_SUBSTRING_P(beg, len, RSTRING_LEN(str))) { long olen; str2 = rb_str_new_shared(rb_str_new_frozen(str)); RSTRING(str2)->as.heap.ptr += beg; @@ -2045,7 +2122,7 @@ rb_str_substr(VALUE str, long beg, long len) char *p = rb_str_subpos(str, beg, &len); if (!p) return Qnil; - if (len > RSTRING_EMBED_LEN_MAX && SHARABLE_SUBSTRING_P(p, len, RSTRING_END(str))) { + if (len > (RSTRING_EMBED_LEN_MAX + 1 - TERM_LEN(str)) && SHARABLE_SUBSTRING_P(p, len, RSTRING_END(str))) { long ofs = p - RSTRING_PTR(str); str2 = rb_str_new_frozen(str); str2 = str_new_shared(rb_obj_class(str2), str2); @@ -2141,7 +2218,7 @@ rb_str_resize(VALUE str, long len) TERM_FILL(RSTRING(str)->as.ary + len, termlen); return str; } - str_make_independent_expand(str, len - slen); + str_make_independent_expand(str, slen, len - slen, termlen); } else if (len + termlen <= RSTRING_EMBED_LEN_MAX + 1) { char *ptr = STR_HEAP_PTR(str); @@ -2155,7 +2232,8 @@ rb_str_resize(VALUE str, long len) } else if (!independent) { if (len == slen) return str; - str_make_independent_expand(str, len - slen); + str_make_independent_expand(str, slen, len - slen, termlen); + } else if ((capa = RSTRING(str)->as.heap.aux.capa) < len || (capa - len) > (len < 1024 ? len : 1024)) { @@ -2183,7 +2261,7 @@ str_buf_cat(VALUE str, const char *ptr, long len) rb_str_modify(str); if (len == 0) return 0; if (STR_EMBED_P(str)) { - capa = RSTRING_EMBED_LEN_MAX; + capa = RSTRING_EMBED_LEN_MAX + 1 - termlen; sptr = RSTRING(str)->as.ary; olen = RSTRING_EMBED_LEN(str); } @@ -3721,7 +3799,7 @@ rb_str_drop_bytes(VALUE str, long len) str_modifiable(str); if (len > olen) len = olen; nlen = olen - len; - if (nlen <= RSTRING_EMBED_LEN_MAX) { + if (nlen <= (RSTRING_EMBED_LEN_MAX + 1 - TERM_LEN(str))) { char *oldptr = ptr; int fl = (int)(RBASIC(str)->flags & (STR_NOEMBED|STR_SHARED|STR_NOFREE)); STR_SET_EMBED(str); @@ -4594,7 +4672,7 @@ str_byte_substr(VALUE str, long beg, long len) else p = s + beg; - if (len > RSTRING_EMBED_LEN_MAX && SHARABLE_SUBSTRING_P(beg, len, n)) { + if (len > (RSTRING_EMBED_LEN_MAX + 1 - TERM_LEN(str)) && SHARABLE_SUBSTRING_P(beg, len, n)) { str2 = rb_str_new_frozen(str); str2 = str_new_shared(rb_obj_class(str2), str2); RSTRING(str2)->as.heap.ptr += beg; @@ -7801,9 +7879,11 @@ rb_str_justify(int argc, VALUE *argv, VALUE str, char jflag) long n, size, llen, rlen, llen2 = 0, rlen2 = 0; volatile VALUE pad; int singlebyte = 1, cr; + int termlen; rb_scan_args(argc, argv, "11", &w, &pad); enc = STR_ENC_GET(str); + termlen = rb_enc_mbminlen(enc); width = NUM2LONG(w); if (argc == 2) { StringValue(pad); @@ -7833,7 +7913,7 @@ rb_str_justify(int argc, VALUE *argv, VALUE str, char jflag) rb_raise(rb_eArgError, "argument too big"); } len += size; - res = rb_str_new_with_class(str, 0, len); + res = str_new0(rb_obj_class(str), 0, len, termlen); p = RSTRING_PTR(res); if (flen <= 1) { memset(p, *f, llen); @@ -7867,7 +7947,7 @@ rb_str_justify(int argc, VALUE *argv, VALUE str, char jflag) p += rlen2; } } - TERM_FILL(p, rb_enc_mbminlen(enc)); + TERM_FILL(p, termlen); STR_SET_LEN(res, p-RSTRING_PTR(res)); OBJ_INFECT_RAW(res, str); if (!NIL_P(pad)) OBJ_INFECT_RAW(res, pad); @@ -1,6 +1,6 @@ #define RUBY_VERSION "2.2.6" #define RUBY_RELEASE_DATE "2016-10-01" -#define RUBY_PATCHLEVEL 370 +#define RUBY_PATCHLEVEL 371 #define RUBY_RELEASE_YEAR 2016 #define RUBY_RELEASE_MONTH 10 |