merge revision(s) 55547,55551,55552,55555,55557,55559,55568,55575,55691: [Backport #12536]

	* string.c: Fix memory corruptions when using UTF-16/32 strings.
	  [Bug #12536] [ruby-dev:49699]

	* string.c (TERM_LEN_MAX): Macro for the longest TERM_FILL length,
	  the same as largest value of rb_enc_mbminlen(enc) among encodings.

	* string.c (str_new, rb_str_buf_new, str_shared_replace): Allocate
	  +TERM_LEN_MAX bytes instead of +1. This change may increase memory
	  usage.

	* string.c (rb_str_new_with_class): Use TERM_LEN of the "obj".

	* string.c (rb_str_plus, rb_str_justify): Use str_new0 which is aware
	  of termlen.

	* string.c (str_shared_replace): Copy +termlen bytes instead of +1.

	* string.c (rb_str_times): termlen should not be included in capa.

	* string.c (RESIZE_CAPA_TERM): When using RSTRING_EMBED_LEN_MAX,
	  termlen should be counted with it because embedded strings are
	  also processed by TERM_FILL.

	* string.c (rb_str_capacity, str_shared_replace, str_buf_cat): ditto.

	* string.c (rb_str_drop_bytes, rb_str_setbyte, str_byte_substr): ditto.

	* string.c (rb_str_subseq, str_substr): When RSTRING_EMBED_LEN_MAX
	  is used, TERM_LEN(str) should be considered with it because
	  embedded strings are also processed by TERM_FILL.
	  Additional fix for [Bug #12536] [ruby-dev:49699].
	  Additional fix for [Bug #12536] [ruby-dev:49699].

	* string.c (rb_usascii_str_new, rb_utf8_str_new): Specify termlen
	  which is apparently 1 for the encodings.

	* string.c (str_new0_cstr): New static function to create a String
	  object from a C string with specifying termlen.

	* string.c (rb_usascii_str_new_cstr, rb_utf8_str_new_cstr): Specify
	  termlen by using new str_new0_cstr().

	* string.c (str_new_static): Specify termlen from the given encoding
	  when creating a new String object is needed.

	* string.c (rb_tainted_str_new_with_enc): New function to create a
	  tainted String object with the given encoding. This means that
	  the termlen is correctly specified. Curretly static function.
	  The function name might be renamed to rb_tainted_enc_str_new
	  or rb_enc_tainted_str_new.

	* string.c (rb_external_str_new_with_enc): Use encoding by using the
	  above rb_tainted_str_new_with_enc().

	* string.c (str_fill_term): When termlen increases, re-allocation
	  of memory for termlen should always be needed.
	  In this fix, if possible, decrease capa instead of realloc.
	  [Bug #12536] [ruby-dev:49699]

	* string.c: Partially reverts r55547 and r55555.
	  ChangeLog about the reverted changes are also deleted in this file.
	  [Bug #12536] [ruby-dev:49699] [ruby-dev:49702]

	* string.c: Specify termlen as far as possible.
	  the termlen is correctly specified. Currently static function.

	* string.c (rb_str_change_terminator_length): New function to change
	  termlen and resize heap for the terminator. This is split from
	  rb_str_fill_terminator (str_fill_term) because filling terminator
	  and changing terminator length are different things. [Bug #12536]

	* internal.h: declaration for rb_str_change_terminator_length.

	* string.c (str_fill_term): Simplify only to zero-fill the terminator.
	  For non-shared strings, it assumes that (capa + termlen) bytes of
	  heap is allocated. This partially reverts r55557.

	* encoding.c (rb_enc_associate_index): rb_str_change_terminator_length
	  is used, and it should be called whenever the termlen is changed.

	* string.c (str_capacity): New static function to return capacity
	  of a string with the given termlen, because the termlen may
	  sometimes be different from TERM_LEN(str) especially during
	  changing termlen or filling terminator with specific termlen.

	* string.c (rb_str_capacity): Use str_capacity.

	* string.c (str_buf_cat): Fix capa size for embed string.
	  Fix bug in r55547. [Bug #12536]


git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/branches/ruby_2_2@56301 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

1 files changed, 117 insertions, 37 deletions

diff --git a/string.c b/string.c
index afdb4db92e..d96c57c3a5 100644
--- a/string.c
+++ b/string.c
@@ -102,7 +102,7 @@ VALUE rb_cSymbol;
 } while (0)
 #define RESIZE_CAPA_TERM(str,capacity,termlen) do {\
     if (STR_EMBED_P(str)) {\
-	if ((capacity) > RSTRING_EMBED_LEN_MAX) {\
+	if ((capacity) > (RSTRING_EMBED_LEN_MAX + 1 - (termlen))) {\
 	    char *const tmp = ALLOC_N(char, (capacity)+termlen);\
 	    const long tlen = RSTRING_LEN(str);\
 	    memcpy(tmp, RSTRING_PTR(str), tlen);\
@@ -138,6 +138,16 @@ VALUE rb_cSymbol;
 #define SHARABLE_SUBSTRING_P(beg, len, end) 1
 #endif
 
+static void str_make_independent_expand(VALUE str, long len, long expand, const int termlen);
+
+static inline void
+str_make_independent(VALUE str)
+{
+    long len = RSTRING_LEN(str);
+    int termlen = TERM_LEN(str);
+    str_make_independent_expand((str), len, 0L, termlen);
+}
+
 static rb_encoding *
 get_actual_encoding(const int encidx, VALUE str)
 {
@@ -524,11 +534,11 @@ str_mod_check(VALUE s, const char *p, long len)
     }
 }
 
-size_t
-rb_str_capacity(VALUE str)
+static size_t
+str_capacity(VALUE str, const int termlen)
 {
     if (STR_EMBED_P(str)) {
-	return RSTRING_EMBED_LEN_MAX;
+	return (RSTRING_EMBED_LEN_MAX + 1 - termlen);
     }
     else if (FL_TEST(str, STR_SHARED|STR_NOFREE)) {
 	return RSTRING(str)->as.heap.len;
@@ -538,6 +548,12 @@ rb_str_capacity(VALUE str)
     }
 }
 
+size_t
+rb_str_capacity(VALUE str)
+{
+    return str_capacity(str, TERM_LEN(str));
+}
+
 static inline void
 must_not_null(const char *ptr)
 {
@@ -576,7 +592,7 @@ str_new0(VALUE klass, const char *ptr, long len, int termlen)
     }
 
     str = str_alloc(klass);
-    if (len > RSTRING_EMBED_LEN_MAX) {
+    if (len > (RSTRING_EMBED_LEN_MAX + 1 - termlen)) {
 	RSTRING(str)->as.heap.aux.capa = len;
 	RSTRING(str)->as.heap.ptr = ALLOC_N(char, len + termlen);
 	STR_SET_NOEMBED(str);
@@ -675,7 +691,8 @@ str_new_static(VALUE klass, const char *ptr, long len, int encindex)
     }
 
     if (!ptr) {
-	str = str_new(klass, ptr, len);
+	rb_encoding *enc = rb_enc_get_from_index(encindex);
+	str = str_new0(klass, ptr, len, rb_enc_mbminlen(enc));
     }
     else {
 	if (RUBY_DTRACE_STRING_CREATE_ENABLED()) {
@@ -725,6 +742,15 @@ rb_tainted_str_new(const char *ptr, long len)
     return str;
 }
 
+static VALUE
+rb_tainted_str_new_with_enc(const char *ptr, long len, rb_encoding *enc)
+{
+    VALUE str = rb_enc_str_new(ptr, len, enc);
+
+    OBJ_TAINT(str);
+    return str;
+}
+
 VALUE
 rb_tainted_str_new_cstr(const char *ptr)
 {
@@ -817,7 +843,7 @@ rb_external_str_new_with_enc(const char *ptr, long len, rb_encoding *eenc)
 {
     VALUE str;
 
-    str = rb_tainted_str_new(ptr, len);
+    str = rb_tainted_str_new_with_enc(ptr, len, eenc);
     return rb_external_str_with_enc(str, eenc);
 }
 
@@ -980,7 +1006,7 @@ rb_str_new_frozen(VALUE orig)
 VALUE
 rb_str_new_with_class(VALUE obj, const char *ptr, long len)
 {
-    return str_new(rb_obj_class(obj), ptr, len);
+    return str_new0(rb_obj_class(obj), ptr, len, TERM_LEN(obj));
 }
 
 static VALUE
@@ -1081,15 +1107,17 @@ rb_str_shared_replace(VALUE str, VALUE str2)
 {
     rb_encoding *enc;
     int cr;
+    int termlen;
     if (str == str2) return;
     enc = STR_ENC_GET(str2);
     cr = ENC_CODERANGE(str2);
     str_discard(str);
     OBJ_INFECT(str, str2);
+    termlen = rb_enc_mbminlen(enc);
 
-    if (RSTRING_LEN(str2) <= RSTRING_EMBED_LEN_MAX) {
+    if (RSTRING_LEN(str2) <= (RSTRING_EMBED_LEN_MAX + 1 - termlen)) {
 	STR_SET_EMBED(str);
-	memcpy(RSTRING_PTR(str), RSTRING_PTR(str2), RSTRING_LEN(str2)+1);
+	memcpy(RSTRING_PTR(str), RSTRING_PTR(str2), RSTRING_LEN(str2)+termlen);
 	STR_SET_EMBED_LEN(str, RSTRING_LEN(str2));
         rb_enc_associate(str, enc);
         ENC_CODERANGE_SET(str, cr);
@@ -1465,16 +1493,18 @@ rb_str_plus(VALUE str1, VALUE str2)
     rb_encoding *enc;
     char *ptr1, *ptr2, *ptr3;
     long len1, len2;
+    int termlen;
 
     StringValue(str2);
     enc = rb_enc_check(str1, str2);
     RSTRING_GETMEM(str1, ptr1, len1);
     RSTRING_GETMEM(str2, ptr2, len2);
-    str3 = rb_str_new(0, len1+len2);
+    termlen = rb_enc_mbminlen(enc);
+    str3 = str_new0(rb_cString, 0, len1+len2, termlen);
     ptr3 = RSTRING_PTR(str3);
     memcpy(ptr3, ptr1, len1);
     memcpy(ptr3+len1, ptr2, len2);
-    TERM_FILL(&ptr3[len1+len2], rb_enc_mbminlen(enc));
+    TERM_FILL(&ptr3[len1+len2], termlen);
 
     FL_SET_RAW(str3, OBJ_TAINTED_RAW(str1) | OBJ_TAINTED_RAW(str2));
     ENCODING_CODERANGE_SET(str3, rb_enc_to_index(enc),
@@ -1561,15 +1591,32 @@ rb_str_format_m(VALUE str, VALUE arg)
 }
 
 static inline void
-str_modifiable(VALUE str)
+rb_check_lockedtmp(VALUE str)
 {
     if (FL_TEST(str, STR_TMPLOCK)) {
 	rb_raise(rb_eRuntimeError, "can't modify string; temporarily locked");
     }
+}
+
+static inline void
+str_modifiable(VALUE str)
+{
+    rb_check_lockedtmp(str);
     rb_check_frozen(str);
 }
 
 static inline int
+str_dependent_p(VALUE str)
+{
+    if (STR_EMBED_P(str) || !FL_TEST(str, STR_SHARED|STR_NOFREE)) {
+	return 0;
+    }
+    else {
+	return 1;
+    }
+}
+
+static inline int
 str_independent(VALUE str)
 {
     str_modifiable(str);
@@ -1582,16 +1629,15 @@ str_independent(VALUE str)
 }
 
 static void
-str_make_independent_expand(VALUE str, long expand)
+str_make_independent_expand(VALUE str, long len, long expand, const int termlen)
 {
     char *ptr;
-    long len = RSTRING_LEN(str);
-    const int termlen = TERM_LEN(str);
+    const char *oldptr;
     long capa = len + expand;
 
     if (len > capa) len = capa;
 
-    if (capa + termlen - 1 <= RSTRING_EMBED_LEN_MAX && !STR_EMBED_P(str)) {
+    if (!STR_EMBED_P(str) && capa + termlen <= RSTRING_EMBED_LEN_MAX + 1) {
 	ptr = RSTRING(str)->as.heap.ptr;
 	STR_SET_EMBED(str);
 	memcpy(RSTRING(str)->as.ary, ptr, len);
@@ -1600,9 +1646,10 @@ str_make_independent_expand(VALUE str, long expand)
 	return;
     }
 
-    ptr = ALLOC_N(char, capa + termlen);
-    if (RSTRING_PTR(str)) {
-	memcpy(ptr, RSTRING_PTR(str), len);
+    ptr = ALLOC_N(char, (size_t)capa + termlen);
+    oldptr = RSTRING_PTR(str);
+    if (oldptr) {
+	memcpy(ptr, oldptr, len);
     }
     STR_SET_NOEMBED(str);
     FL_UNSET(str, STR_SHARED|STR_NOFREE);
@@ -1612,8 +1659,6 @@ str_make_independent_expand(VALUE str, long expand)
     RSTRING(str)->as.heap.aux.capa = capa;
 }
 
-#define str_make_independent(str) str_make_independent_expand((str), 0L)
-
 void
 rb_str_modify(VALUE str)
 {
@@ -1625,16 +1670,17 @@ rb_str_modify(VALUE str)
 void
 rb_str_modify_expand(VALUE str, long expand)
 {
+    int termlen = TERM_LEN(str);
+    long len = RSTRING_LEN(str);
+
     if (expand < 0) {
 	rb_raise(rb_eArgError, "negative expanding string size");
     }
     if (!str_independent(str)) {
-	str_make_independent_expand(str, expand);
+	str_make_independent_expand(str, len, expand, termlen);
     }
     else if (expand > 0) {
-	long len = RSTRING_LEN(str);
 	long capa = len + expand;
-	int termlen = TERM_LEN(str);
 	if (expand >= LONG_MAX - len - termlen) {
 	    rb_raise(rb_eArgError, "string size too big");
 	}
@@ -1643,7 +1689,7 @@ rb_str_modify_expand(VALUE str, long expand)
 	    RSTRING(str)->as.heap.aux.capa = capa;
 	}
 	else if (capa + termlen > RSTRING_EMBED_LEN_MAX + 1) {
-	    str_make_independent_expand(str, expand);
+	    str_make_independent_expand(str, len, expand, termlen);
 	}
     }
     ENC_CODERANGE_CLEAR(str);
@@ -1721,9 +1767,12 @@ str_null_char(const char *s, long len, const int minlen, rb_encoding *enc)
 static char *
 str_fill_term(VALUE str, char *s, long len, int oldtermlen, int termlen)
 {
-    long capa = rb_str_capacity(str) + 1;
+    long capa = str_capacity(str, termlen);
 
-    if (capa < len + termlen) {
+    /* This function assumes that (capa + termlen) bytes of memory
+     * is allocated, like many other functions in this file.
+     */
+    if (capa < len) {
 	rb_str_modify_expand(str, termlen);
     }
     else if (!str_independent(str)) {
@@ -1735,6 +1784,34 @@ str_fill_term(VALUE str, char *s, long len, int oldtermlen, int termlen)
     return s;
 }
 
+void
+rb_str_change_terminator_length(VALUE str, const int oldtermlen, const int termlen)
+{
+    long capa = str_capacity(str, oldtermlen);
+    long len = RSTRING_LEN(str);
+
+    if (capa < len + termlen - oldtermlen) {
+	rb_check_lockedtmp(str);
+	str_make_independent_expand(str, len, 0L, termlen);
+    }
+    else if (str_dependent_p(str)) {
+	if (termlen > oldtermlen)
+	    str_make_independent_expand(str, len, 0L, termlen);
+    }
+    else {
+	if (!STR_EMBED_P(str)) {
+	    /* modify capa instead of realloc */
+	    assert(!FL_TEST((str), STR_SHARED));
+	    RSTRING(str)->as.heap.aux.capa = capa - (termlen - oldtermlen);
+	}
+	if (termlen > oldtermlen) {
+	    TERM_FILL(RSTRING_PTR(str) + len, termlen);
+	}
+    }
+
+    return;
+}
+
 char *
 rb_string_value_cstr(volatile VALUE *ptr)
 {
@@ -1935,7 +2012,7 @@ rb_str_subseq(VALUE str, long beg, long len)
 {
     VALUE str2;
 
-    if (RSTRING_EMBED_LEN_MAX < len && SHARABLE_SUBSTRING_P(beg, len, RSTRING_LEN(str))) {
+    if ((RSTRING_EMBED_LEN_MAX + 1 - TERM_LEN(str)) < len && SHARABLE_SUBSTRING_P(beg, len, RSTRING_LEN(str))) {
 	long olen;
 	str2 = rb_str_new_shared(rb_str_new_frozen(str));
 	RSTRING(str2)->as.heap.ptr += beg;
@@ -2045,7 +2122,7 @@ rb_str_substr(VALUE str, long beg, long len)
     char *p = rb_str_subpos(str, beg, &len);
 
     if (!p) return Qnil;
-    if (len > RSTRING_EMBED_LEN_MAX && SHARABLE_SUBSTRING_P(p, len, RSTRING_END(str))) {
+    if (len > (RSTRING_EMBED_LEN_MAX + 1 - TERM_LEN(str)) && SHARABLE_SUBSTRING_P(p, len, RSTRING_END(str))) {
 	long ofs = p - RSTRING_PTR(str);
 	str2 = rb_str_new_frozen(str);
 	str2 = str_new_shared(rb_obj_class(str2), str2);
@@ -2141,7 +2218,7 @@ rb_str_resize(VALUE str, long len)
 		TERM_FILL(RSTRING(str)->as.ary + len, termlen);
 		return str;
 	    }
-	    str_make_independent_expand(str, len - slen);
+	    str_make_independent_expand(str, slen, len - slen, termlen);
 	}
 	else if (len + termlen <= RSTRING_EMBED_LEN_MAX + 1) {
 	    char *ptr = STR_HEAP_PTR(str);
@@ -2155,7 +2232,8 @@ rb_str_resize(VALUE str, long len)
 	}
 	else if (!independent) {
 	    if (len == slen) return str;
-	    str_make_independent_expand(str, len - slen);
+	    str_make_independent_expand(str, slen, len - slen, termlen);
+
 	}
 	else if ((capa = RSTRING(str)->as.heap.aux.capa) < len ||
 		 (capa - len) > (len < 1024 ? len : 1024)) {
@@ -2183,7 +2261,7 @@ str_buf_cat(VALUE str, const char *ptr, long len)
     rb_str_modify(str);
     if (len == 0) return 0;
     if (STR_EMBED_P(str)) {
-	capa = RSTRING_EMBED_LEN_MAX;
+	capa = RSTRING_EMBED_LEN_MAX + 1 - termlen;
 	sptr = RSTRING(str)->as.ary;
 	olen = RSTRING_EMBED_LEN(str);
     }
@@ -3721,7 +3799,7 @@ rb_str_drop_bytes(VALUE str, long len)
     str_modifiable(str);
     if (len > olen) len = olen;
     nlen = olen - len;
-    if (nlen <= RSTRING_EMBED_LEN_MAX) {
+    if (nlen <= (RSTRING_EMBED_LEN_MAX + 1 - TERM_LEN(str))) {
 	char *oldptr = ptr;
 	int fl = (int)(RBASIC(str)->flags & (STR_NOEMBED|STR_SHARED|STR_NOFREE));
 	STR_SET_EMBED(str);
@@ -4594,7 +4672,7 @@ str_byte_substr(VALUE str, long beg, long len)
     else
 	p = s + beg;
 
-    if (len > RSTRING_EMBED_LEN_MAX && SHARABLE_SUBSTRING_P(beg, len, n)) {
+    if (len > (RSTRING_EMBED_LEN_MAX + 1 - TERM_LEN(str)) && SHARABLE_SUBSTRING_P(beg, len, n)) {
 	str2 = rb_str_new_frozen(str);
 	str2 = str_new_shared(rb_obj_class(str2), str2);
 	RSTRING(str2)->as.heap.ptr += beg;
@@ -7801,9 +7879,11 @@ rb_str_justify(int argc, VALUE *argv, VALUE str, char jflag)
     long n, size, llen, rlen, llen2 = 0, rlen2 = 0;
     volatile VALUE pad;
     int singlebyte = 1, cr;
+    int termlen;
 
     rb_scan_args(argc, argv, "11", &w, &pad);
     enc = STR_ENC_GET(str);
+    termlen = rb_enc_mbminlen(enc);
     width = NUM2LONG(w);
     if (argc == 2) {
 	StringValue(pad);
@@ -7833,7 +7913,7 @@ rb_str_justify(int argc, VALUE *argv, VALUE str, char jflag)
        rb_raise(rb_eArgError, "argument too big");
     }
     len += size;
-    res = rb_str_new_with_class(str, 0, len);
+    res = str_new0(rb_obj_class(str), 0, len, termlen);
     p = RSTRING_PTR(res);
     if (flen <= 1) {
        memset(p, *f, llen);
@@ -7867,7 +7947,7 @@ rb_str_justify(int argc, VALUE *argv, VALUE str, char jflag)
            p += rlen2;
 	}
     }
-    TERM_FILL(p, rb_enc_mbminlen(enc));
+    TERM_FILL(p, termlen);
     STR_SET_LEN(res, p-RSTRING_PTR(res));
     OBJ_INFECT_RAW(res, str);
     if (!NIL_P(pad)) OBJ_INFECT_RAW(res, pad);