sprintf.c: overflow check

* sprintf.c (ruby_do_vsnprintf): pathologically, get rid of
  negative value when the result length exceeds INT_MAX.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@62328 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

1 files changed, 6 insertions, 3 deletions

diff --git a/sprintf.c b/sprintf.c
index 71802a4907..e91d1f7e88 100644
--- a/sprintf.c
+++ b/sprintf.c
@@ -1275,7 +1275,7 @@ ruby_vsnprintf(char *str, size_t n, const char *fmt, va_list ap)
 static int
 ruby_do_vsnprintf(char *str, size_t n, const char *fmt, va_list ap)
 {
-    int ret;
+    ssize_t ret;
     rb_printf_buffer f;
 
     f._flags = __SWR | __SSTR;
@@ -1283,9 +1283,12 @@ ruby_do_vsnprintf(char *str, size_t n, const char *fmt, va_list ap)
     f._bf._size = f._w = str ? (n - 1) : 0;
     f.vwrite = BSD__sfvwrite;
     f.vextra = 0;
-    ret = (int)BSD_vfprintf(&f, fmt, ap);
+    ret = BSD_vfprintf(&f, fmt, ap);
     if (str) *f._p = 0;
-    return ret;
+#if SIZEOF_SIZE_T > SIZEOF_INT
+    if (n > INT_MAX) return INT_MAX;
+#endif
+    return (int)ret;
 }
 
 int