* gc.c: fix a regression by r31690 on AIX because AIX malloc

  return NULL if it's passed 0. But some caller don't expect it.
  patch by Yutaka Kanemoto.
(vm_malloc_prepare): return calculated size.
(vm_xmalloc): use above result.
(vm_xcalloc): ditto.



git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@32103 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

2 files changed, 18 insertions, 5 deletions

diff --git a/ChangeLog b/ChangeLog
index 0991c3e5b6..c3ca595669 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,12 @@
+Wed Jun 15 23:30:45 2011  KOSAKI Motohiro  <kosaki.motohiro@gmail.com>
+
+	* gc.c: fix a regression by r31690 on AIX because AIX malloc
+	  return NULL if it's passed 0. But some caller don't expect it.
+	  patch by Yutaka Kanemoto.
+	(vm_malloc_prepare): return calculated size.
+	(vm_xmalloc): use above result.
+	(vm_xcalloc): ditto.
+
 Wed Jun 15 23:11:35 2011  KOSAKI Motohiro  <kosaki.motohiro@gmail.com>
 
 	* thread.c: remove BLOCKING_REGION_CORE() macro. It's no longer used
diff --git a/gc.c b/gc.c
index 7bf2a123c3..f1fb7568fb 100644
--- a/gc.c
+++ b/gc.c
@@ -711,7 +711,7 @@ garbage_collect_with_gvl(rb_objspace_t *objspace)
 
 static void vm_xfree(rb_objspace_t *objspace, void *ptr);
 
-static inline void
+static inline size_t
 vm_malloc_prepare(rb_objspace_t *objspace, size_t size)
 {
     if ((ssize_t)size < 0) {
@@ -727,6 +727,8 @@ vm_malloc_prepare(rb_objspace_t *objspace, size_t size)
 	(malloc_increase+size) > malloc_limit) {
 	garbage_collect_with_gvl(objspace);
     }
+
+    return size;
 }
 
 static inline void *
@@ -757,7 +759,7 @@ vm_xmalloc(rb_objspace_t *objspace, size_t size)
 {
     void *mem;
 
-    vm_malloc_prepare(objspace, size);
+    size = vm_malloc_prepare(objspace, size);
     TRY_WITH_GC(mem = malloc(size));
     return vm_malloc_fixup(objspace, mem, size);
 }
@@ -844,10 +846,12 @@ static void *
 vm_xcalloc(rb_objspace_t *objspace, size_t count, size_t elsize)
 {
     void *mem;
-    const size_t size = xmalloc2_size(count, elsize);
+    size_t size;
+
+    size = xmalloc2_size(count, elsize);
+    size = vm_malloc_prepare(objspace, size);
 
-    vm_malloc_prepare(objspace, size);
-    TRY_WITH_GC(mem = calloc(count, elsize));
+    TRY_WITH_GC(mem = calloc(1, size));
     return vm_malloc_fixup(objspace, mem, size);
 }