Handle small pools in shape capacity calculation

When pool slot sizes can be smaller than sizeof(struct RBasic) (e.g. a
32-byte pool on 64-bit where RBasic is 16 bytes), the capacity
calculation would underflow. Guard against this by setting capacity to
0 for pools too small to hold fields.

2 files changed, 11 insertions, 4 deletions

diff --git a/shape.c b/shape.c
index 90036722f1..93ccd3eb59 100644
--- a/shape.c
+++ b/shape.c
@@ -477,14 +477,14 @@ static attr_index_t
 shape_grow_capa(attr_index_t current_capa)
 {
     const attr_index_t *capacities = rb_shape_tree.capacities;
+    size_t heaps_count = rb_shape_tree.heaps_count;
 
     // First try to use the next size that will be embeddable in a larger object slot.
-    attr_index_t capa;
-    while ((capa = *capacities)) {
+    for (size_t i = 0; i < heaps_count; i++) {
+        attr_index_t capa = capacities[i];
         if (capa > current_capa) {
             return capa;
         }
-        capacities++;
     }
 
     return (attr_index_t)rb_malloc_grow_capa(current_capa, sizeof(VALUE));
@@ -1543,8 +1543,14 @@ Init_default_shapes(void)
     capacities[heaps_count] = 0;
     size_t index;
     for (index = 0; index < heaps_count; index++) {
-        capacities[index] = (heap_sizes[index] - sizeof(struct RBasic)) / sizeof(VALUE);
+        if (heap_sizes[index] > sizeof(struct RBasic)) {
+            capacities[index] = (heap_sizes[index] - sizeof(struct RBasic)) / sizeof(VALUE);
+        }
+        else {
+            capacities[index] = 0;
+        }
     }
+    rb_shape_tree.heaps_count = heaps_count;
     rb_shape_tree.capacities = capacities;
 
 #ifdef HAVE_MMAP
diff --git a/shape.h b/shape.h
index ad9b148247..09487006bc 100644
--- a/shape.h
+++ b/shape.h
@@ -115,6 +115,7 @@ typedef struct {
     rb_shape_t *shape_list;
     rb_shape_t *root_shape;
     const attr_index_t *capacities;
+    size_t heaps_count;
     rb_atomic_t next_shape_id;
 
     redblack_node_t *shape_cache;