Backport #9498 to Ruby 3.3 (#9805)

Handle mmap failures for redblack tree cache

The redblack tree cache is totally optional, so if we can't allocate
room for the cache, then just pretend as if the cache is full if mmap
fails

Co-authored-by: Aaron Patterson <tenderlove@ruby-lang.org>

1 files changed, 8 insertions, 0 deletions

diff --git a/shape.c b/shape.c
index 4cd4acd7f9..8d8314db33 100644
--- a/shape.c
+++ b/shape.c
@@ -1233,6 +1233,14 @@ Init_default_shapes(void)
     rb_shape_tree_ptr->shape_cache = (redblack_node_t *)mmap(NULL, rb_size_mul_or_raise(REDBLACK_CACHE_SIZE, sizeof(redblack_node_t), rb_eRuntimeError),
                          PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
     rb_shape_tree_ptr->cache_size = 0;
+
+    // If mmap fails, then give up on the redblack tree cache.
+    // We set the cache size such that the redblack node allocators think
+    // the cache is full.
+    if (GET_SHAPE_TREE()->shape_cache == MAP_FAILED) {
+        GET_SHAPE_TREE()->shape_cache = 0;
+        GET_SHAPE_TREE()->cache_size = REDBLACK_CACHE_SIZE;
+    }
 #endif
 
     // Shapes by size pool