Use VWA for bignum

Previously we only allocated bignums from the 40 byte sizepool, and
embedded bignum used a fixed size.

3 files changed, 77 insertions, 22 deletions

diff --git a/bignum.c b/bignum.c
index 054b6c1cc9..9e8d6a91f1 100644
--- a/bignum.c
+++ b/bignum.c
@@ -79,7 +79,6 @@ STATIC_ASSERT(sizeof_bdigit_and_dbl, SIZEOF_BDIGIT*2 <= SIZEOF_BDIGIT_DBL);
 STATIC_ASSERT(bdigit_signedness, 0 < (BDIGIT)-1);
 STATIC_ASSERT(bdigit_dbl_signedness, 0 < (BDIGIT_DBL)-1);
 STATIC_ASSERT(bdigit_dbl_signed_signedness, 0 > (BDIGIT_DBL_SIGNED)-1);
-STATIC_ASSERT(rbignum_embed_len_max, BIGNUM_EMBED_LEN_MAX <= (BIGNUM_EMBED_LEN_MASK >> BIGNUM_EMBED_LEN_SHIFT));
 
 #if SIZEOF_BDIGIT < SIZEOF_LONG
 STATIC_ASSERT(sizeof_long_and_sizeof_bdigit, SIZEOF_LONG % SIZEOF_BDIGIT == 0);
@@ -2990,25 +2989,56 @@ rb_cmpint(VALUE val, VALUE a, VALUE b)
             ((l) << BIGNUM_EMBED_LEN_SHIFT)) : \
      (void)(RBIGNUM(b)->as.heap.len = (l)))
 
+static size_t
+big_embed_capa(VALUE big)
+{
+    size_t size = rb_gc_obj_slot_size(big) - offsetof(struct RBignum, as.ary);
+    RUBY_ASSERT(size % sizeof(BDIGIT) == 0);
+    size_t capa = size / sizeof(BDIGIT);
+    RUBY_ASSERT(capa <= BIGNUM_EMBED_LEN_MAX);
+    return capa;
+}
+
+static size_t
+big_embed_size(size_t capa)
+{
+    size_t size = offsetof(struct RBignum, as.ary) + (sizeof(BDIGIT) * capa);
+    if (size < sizeof(struct RBignum)) {
+        size = sizeof(struct RBignum);
+    }
+    return size;
+}
+
+static bool
+big_embeddable_p(size_t capa)
+{
+    if (capa > BIGNUM_EMBED_LEN_MAX) {
+        return false;
+    }
+    return rb_gc_size_allocatable_p(big_embed_size(capa));
+}
+
 static void
 rb_big_realloc(VALUE big, size_t len)
 {
     BDIGIT *ds;
+    size_t embed_capa = big_embed_capa(big);
+
     if (BIGNUM_EMBED_P(big)) {
-        if (BIGNUM_EMBED_LEN_MAX < len) {
+        if (embed_capa < len) {
             ds = ALLOC_N(BDIGIT, len);
-            MEMCPY(ds, RBIGNUM(big)->as.ary, BDIGIT, BIGNUM_EMBED_LEN_MAX);
+            MEMCPY(ds, RBIGNUM(big)->as.ary, BDIGIT, embed_capa);
             RBIGNUM(big)->as.heap.len = BIGNUM_LEN(big);
             RBIGNUM(big)->as.heap.digits = ds;
             FL_UNSET_RAW(big, BIGNUM_EMBED_FLAG);
         }
     }
     else {
-        if (len <= BIGNUM_EMBED_LEN_MAX) {
+        if (len <= embed_capa) {
             ds = RBIGNUM(big)->as.heap.digits;
             FL_SET_RAW(big, BIGNUM_EMBED_FLAG);
             BIGNUM_SET_LEN(big, len);
-            (void)VALGRIND_MAKE_MEM_UNDEFINED((void*)RBIGNUM(big)->as.ary, sizeof(RBIGNUM(big)->as.ary));
+            (void)VALGRIND_MAKE_MEM_UNDEFINED((void*)RBIGNUM(big)->as.ary, embed_capa * sizeof(BDIGIT));
             if (ds) {
                 MEMCPY(RBIGNUM(big)->as.ary, ds, BDIGIT, len);
                 xfree(ds);
@@ -3035,16 +3065,24 @@ rb_big_resize(VALUE big, size_t len)
 static VALUE
 bignew_1(VALUE klass, size_t len, int sign)
 {
-    NEWOBJ_OF(big, struct RBignum, klass,
-            T_BIGNUM | (RGENGC_WB_PROTECTED_BIGNUM ? FL_WB_PROTECTED : 0), sizeof(struct RBignum), 0);
-    VALUE bigv = (VALUE)big;
-    BIGNUM_SET_SIGN(bigv, sign);
-    if (len <= BIGNUM_EMBED_LEN_MAX) {
-        FL_SET_RAW(bigv, BIGNUM_EMBED_FLAG);
+    VALUE bigv;
+
+    if (big_embeddable_p(len)) {
+        size_t size = big_embed_size(len);
+        RUBY_ASSERT(rb_gc_size_allocatable_p(size));
+        NEWOBJ_OF(big, struct RBignum, klass,
+                T_BIGNUM | BIGNUM_EMBED_FLAG | (RGENGC_WB_PROTECTED_BIGNUM ? FL_WB_PROTECTED : 0),
+                size, 0);
+        bigv = (VALUE)big;
+        BIGNUM_SET_SIGN(bigv, sign);
         BIGNUM_SET_LEN(bigv, len);
-        (void)VALGRIND_MAKE_MEM_UNDEFINED((void*)big->as.ary, sizeof(big->as.ary));
+        (void)VALGRIND_MAKE_MEM_UNDEFINED((void*)big->as.ary, len * sizeof(BDIGIT));
     }
     else {
+        NEWOBJ_OF(big, struct RBignum, klass,
+                T_BIGNUM | (RGENGC_WB_PROTECTED_BIGNUM ? FL_WB_PROTECTED : 0), sizeof(struct RBignum), 0);
+        bigv = (VALUE)big;
+        BIGNUM_SET_SIGN(bigv, sign);
         big->as.heap.digits = ALLOC_N(BDIGIT, len);
         big->as.heap.len = len;
     }
diff --git a/ext/-test-/bignum/depend b/ext/-test-/bignum/depend
index 049f0c7b52..82972f1032 100644
--- a/ext/-test-/bignum/depend
+++ b/ext/-test-/bignum/depend
@@ -6,6 +6,7 @@ big2str.o: $(hdrdir)/ruby/backward.h
 big2str.o: $(hdrdir)/ruby/backward/2/assume.h
 big2str.o: $(hdrdir)/ruby/backward/2/attributes.h
 big2str.o: $(hdrdir)/ruby/backward/2/bool.h
+big2str.o: $(hdrdir)/ruby/backward/2/gcc_version_since.h
 big2str.o: $(hdrdir)/ruby/backward/2/inttypes.h
 big2str.o: $(hdrdir)/ruby/backward/2/limits.h
 big2str.o: $(hdrdir)/ruby/backward/2/long_long.h
@@ -159,6 +160,7 @@ big2str.o: $(hdrdir)/ruby/ruby.h
 big2str.o: $(hdrdir)/ruby/st.h
 big2str.o: $(hdrdir)/ruby/subst.h
 big2str.o: $(top_srcdir)/internal/bignum.h
+big2str.o: $(top_srcdir)/internal/compilers.h
 big2str.o: big2str.c
 bigzero.o: $(RUBY_EXTCONF_H)
 bigzero.o: $(arch_hdrdir)/ruby/config.h
@@ -167,6 +169,7 @@ bigzero.o: $(hdrdir)/ruby/backward.h
 bigzero.o: $(hdrdir)/ruby/backward/2/assume.h
 bigzero.o: $(hdrdir)/ruby/backward/2/attributes.h
 bigzero.o: $(hdrdir)/ruby/backward/2/bool.h
+bigzero.o: $(hdrdir)/ruby/backward/2/gcc_version_since.h
 bigzero.o: $(hdrdir)/ruby/backward/2/inttypes.h
 bigzero.o: $(hdrdir)/ruby/backward/2/limits.h
 bigzero.o: $(hdrdir)/ruby/backward/2/long_long.h
@@ -320,6 +323,7 @@ bigzero.o: $(hdrdir)/ruby/ruby.h
 bigzero.o: $(hdrdir)/ruby/st.h
 bigzero.o: $(hdrdir)/ruby/subst.h
 bigzero.o: $(top_srcdir)/internal/bignum.h
+bigzero.o: $(top_srcdir)/internal/compilers.h
 bigzero.o: bigzero.c
 div.o: $(RUBY_EXTCONF_H)
 div.o: $(arch_hdrdir)/ruby/config.h
@@ -328,6 +332,7 @@ div.o: $(hdrdir)/ruby/backward.h
 div.o: $(hdrdir)/ruby/backward/2/assume.h
 div.o: $(hdrdir)/ruby/backward/2/attributes.h
 div.o: $(hdrdir)/ruby/backward/2/bool.h
+div.o: $(hdrdir)/ruby/backward/2/gcc_version_since.h
 div.o: $(hdrdir)/ruby/backward/2/inttypes.h
 div.o: $(hdrdir)/ruby/backward/2/limits.h
 div.o: $(hdrdir)/ruby/backward/2/long_long.h
@@ -481,6 +486,7 @@ div.o: $(hdrdir)/ruby/ruby.h
 div.o: $(hdrdir)/ruby/st.h
 div.o: $(hdrdir)/ruby/subst.h
 div.o: $(top_srcdir)/internal/bignum.h
+div.o: $(top_srcdir)/internal/compilers.h
 div.o: div.c
 init.o: $(RUBY_EXTCONF_H)
 init.o: $(arch_hdrdir)/ruby/config.h
@@ -650,6 +656,7 @@ intpack.o: $(hdrdir)/ruby/backward.h
 intpack.o: $(hdrdir)/ruby/backward/2/assume.h
 intpack.o: $(hdrdir)/ruby/backward/2/attributes.h
 intpack.o: $(hdrdir)/ruby/backward/2/bool.h
+intpack.o: $(hdrdir)/ruby/backward/2/gcc_version_since.h
 intpack.o: $(hdrdir)/ruby/backward/2/inttypes.h
 intpack.o: $(hdrdir)/ruby/backward/2/limits.h
 intpack.o: $(hdrdir)/ruby/backward/2/long_long.h
@@ -803,6 +810,7 @@ intpack.o: $(hdrdir)/ruby/ruby.h
 intpack.o: $(hdrdir)/ruby/st.h
 intpack.o: $(hdrdir)/ruby/subst.h
 intpack.o: $(top_srcdir)/internal/bignum.h
+intpack.o: $(top_srcdir)/internal/compilers.h
 intpack.o: intpack.c
 mul.o: $(RUBY_EXTCONF_H)
 mul.o: $(arch_hdrdir)/ruby/config.h
@@ -811,6 +819,7 @@ mul.o: $(hdrdir)/ruby/backward.h
 mul.o: $(hdrdir)/ruby/backward/2/assume.h
 mul.o: $(hdrdir)/ruby/backward/2/attributes.h
 mul.o: $(hdrdir)/ruby/backward/2/bool.h
+mul.o: $(hdrdir)/ruby/backward/2/gcc_version_since.h
 mul.o: $(hdrdir)/ruby/backward/2/inttypes.h
 mul.o: $(hdrdir)/ruby/backward/2/limits.h
 mul.o: $(hdrdir)/ruby/backward/2/long_long.h
@@ -964,6 +973,7 @@ mul.o: $(hdrdir)/ruby/ruby.h
 mul.o: $(hdrdir)/ruby/st.h
 mul.o: $(hdrdir)/ruby/subst.h
 mul.o: $(top_srcdir)/internal/bignum.h
+mul.o: $(top_srcdir)/internal/compilers.h
 mul.o: mul.c
 str2big.o: $(RUBY_EXTCONF_H)
 str2big.o: $(arch_hdrdir)/ruby/config.h
@@ -972,6 +982,7 @@ str2big.o: $(hdrdir)/ruby/backward.h
 str2big.o: $(hdrdir)/ruby/backward/2/assume.h
 str2big.o: $(hdrdir)/ruby/backward/2/attributes.h
 str2big.o: $(hdrdir)/ruby/backward/2/bool.h
+str2big.o: $(hdrdir)/ruby/backward/2/gcc_version_since.h
 str2big.o: $(hdrdir)/ruby/backward/2/inttypes.h
 str2big.o: $(hdrdir)/ruby/backward/2/limits.h
 str2big.o: $(hdrdir)/ruby/backward/2/long_long.h
@@ -1125,5 +1136,6 @@ str2big.o: $(hdrdir)/ruby/ruby.h
 str2big.o: $(hdrdir)/ruby/st.h
 str2big.o: $(hdrdir)/ruby/subst.h
 str2big.o: $(top_srcdir)/internal/bignum.h
+str2big.o: $(top_srcdir)/internal/compilers.h
 str2big.o: str2big.c
 # AUTOGENERATED DEPENDENCIES END
diff --git a/internal/bignum.h b/internal/bignum.h
index 0ba21a4923..e5b6b42563 100644
--- a/internal/bignum.h
+++ b/internal/bignum.h
@@ -9,6 +9,7 @@
  * @brief      Internal header for Bignums.
  */
 #include "ruby/internal/config.h"      /* for HAVE_LIBGMP */
+#include "internal/compilers.h"        /* for FLEX_ARY_LEN */
 #include <stddef.h>             /* for size_t */
 
 #ifdef HAVE_SYS_TYPES_H
@@ -76,18 +77,17 @@
 #define RBIGNUM(obj) ((struct RBignum *)(obj))
 #define BIGNUM_SIGN_BIT FL_USER1
 #define BIGNUM_EMBED_FLAG ((VALUE)FL_USER2)
-#define BIGNUM_EMBED_LEN_NUMBITS 3
+
+/* This is likely more bits than we need today and will also need adjustment if
+ * we change GC slot sizes.
+ */
+#define BIGNUM_EMBED_LEN_NUMBITS 9
 #define BIGNUM_EMBED_LEN_MASK \
-    (~(~(VALUE)0U << BIGNUM_EMBED_LEN_NUMBITS) << BIGNUM_EMBED_LEN_SHIFT)
+    (RUBY_FL_USER11 | RUBY_FL_USER10 | RUBY_FL_USER9 | RUBY_FL_USER8 | RUBY_FL_USER7 | \
+     RUBY_FL_USER6 | RUBY_FL_USER5 | RUBY_FL_USER4 | RUBY_FL_USER3)
 #define BIGNUM_EMBED_LEN_SHIFT \
     (FL_USHIFT+3) /* bit offset of BIGNUM_EMBED_LEN_MASK */
-#ifndef BIGNUM_EMBED_LEN_MAX
-# if (SIZEOF_VALUE*RBIMPL_RVALUE_EMBED_LEN_MAX/SIZEOF_ACTUAL_BDIGIT) < (1 << BIGNUM_EMBED_LEN_NUMBITS)-1
-#  define BIGNUM_EMBED_LEN_MAX (SIZEOF_VALUE*RBIMPL_RVALUE_EMBED_LEN_MAX/SIZEOF_ACTUAL_BDIGIT)
-# else
-#  define BIGNUM_EMBED_LEN_MAX ((1 << BIGNUM_EMBED_LEN_NUMBITS)-1)
-# endif
-#endif
+#define BIGNUM_EMBED_LEN_MAX (BIGNUM_EMBED_LEN_MASK >> BIGNUM_EMBED_LEN_SHIFT)
 
 enum rb_int_parse_flags {
     RB_INT_PARSE_SIGN       = 0x01,
@@ -104,7 +104,12 @@ struct RBignum {
             size_t len;
             BDIGIT *digits;
         } heap;
-        BDIGIT ary[BIGNUM_EMBED_LEN_MAX];
+        /* This is a length 1 array because:
+         *   1. GCC has a bug that does not optimize C flexible array members
+         *      (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=102452)
+         *   2. Zero length arrays are not supported by all compilers
+         */
+        BDIGIT ary[1];
     } as;
 };