summaryrefslogtreecommitdiff
path: root/shape.h
diff options
context:
space:
mode:
authorPeter Zhu <peter@peterzhu.ca>2023-12-04 09:26:26 -0500
committerPeter Zhu <peter@peterzhu.ca>2023-12-04 13:57:12 -0500
commit71babe5536bdb2238509752d8706194ee57ff485 (patch)
tree6e0d057e5b43029288a8f2f40617b4fa2aec26c0 /shape.h
parentd35aa58b2fac5ce0172c3a940a9b5f33af21bc56 (diff)
Use 32-bit integers for redblack_id_t
On 32-bit systems, the shape cache size is 1048576 (value of REDBLACK_CACHE_SIZE), but a 16-bit unsigned integer can only go up to 65536. This means that the redblack_id_t can overflow and lead to a corrupted red-black tree. The following script crashes on 32-bit systems: o = Object.new 1_000_000.times do |i| o.instance_variable_set(:"@i#{i}", i) end
Diffstat (limited to 'shape.h')
-rw-r--r--shape.h4
1 files changed, 2 insertions, 2 deletions
diff --git a/shape.h b/shape.h
index a222a75ad8..4fc46b4525 100644
--- a/shape.h
+++ b/shape.h
@@ -9,7 +9,6 @@
#define SHAPE_IN_BASIC_FLAGS 1
typedef uint32_t attr_index_t;
typedef uint32_t shape_id_t;
-typedef uint32_t redblack_id_t;
# define SHAPE_ID_NUM_BITS 32
#else
@@ -18,11 +17,12 @@ typedef uint32_t redblack_id_t;
#define SHAPE_IN_BASIC_FLAGS 0
typedef uint16_t attr_index_t;
typedef uint16_t shape_id_t;
-typedef uint16_t redblack_id_t;
# define SHAPE_ID_NUM_BITS 16
#endif
+typedef uint32_t redblack_id_t;
+
#define MAX_IVARS (attr_index_t)(-1)
# define SHAPE_MASK (((uintptr_t)1 << SHAPE_ID_NUM_BITS) - 1)