diff options
Diffstat (limited to 'internal')
| -rw-r--r-- | internal/array.h | 2 | ||||
| -rw-r--r-- | internal/basic_operators.h | 1 | ||||
| -rw-r--r-- | internal/bignum.h | 32 | ||||
| -rw-r--r-- | internal/bits.h | 31 | ||||
| -rw-r--r-- | internal/box.h | 83 | ||||
| -rw-r--r-- | internal/class.h | 374 | ||||
| -rw-r--r-- | internal/cmdlineopt.h | 6 | ||||
| -rw-r--r-- | internal/concurrent_set.h | 21 | ||||
| -rw-r--r-- | internal/cont.h | 1 | ||||
| -rw-r--r-- | internal/encoding.h | 3 | ||||
| -rw-r--r-- | internal/error.h | 16 | ||||
| -rw-r--r-- | internal/eval.h | 3 | ||||
| -rw-r--r-- | internal/gc.h | 41 | ||||
| -rw-r--r-- | internal/imemo.h | 120 | ||||
| -rw-r--r-- | internal/inits.h | 4 | ||||
| -rw-r--r-- | internal/io.h | 4 | ||||
| -rw-r--r-- | internal/namespace.h | 78 | ||||
| -rw-r--r-- | internal/numeric.h | 49 | ||||
| -rw-r--r-- | internal/range.h | 6 | ||||
| -rw-r--r-- | internal/re.h | 5 | ||||
| -rw-r--r-- | internal/sanitizers.h | 16 | ||||
| -rw-r--r-- | internal/set_table.h | 29 | ||||
| -rw-r--r-- | internal/signal.h | 1 | ||||
| -rw-r--r-- | internal/st.h | 2 | ||||
| -rw-r--r-- | internal/string.h | 1 | ||||
| -rw-r--r-- | internal/struct.h | 75 | ||||
| -rw-r--r-- | internal/symbol.h | 9 | ||||
| -rw-r--r-- | internal/thread.h | 12 | ||||
| -rw-r--r-- | internal/time.h | 7 | ||||
| -rw-r--r-- | internal/variable.h | 20 | ||||
| -rw-r--r-- | internal/vm.h | 9 |
31 files changed, 648 insertions, 413 deletions
diff --git a/internal/array.h b/internal/array.h index 398676df4a..3a689646fb 100644 --- a/internal/array.h +++ b/internal/array.h @@ -140,6 +140,8 @@ RARRAY_AREF(VALUE ary, long i) VALUE val; RBIMPL_ASSERT_TYPE(ary, RUBY_T_ARRAY); + RUBY_ASSERT(i < RARRAY_LEN(ary)); + RBIMPL_WARNING_PUSH(); #if defined(__GNUC__) && !defined(__clang__) && __GNUC__ == 13 RBIMPL_WARNING_IGNORED(-Warray-bounds); diff --git a/internal/basic_operators.h b/internal/basic_operators.h index 5dc8d7fe8d..493d2fa7f7 100644 --- a/internal/basic_operators.h +++ b/internal/basic_operators.h @@ -24,6 +24,7 @@ enum ruby_basic_operators { BOP_SUCC, BOP_GT, BOP_GE, + BOP_GTGT, BOP_NOT, BOP_NEQ, BOP_MATCH, diff --git a/internal/bignum.h b/internal/bignum.h index db8d3aee83..0692bafed3 100644 --- a/internal/bignum.h +++ b/internal/bignum.h @@ -9,6 +9,7 @@ * @brief Internal header for Bignums. */ #include "ruby/internal/config.h" /* for HAVE_LIBGMP */ +#include "internal/compilers.h" /* for FLEX_ARY_LEN */ #include <stddef.h> /* for size_t */ #ifdef HAVE_SYS_TYPES_H @@ -76,18 +77,17 @@ #define RBIGNUM(obj) ((struct RBignum *)(obj)) #define BIGNUM_SIGN_BIT FL_USER1 #define BIGNUM_EMBED_FLAG ((VALUE)FL_USER2) -#define BIGNUM_EMBED_LEN_NUMBITS 3 + +/* This is likely more bits than we need today and will also need adjustment if + * we change GC slot sizes. + */ +#define BIGNUM_EMBED_LEN_NUMBITS 9 #define BIGNUM_EMBED_LEN_MASK \ - (~(~(VALUE)0U << BIGNUM_EMBED_LEN_NUMBITS) << BIGNUM_EMBED_LEN_SHIFT) + (RUBY_FL_USER11 | RUBY_FL_USER10 | RUBY_FL_USER9 | RUBY_FL_USER8 | RUBY_FL_USER7 | \ + RUBY_FL_USER6 | RUBY_FL_USER5 | RUBY_FL_USER4 | RUBY_FL_USER3) #define BIGNUM_EMBED_LEN_SHIFT \ (FL_USHIFT+3) /* bit offset of BIGNUM_EMBED_LEN_MASK */ -#ifndef BIGNUM_EMBED_LEN_MAX -# if (SIZEOF_VALUE*RBIMPL_RVALUE_EMBED_LEN_MAX/SIZEOF_ACTUAL_BDIGIT) < (1 << BIGNUM_EMBED_LEN_NUMBITS)-1 -# define BIGNUM_EMBED_LEN_MAX (SIZEOF_VALUE*RBIMPL_RVALUE_EMBED_LEN_MAX/SIZEOF_ACTUAL_BDIGIT) -# else -# define BIGNUM_EMBED_LEN_MAX ((1 << BIGNUM_EMBED_LEN_NUMBITS)-1) -# endif -#endif +#define BIGNUM_EMBED_LEN_MAX (BIGNUM_EMBED_LEN_MASK >> BIGNUM_EMBED_LEN_SHIFT) enum rb_int_parse_flags { RB_INT_PARSE_SIGN = 0x01, @@ -104,7 +104,12 @@ struct RBignum { size_t len; BDIGIT *digits; } heap; - BDIGIT ary[BIGNUM_EMBED_LEN_MAX]; + /* This is a length 1 array because: + * 1. GCC has a bug that does not optimize C flexible array members + * (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=102452) + * 2. Zero length arrays are not supported by all compilers + */ + BDIGIT ary[1]; } as; }; @@ -121,6 +126,7 @@ VALUE rb_integer_float_eq(VALUE x, VALUE y); VALUE rb_str_convert_to_inum(VALUE str, int base, int badcheck, int raise_exception); VALUE rb_big_comp(VALUE x); VALUE rb_big_aref(VALUE x, VALUE y); +VALUE rb_big_aref2(VALUE num, VALUE beg, VALUE len); VALUE rb_big_abs(VALUE x); VALUE rb_big_size_m(VALUE big); VALUE rb_big_bit_length(VALUE big); @@ -163,7 +169,13 @@ VALUE rb_str2big_gmp(VALUE arg, int base, int badcheck); VALUE rb_int_parse_cstr(const char *str, ssize_t len, char **endp, size_t *ndigits, int base, int flags); RUBY_SYMBOL_EXPORT_END +#if HAVE_LONG_LONG +VALUE rb_ull2big(unsigned LONG_LONG n); +VALUE rb_ll2big(LONG_LONG n); +#endif + #if defined(HAVE_INT128_T) +VALUE rb_uint128t2big(uint128_t n); VALUE rb_int128t2big(int128_t n); #endif diff --git a/internal/bits.h b/internal/bits.h index 2b5aecf112..698ab3e219 100644 --- a/internal/bits.h +++ b/internal/bits.h @@ -30,13 +30,13 @@ #include <stdint.h> /* for uintptr_t */ #include "internal/compilers.h" /* for MSC_VERSION_SINCE */ -#if MSC_VERSION_SINCE(1310) +#ifdef _MSC_VER # include <stdlib.h> /* for _byteswap_uint64 */ #endif #if defined(HAVE_X86INTRIN_H) # include <x86intrin.h> /* for _lzcnt_u64 */ -#elif MSC_VERSION_SINCE(1310) +#elif defined(_MSC_VER) # include <intrin.h> /* for the following intrinsics */ #endif @@ -50,16 +50,13 @@ # pragma intrinsic(__lzcnt64) #endif -#if MSC_VERSION_SINCE(1310) +#if defined(_MSC_VER) # pragma intrinsic(_rotl) # pragma intrinsic(_rotr) # ifdef _WIN64 # pragma intrinsic(_rotl64) # pragma intrinsic(_rotr64) # endif -#endif - -#if MSC_VERSION_SINCE(1400) # pragma intrinsic(_BitScanForward) # pragma intrinsic(_BitScanReverse) # ifdef _WIN64 @@ -266,7 +263,7 @@ ruby_swap16(uint16_t x) #if __has_builtin(__builtin_bswap16) return __builtin_bswap16(x); -#elif MSC_VERSION_SINCE(1310) +#elif defined(_MSC_VER) return _byteswap_ushort(x); #else @@ -281,7 +278,7 @@ ruby_swap32(uint32_t x) #if __has_builtin(__builtin_bswap32) return __builtin_bswap32(x); -#elif MSC_VERSION_SINCE(1310) +#elif defined(_MSC_VER) return _byteswap_ulong(x); #else @@ -298,7 +295,7 @@ ruby_swap64(uint64_t x) #if __has_builtin(__builtin_bswap64) return __builtin_bswap64(x); -#elif MSC_VERSION_SINCE(1310) +#elif defined(_MSC_VER) return _byteswap_uint64(x); #else @@ -323,7 +320,7 @@ nlz_int32(uint32_t x) #elif defined(__x86_64__) && defined(__LZCNT__) return (unsigned int)_lzcnt_u32(x); -#elif MSC_VERSION_SINCE(1400) /* &&! defined(__AVX2__) */ +#elif defined(_MSC_VER) /* &&! defined(__AVX2__) */ unsigned long r; return _BitScanReverse(&r, x) ? (31 - (int)r) : 32; @@ -352,7 +349,7 @@ nlz_int64(uint64_t x) #elif defined(__x86_64__) && defined(__LZCNT__) return (unsigned int)_lzcnt_u64(x); -#elif defined(_WIN64) && MSC_VERSION_SINCE(1400) /* &&! defined(__AVX2__) */ +#elif defined(_WIN64) && defined(_MSC_VER) /* &&! defined(__AVX2__) */ unsigned long r; return _BitScanReverse64(&r, x) ? (63u - (unsigned int)r) : 64; @@ -538,7 +535,7 @@ ntz_int32(uint32_t x) #if defined(__x86_64__) && defined(__BMI__) return (unsigned)_tzcnt_u32(x); -#elif MSC_VERSION_SINCE(1400) +#elif defined(_MSC_VER) /* :FIXME: Is there any way to issue TZCNT instead of BSF, apart from using * assembly? Because issuing LZCNT seems possible (see nlz.h). */ unsigned long r; @@ -560,7 +557,7 @@ ntz_int64(uint64_t x) #if defined(__x86_64__) && defined(__BMI__) return (unsigned)_tzcnt_u64(x); -#elif defined(_WIN64) && MSC_VERSION_SINCE(1400) +#elif defined(_WIN64) && defined(_MSC_VER) unsigned long r; return _BitScanForward64(&r, x) ? (int)r : 64; @@ -608,10 +605,10 @@ RUBY_BIT_ROTL(VALUE v, int n) #elif __has_builtin(__builtin_rotateleft64) && (SIZEOF_VALUE * CHAR_BIT == 64) return __builtin_rotateleft64(v, n); -#elif MSC_VERSION_SINCE(1310) && (SIZEOF_VALUE * CHAR_BIT == 32) +#elif defined(_MSC_VER) && (SIZEOF_VALUE * CHAR_BIT == 32) return _rotl(v, n); -#elif MSC_VERSION_SINCE(1310) && (SIZEOF_VALUE * CHAR_BIT == 64) +#elif defined(_MSC_VER) && (SIZEOF_VALUE * CHAR_BIT == 64) return _rotl64(v, n); #elif defined(_lrotl) && (SIZEOF_VALUE == SIZEOF_LONG) @@ -632,10 +629,10 @@ RUBY_BIT_ROTR(VALUE v, int n) #elif __has_builtin(__builtin_rotateright64) && (SIZEOF_VALUE * CHAR_BIT == 64) return __builtin_rotateright64(v, n); -#elif MSC_VERSION_SINCE(1310) && (SIZEOF_VALUE * CHAR_BIT == 32) +#elif defined(_MSC_VER) && (SIZEOF_VALUE * CHAR_BIT == 32) return _rotr(v, n); -#elif MSC_VERSION_SINCE(1310) && (SIZEOF_VALUE * CHAR_BIT == 64) +#elif defined(_MSC_VER) && (SIZEOF_VALUE * CHAR_BIT == 64) return _rotr64(v, n); #elif defined(_lrotr) && (SIZEOF_VALUE == SIZEOF_LONG) diff --git a/internal/box.h b/internal/box.h new file mode 100644 index 0000000000..b62b6a9bc9 --- /dev/null +++ b/internal/box.h @@ -0,0 +1,83 @@ +#ifndef INTERNAL_BOX_H /*-*-C-*-vi:se ft=c:*/ +#define INTERNAL_BOX_H + +#include "ruby/ruby.h" /* for VALUE */ + +/** + * @author Ruby developers <ruby-core@ruby-lang.org> + * @copyright This file is a part of the programming language Ruby. + * Permission is hereby granted, to either redistribute and/or + * modify this file, provided that the conditions mentioned in the + * file COPYING are met. Consult the file for details. + * @brief Internal header for Ruby Box. + */ +struct rb_box_struct { + /* + * To retrieve Ruby::Box object that provides #require and so on. + * That is used from load.c, etc., that uses rb_box_t internally. + */ + VALUE box_object; + long box_id; // box_id to generate ext filenames + + VALUE top_self; + + VALUE load_path; + VALUE load_path_snapshot; + VALUE load_path_check_cache; + VALUE expanded_load_path; + VALUE loaded_features; + VALUE loaded_features_snapshot; + VALUE loaded_features_realpaths; + VALUE loaded_features_realpath_map; + struct st_table *loaded_features_index; + struct st_table *loading_table; + VALUE ruby_dln_libmap; + + VALUE gvar_tbl; + struct st_table *classext_cow_classes; + + bool is_user; + bool is_optional; +}; +typedef struct rb_box_struct rb_box_t; + +#define BOX_OBJ_P(obj) (rb_obj_class(obj) == rb_cBox) + +#define BOX_ROOT_P(box) (box && !box->is_user) +#define BOX_USER_P(box) (box && box->is_user) +#define BOX_OPTIONAL_P(box) (box && box->is_optional) +#define BOX_MAIN_P(box) (box && box->is_user && !box->is_optional) + +#define BOX_METHOD_DEFINITION(mdef) (mdef ? mdef->ns : NULL) +#define BOX_METHOD_ENTRY(me) (me ? BOX_METHOD_DEFINITION(me->def) : NULL) +#define BOX_CC(cc) (cc ? BOX_METHOD_ENTRY(cc->cme_) : NULL) +#define BOX_CC_ENTRIES(ccs) (ccs ? BOX_METHOD_ENTRY(ccs->cme) : NULL) + +RUBY_EXTERN bool ruby_box_enabled; +RUBY_EXTERN bool ruby_box_init_done; +RUBY_EXTERN bool ruby_box_crashed; + +static inline bool +rb_box_available(void) +{ + return ruby_box_enabled; +} + +const rb_box_t * rb_root_box(void); +const rb_box_t * rb_main_box(void); +const rb_box_t * rb_current_box(void); +const rb_box_t * rb_loading_box(void); +const rb_box_t * rb_current_box_in_crash_report(void); + +void rb_box_entry_mark(void *); +void rb_box_gc_update_references(void *ptr); + +rb_box_t * rb_get_box_t(VALUE ns); +VALUE rb_get_box_object(rb_box_t *ns); + +VALUE rb_box_local_extension(VALUE box, VALUE fname, VALUE path, VALUE *cleanup); +void rb_box_cleanup_local_extension(VALUE cleanup); + +void rb_initialize_main_box(void); +void rb_box_init_done(void); +#endif /* INTERNAL_BOX_H */ diff --git a/internal/class.h b/internal/class.h index 82f8f0e9dc..ea68b07fc2 100644 --- a/internal/class.h +++ b/internal/class.h @@ -10,7 +10,7 @@ */ #include "id.h" #include "id_table.h" /* for struct rb_id_table */ -#include "internal/namespace.h" /* for rb_current_namespace */ +#include "internal/box.h" #include "internal/serial.h" /* for rb_serial_t */ #include "internal/static_assert.h" #include "internal/variable.h" /* for rb_class_ivar_set */ @@ -27,37 +27,37 @@ # undef RCLASS_SUPER #endif -struct rb_ns_subclasses { +struct rb_box_subclasses { long refcount; struct st_table *tbl; }; -typedef struct rb_ns_subclasses rb_ns_subclasses_t; +typedef struct rb_box_subclasses rb_box_subclasses_t; static inline long -rb_ns_subclasses_ref_count(rb_ns_subclasses_t *ns_sub) +rb_box_subclasses_ref_count(rb_box_subclasses_t *box_sub) { - return ns_sub->refcount; + return box_sub->refcount; } -static inline rb_ns_subclasses_t * -rb_ns_subclasses_ref_inc(rb_ns_subclasses_t *ns_sub) +static inline rb_box_subclasses_t * +rb_box_subclasses_ref_inc(rb_box_subclasses_t *box_sub) { - ns_sub->refcount++; - return ns_sub; + box_sub->refcount++; + return box_sub; } static inline void -rb_ns_subclasses_ref_dec(rb_ns_subclasses_t *ns_sub) +rb_box_subclasses_ref_dec(rb_box_subclasses_t *box_sub) { - ns_sub->refcount--; - if (ns_sub->refcount == 0) { - st_free_table(ns_sub->tbl); - xfree(ns_sub); + box_sub->refcount--; + if (box_sub->refcount == 0) { + st_free_table(box_sub->tbl); + xfree(box_sub); } } struct rb_subclass_anchor { - rb_ns_subclasses_t *ns_subclasses; + rb_box_subclasses_t *box_subclasses; struct rb_subclass_entry *head; }; typedef struct rb_subclass_anchor rb_subclass_anchor_t; @@ -77,13 +77,13 @@ struct rb_cvar_class_tbl_entry { }; struct rb_classext_struct { - const rb_namespace_t *ns; + const rb_box_t *box; VALUE super; - VALUE *fields; // Fields are either ivar or other internal properties stored inline + VALUE fields_obj; // Fields are either ivar or other internal properties stored inline struct rb_id_table *m_tbl; struct rb_id_table *const_tbl; struct rb_id_table *callable_m_tbl; - struct rb_id_table *cc_tbl; /* ID -> [[ci1, cc1], [ci2, cc2] ...] */ + VALUE cc_tbl; /* { ID => { cme, [cc1, cc2, ...] }, ... } */ struct rb_id_table *cvc_tbl; VALUE *superclasses; /** @@ -92,19 +92,19 @@ struct rb_classext_struct { */ struct rb_subclass_anchor *subclasses; /** - * The `ns_super_subclasses` points the `ns_subclasses` struct to retreive the subclasses - * of the super class in a specific namespace. + * The `box_super_subclasses` points the `box_subclasses` struct to retreive the subclasses + * of the super class in a specific box. * In compaction GCs, collecting a classext should trigger the deletion of a rb_subclass_entry * from the super's subclasses. But it may be prevented by the read barrier. * Fetching the super's subclasses for a ns is to avoid the read barrier in that process. */ - rb_ns_subclasses_t *ns_super_subclasses; + rb_box_subclasses_t *box_super_subclasses; /** - * In the case that this is an `ICLASS`, `ns_module_subclasses` points to the link + * In the case that this is an `ICLASS`, `box_module_subclasses` points to the link * in the module's `subclasses` list that indicates that the klass has been * included. Hopefully that makes sense. */ - rb_ns_subclasses_t *ns_module_subclasses; + rb_box_subclasses_t *box_module_subclasses; const VALUE origin_; const VALUE refined_class; @@ -127,7 +127,6 @@ struct rb_classext_struct { bool shared_const_tbl : 1; bool iclass_is_origin : 1; bool iclass_origin_shared_mtbl : 1; - bool superclasses_owner : 1; bool superclasses_with_self : 1; VALUE classpath; }; @@ -137,24 +136,29 @@ STATIC_ASSERT(shape_max_variations, SHAPE_MAX_VARIATIONS < (1 << (sizeof(((rb_cl struct RClass { struct RBasic basic; - st_table *ns_classext_tbl; // ns_object -> (rb_classext_t *) VALUE object_id; /* - * If ns_classext_tbl is NULL, then the prime classext is readable (because no other classext exists). + * If box_classext_tbl is NULL, then the prime classext is readable (because no other classext exists). * For the check whether writable or not, check flag RCLASS_PRIME_CLASSEXT_WRITABLE */ }; -// Assert that classes can be embedded in heaps[2] (which has 160B slot size) -// On 32bit platforms there is no variable width allocation so it doesn't matter. -// TODO: restore this assertion after shrinking rb_classext_t -// STATIC_ASSERT(sizeof_rb_classext_t, sizeof(struct RClass) + sizeof(rb_classext_t) <= 4 * RVALUE_SIZE || SIZEOF_VALUE < SIZEOF_LONG_LONG); - struct RClass_and_rb_classext_t { struct RClass rclass; rb_classext_t classext; }; +#if SIZEOF_VALUE >= SIZEOF_LONG_LONG +// Assert that classes can be embedded in heaps[2] (which has 160B slot size) +// On 32bit platforms there is no variable width allocation so it doesn't matter. +STATIC_ASSERT(sizeof_rb_classext_t, sizeof(struct RClass_and_rb_classext_t) <= 4 * RVALUE_SIZE); +#endif + +struct RClass_boxable { + struct RClass_and_rb_classext_t base; + st_table *box_classext_tbl; // box_object -> (rb_classext_t *) +}; + static const uint16_t RCLASS_MAX_SUPERCLASS_DEPTH = ((uint16_t)-1); static inline bool RCLASS_SINGLETON_P(VALUE klass); @@ -166,17 +170,16 @@ static inline void RCLASS_SET_PRIME_CLASSEXT_WRITABLE(VALUE obj, bool writable); #define RCLASS_EXT_PRIME(c) (&((struct RClass_and_rb_classext_t*)(c))->classext) #define RCLASS_EXT_PRIME_P(ext, c) (&((struct RClass_and_rb_classext_t*)(c))->classext == ext) -static inline rb_classext_t * RCLASS_EXT_READABLE_IN_NS(VALUE obj, const rb_namespace_t *ns); +static inline rb_classext_t * RCLASS_EXT_READABLE_IN_BOX(VALUE obj, const rb_box_t *box); static inline rb_classext_t * RCLASS_EXT_READABLE(VALUE obj); -static inline rb_classext_t * RCLASS_EXT_WRITABLE_IN_NS(VALUE obj, const rb_namespace_t *ns); +static inline rb_classext_t * RCLASS_EXT_WRITABLE_IN_BOX(VALUE obj, const rb_box_t *box); static inline rb_classext_t * RCLASS_EXT_WRITABLE(VALUE obj); // Raw accessor -#define RCLASS_CLASSEXT_TBL(klass) (RCLASS(klass)->ns_classext_tbl) - -#define RCLASSEXT_NS(ext) (ext->ns) +#define RCLASSEXT_BOX(ext) (ext->box) #define RCLASSEXT_SUPER(ext) (ext->super) -#define RCLASSEXT_FIELDS(ext) (ext->fields) +#define RCLASSEXT_FIELDS(ext) (ext->fields_obj ? ROBJECT_FIELDS(ext->fields_obj) : NULL) +#define RCLASSEXT_FIELDS_OBJ(ext) (ext->fields_obj) #define RCLASSEXT_M_TBL(ext) (ext->m_tbl) #define RCLASSEXT_CONST_TBL(ext) (ext->const_tbl) #define RCLASSEXT_CALLABLE_M_TBL(ext) (ext->callable_m_tbl) @@ -185,20 +188,17 @@ static inline rb_classext_t * RCLASS_EXT_WRITABLE(VALUE obj); #define RCLASSEXT_SUPERCLASS_DEPTH(ext) (ext->superclass_depth) #define RCLASSEXT_SUPERCLASSES(ext) (ext->superclasses) #define RCLASSEXT_SUBCLASSES(ext) (ext->subclasses) -#define RCLASSEXT_NS_SUPER_SUBCLASSES(ext) (ext->ns_super_subclasses) -#define RCLASSEXT_NS_MODULE_SUBCLASSES(ext) (ext->ns_module_subclasses) +#define RCLASSEXT_BOX_SUPER_SUBCLASSES(ext) (ext->box_super_subclasses) +#define RCLASSEXT_BOX_MODULE_SUBCLASSES(ext) (ext->box_module_subclasses) #define RCLASSEXT_ORIGIN(ext) (ext->origin_) #define RCLASSEXT_REFINED_CLASS(ext) (ext->refined_class) // class.allocator/singleton_class.attached_object are not accessed directly via RCLASSEXT_* #define RCLASSEXT_INCLUDER(ext) (ext->as.iclass.includer) -#define RCLASSEXT_MAX_IV_COUNT(ext) (ext->max_iv_count) -#define RCLASSEXT_VARIATION_COUNT(ext) (ext->variation_count) #define RCLASSEXT_PERMANENT_CLASSPATH(ext) (ext->permanent_classpath) #define RCLASSEXT_CLONED(ext) (ext->cloned) #define RCLASSEXT_SHARED_CONST_TBL(ext) (ext->shared_const_tbl) #define RCLASSEXT_ICLASS_IS_ORIGIN(ext) (ext->iclass_is_origin) #define RCLASSEXT_ICLASS_ORIGIN_SHARED_MTBL(ext) (ext->iclass_origin_shared_mtbl) -#define RCLASSEXT_SUPERCLASSES_OWNER(ext) (ext->superclasses_owner) #define RCLASSEXT_SUPERCLASSES_WITH_SELF(ext) (ext->superclasses_with_self) #define RCLASSEXT_CLASSPATH(ext) (ext->classpath) @@ -206,10 +206,10 @@ static inline void RCLASSEXT_SET_ORIGIN(rb_classext_t *ext, VALUE klass, VALUE o static inline void RCLASSEXT_SET_INCLUDER(rb_classext_t *ext, VALUE klass, VALUE includer); /* Prime classext entry accessor for very specific reason */ -#define RCLASS_PRIME_NS(c) (RCLASS_EXT_PRIME(c)->ns) +#define RCLASS_PRIME_BOX(c) (RCLASS_EXT_PRIME(c)->box) // To invalidate CC by inserting&invalidating method entry into tables containing the target cme // See clear_method_cache_by_id_in_class() -#define RCLASS_PRIME_FIELDS(c) (RCLASS_EXT_PRIME(c)->fields) +#define RCLASS_PRIME_FIELDS_OBJ(c) (RCLASS_EXT_PRIME(c)->fields_obj) #define RCLASS_PRIME_M_TBL(c) (RCLASS_EXT_PRIME(c)->m_tbl) #define RCLASS_PRIME_CONST_TBL(c) (RCLASS_EXT_PRIME(c)->const_tbl) #define RCLASS_PRIME_CALLABLE_M_TBL(c) (RCLASS_EXT_PRIME(c)->callable_m_tbl) @@ -218,7 +218,7 @@ static inline void RCLASSEXT_SET_INCLUDER(rb_classext_t *ext, VALUE klass, VALUE #define RCLASS_CALLABLE_M_TBL_NOT_PRIME_P(c, tbl) (RCLASS_EXT_PRIME(c)->callable_m_tbl != tbl) #define RCLASS_CC_TBL_NOT_PRIME_P(c, tbl) (RCLASS_EXT_PRIME(c)->cc_tbl != tbl) -// Read accessor, regarding namespaces +// Read accessor, regarding box #define RCLASS_SUPER(c) (RCLASS_EXT_READABLE(c)->super) #define RCLASS_M_TBL(c) (RCLASS_EXT_READABLE(c)->m_tbl) #define RCLASS_CONST_TBL(c) (RCLASS_EXT_READABLE(c)->const_tbl) @@ -227,24 +227,28 @@ static inline void RCLASSEXT_SET_INCLUDER(rb_classext_t *ext, VALUE klass, VALUE * so always those should be writable. */ #define RCLASS_CVC_TBL(c) (RCLASS_EXT_READABLE(c)->cvc_tbl) -#define RCLASS_SUPERCLASS_DEPTH(c) (RCLASS_EXT_READABLE(c)->superclass_depth) -#define RCLASS_SUPERCLASSES(c) (RCLASS_EXT_READABLE(c)->superclasses) -#define RCLASS_SUPERCLASSES_WITH_SELF_P(c) (RCLASS_EXT_READABLE(c)->superclasses_with_self) #define RCLASS_SUBCLASSES_X(c) (RCLASS_EXT_READABLE(c)->subclasses) #define RCLASS_SUBCLASSES_FIRST(c) (RCLASS_EXT_READABLE(c)->subclasses->head->next) #define RCLASS_ORIGIN(c) (RCLASS_EXT_READABLE(c)->origin_) #define RICLASS_IS_ORIGIN_P(c) (RCLASS_EXT_READABLE(c)->iclass_is_origin) -#define RCLASS_MAX_IV_COUNT(c) (RCLASS_EXT_READABLE(c)->max_iv_count) -#define RCLASS_VARIATION_COUNT(c) (RCLASS_EXT_READABLE(c)->variation_count) #define RCLASS_PERMANENT_CLASSPATH_P(c) (RCLASS_EXT_READABLE(c)->permanent_classpath) #define RCLASS_CLONED_P(c) (RCLASS_EXT_READABLE(c)->cloned) #define RCLASS_CLASSPATH(c) (RCLASS_EXT_READABLE(c)->classpath) -// namespaces don't make changes on these refined_class/attached_object/includer +// Superclasses can't be changed after initialization +#define RCLASS_SUPERCLASS_DEPTH(c) (RCLASS_EXT_PRIME(c)->superclass_depth) +#define RCLASS_SUPERCLASSES(c) (RCLASS_EXT_PRIME(c)->superclasses) +#define RCLASS_SUPERCLASSES_WITH_SELF_P(c) (RCLASS_EXT_PRIME(c)->superclasses_with_self) + +// Ruby Box doesn't make changes on these refined_class/attached_object/includer #define RCLASS_REFINED_CLASS(c) (RCLASS_EXT_PRIME(c)->refined_class) #define RCLASS_ATTACHED_OBJECT(c) (RCLASS_EXT_PRIME(c)->as.singleton_class.attached_object) #define RCLASS_INCLUDER(c) (RCLASS_EXT_PRIME(c)->as.iclass.includer) +// max IV count and variation count are just hints, so they don't need to be per-box +#define RCLASS_MAX_IV_COUNT(ext) (RCLASS_EXT_PRIME(ext)->max_iv_count) +#define RCLASS_VARIATION_COUNT(ext) (RCLASS_EXT_PRIME(ext)->variation_count) + // Writable classext entries (instead of RCLASS_SET_*) because member data will be operated directly #define RCLASS_WRITABLE_M_TBL(c) (RCLASS_EXT_WRITABLE(c)->m_tbl) #define RCLASS_WRITABLE_CONST_TBL(c) (RCLASS_EXT_WRITABLE(c)->const_tbl) @@ -255,25 +259,17 @@ static inline void RCLASSEXT_SET_INCLUDER(rb_classext_t *ext, VALUE klass, VALUE static inline void RCLASS_SET_SUPER(VALUE klass, VALUE super); static inline void RCLASS_WRITE_SUPER(VALUE klass, VALUE super); -static inline st_table * RCLASS_FIELDS_HASH(VALUE obj); -static inline st_table * RCLASS_WRITABLE_FIELDS_HASH(VALUE obj); -static inline uint32_t RCLASS_FIELDS_COUNT(VALUE obj); -static inline void RCLASS_SET_FIELDS_HASH(VALUE obj, const st_table *table); -static inline void RCLASS_WRITE_FIELDS_HASH(VALUE obj, const st_table *table); -// TODO: rename RCLASS_SET_M_TBL_WORKAROUND (and _WRITE_) to RCLASS_SET_M_TBL with write barrier -static inline void RCLASS_SET_M_TBL_WORKAROUND(VALUE klass, struct rb_id_table *table, bool check_promoted); -static inline void RCLASS_WRITE_M_TBL_WORKAROUND(VALUE klass, struct rb_id_table *table, bool check_promoted); static inline void RCLASS_SET_CONST_TBL(VALUE klass, struct rb_id_table *table, bool shared); static inline void RCLASS_WRITE_CONST_TBL(VALUE klass, struct rb_id_table *table, bool shared); static inline void RCLASS_WRITE_CALLABLE_M_TBL(VALUE klass, struct rb_id_table *table); -static inline void RCLASS_WRITE_CC_TBL(VALUE klass, struct rb_id_table *table); +static inline void RCLASS_WRITE_CC_TBL(VALUE klass, VALUE table); static inline void RCLASS_SET_CVC_TBL(VALUE klass, struct rb_id_table *table); static inline void RCLASS_WRITE_CVC_TBL(VALUE klass, struct rb_id_table *table); -static inline void RCLASS_WRITE_SUPERCLASSES(VALUE klass, size_t depth, VALUE *superclasses, bool owns_it, bool with_self); +static inline void RCLASS_WRITE_SUPERCLASSES(VALUE klass, size_t depth, VALUE *superclasses, bool with_self); static inline void RCLASS_SET_SUBCLASSES(VALUE klass, rb_subclass_anchor_t *anchor); -static inline void RCLASS_WRITE_NS_SUPER_SUBCLASSES(VALUE klass, rb_ns_subclasses_t *ns_subclasses); -static inline void RCLASS_WRITE_NS_MODULE_SUBCLASSES(VALUE klass, rb_ns_subclasses_t *ns_subclasses); +static inline void RCLASS_WRITE_BOX_SUPER_SUBCLASSES(VALUE klass, rb_box_subclasses_t *box_subclasses); +static inline void RCLASS_WRITE_BOX_MODULE_SUBCLASSES(VALUE klass, rb_box_subclasses_t *box_subclasses); static inline void RCLASS_SET_ORIGIN(VALUE klass, VALUE origin); static inline void RCLASS_WRITE_ORIGIN(VALUE klass, VALUE origin); @@ -288,59 +284,87 @@ static inline VALUE RCLASS_SET_ATTACHED_OBJECT(VALUE klass, VALUE attached_objec static inline void RCLASS_SET_INCLUDER(VALUE iclass, VALUE klass); static inline void RCLASS_SET_MAX_IV_COUNT(VALUE klass, attr_index_t count); -static inline void RCLASS_WRITE_MAX_IV_COUNT(VALUE klass, attr_index_t count); static inline void RCLASS_SET_CLONED(VALUE klass, bool cloned); static inline void RCLASS_SET_CLASSPATH(VALUE klass, VALUE classpath, bool permanent); static inline void RCLASS_WRITE_CLASSPATH(VALUE klass, VALUE classpath, bool permanent); #define RCLASS_IS_ROOT FL_USER0 -// 1 is for RUBY_FL_SINGLETON or RMODULE_ALLOCATED_BUT_NOT_INITIALIZED (see class.c) +// 1 is for RUBY_FL_SINGLETON or RMODULE_IS_REFINEMENT #define RCLASS_PRIME_CLASSEXT_WRITABLE FL_USER2 +#define RCLASS_IS_INITIALIZED FL_USER3 // 3 is RMODULE_IS_REFINEMENT for RMODULE -// 4-19: SHAPE_FLAG_MASK +#define RCLASS_BOXABLE FL_USER4 + +static inline st_table * +RCLASS_CLASSEXT_TBL(VALUE klass) +{ + if (FL_TEST_RAW(klass, RCLASS_BOXABLE)) { + struct RClass_boxable *box_klass = (struct RClass_boxable *)klass; + return box_klass->box_classext_tbl; + } + return NULL; +} + +static inline void +RCLASS_SET_CLASSEXT_TBL(VALUE klass, st_table *tbl) +{ + RUBY_ASSERT(FL_TEST_RAW(klass, RCLASS_BOXABLE)); + struct RClass_boxable *box_klass = (struct RClass_boxable *)klass; + box_klass->box_classext_tbl = tbl; +} /* class.c */ -rb_classext_t * rb_class_duplicate_classext(rb_classext_t *orig, VALUE obj, const rb_namespace_t *ns); +rb_classext_t * rb_class_duplicate_classext(rb_classext_t *orig, VALUE obj, const rb_box_t *box); void rb_class_ensure_writable(VALUE obj); +void rb_class_set_box_classext(VALUE obj, const rb_box_t *box, rb_classext_t *ext); + static inline int -RCLASS_SET_NAMESPACE_CLASSEXT(VALUE obj, const rb_namespace_t *ns, rb_classext_t *ext) +RCLASS_SET_BOX_CLASSEXT(VALUE obj, const rb_box_t *box, rb_classext_t *ext) { int first_set = 0; st_table *tbl = RCLASS_CLASSEXT_TBL(obj); - VM_ASSERT(NAMESPACE_USER_P(ns)); // non-prime classext is only for user namespace, with ns_object - VM_ASSERT(ns->ns_object); - VM_ASSERT(RCLASSEXT_NS(ext) == ns); + VM_ASSERT(BOX_USER_P(box)); // non-prime classext is only for user box, with box_object + VM_ASSERT(box->box_object); + VM_ASSERT(RCLASSEXT_BOX(ext) == box); if (!tbl) { - RCLASS_CLASSEXT_TBL(obj) = tbl = st_init_numtable_with_size(1); + tbl = st_init_numtable_with_size(1); + RCLASS_SET_CLASSEXT_TBL(obj, tbl); } if (rb_st_table_size(tbl) == 0) { first_set = 1; } - rb_st_insert(tbl, (st_data_t)ns->ns_object, (st_data_t)ext); + + rb_class_set_box_classext(obj, box, ext); + return first_set; } +#define VM_ASSERT_BOXABLE_TYPE(klass) \ + VM_ASSERT(RB_TYPE_P(klass, T_CLASS) || RB_TYPE_P(klass, T_MODULE) || RB_TYPE_P(klass, T_ICLASS), "%s is not boxable type", rb_type_str(BUILTIN_TYPE(klass))) + static inline bool RCLASS_PRIME_CLASSEXT_READABLE_P(VALUE klass) { - VM_ASSERT(RB_TYPE_P(klass, T_CLASS) || RB_TYPE_P(klass, T_MODULE) || RB_TYPE_P(klass, T_ICLASS)); + VM_ASSERT(klass != 0, "klass should be a valid object"); + VM_ASSERT_BOXABLE_TYPE(klass); // if the lookup table exists, then it means the prime classext is NOT directly readable. - return RCLASS_CLASSEXT_TBL(klass) == NULL; + return !FL_TEST_RAW(klass, RCLASS_BOXABLE) || RCLASS_CLASSEXT_TBL(klass) == NULL; } static inline bool RCLASS_PRIME_CLASSEXT_WRITABLE_P(VALUE klass) { - VM_ASSERT(RB_TYPE_P(klass, T_CLASS) || RB_TYPE_P(klass, T_MODULE) || RB_TYPE_P(klass, T_ICLASS)); + VM_ASSERT(klass != 0, "klass should be a valid object"); + VM_ASSERT_BOXABLE_TYPE(klass); return FL_TEST(klass, RCLASS_PRIME_CLASSEXT_WRITABLE); } static inline void RCLASS_SET_PRIME_CLASSEXT_WRITABLE(VALUE klass, bool writable) { - VM_ASSERT(RB_TYPE_P(klass, T_CLASS) || RB_TYPE_P(klass, T_MODULE) || RB_TYPE_P(klass, T_ICLASS)); - + VM_ASSERT(klass != 0, "klass should be a valid object"); + VM_ASSERT_BOXABLE_TYPE(klass); if (writable) { FL_SET(klass, RCLASS_PRIME_CLASSEXT_WRITABLE); } @@ -350,12 +374,12 @@ RCLASS_SET_PRIME_CLASSEXT_WRITABLE(VALUE klass, bool writable) } static inline rb_classext_t * -RCLASS_EXT_TABLE_LOOKUP_INTERNAL(VALUE obj, const rb_namespace_t *ns) +RCLASS_EXT_TABLE_LOOKUP_INTERNAL(VALUE obj, const rb_box_t *box) { st_data_t classext_ptr; st_table *classext_tbl = RCLASS_CLASSEXT_TBL(obj); if (classext_tbl) { - if (rb_st_lookup(classext_tbl, (st_data_t)ns->ns_object, &classext_ptr)) { + if (rb_st_lookup(classext_tbl, (st_data_t)box->box_object, &classext_ptr)) { return (rb_classext_t *)classext_ptr; } } @@ -363,9 +387,9 @@ RCLASS_EXT_TABLE_LOOKUP_INTERNAL(VALUE obj, const rb_namespace_t *ns) } static inline rb_classext_t * -RCLASS_EXT_READABLE_LOOKUP(VALUE obj, const rb_namespace_t *ns) +RCLASS_EXT_READABLE_LOOKUP(VALUE obj, const rb_box_t *box) { - rb_classext_t *ext = RCLASS_EXT_TABLE_LOOKUP_INTERNAL(obj, ns); + rb_classext_t *ext = RCLASS_EXT_TABLE_LOOKUP_INTERNAL(obj, box); if (ext) return ext; // Classext for the ns not found. Refer the prime one instead. @@ -373,87 +397,78 @@ RCLASS_EXT_READABLE_LOOKUP(VALUE obj, const rb_namespace_t *ns) } static inline rb_classext_t * -RCLASS_EXT_READABLE_IN_NS(VALUE obj, const rb_namespace_t *ns) +RCLASS_EXT_READABLE_IN_BOX(VALUE obj, const rb_box_t *box) { - if (!ns - || NAMESPACE_BUILTIN_P(ns) + if (BOX_ROOT_P(box) || RCLASS_PRIME_CLASSEXT_READABLE_P(obj)) { return RCLASS_EXT_PRIME(obj); } - return RCLASS_EXT_READABLE_LOOKUP(obj, ns); + return RCLASS_EXT_READABLE_LOOKUP(obj, box); } static inline rb_classext_t * RCLASS_EXT_READABLE(VALUE obj) { - const rb_namespace_t *ns; + const rb_box_t *box; if (RCLASS_PRIME_CLASSEXT_READABLE_P(obj)) { return RCLASS_EXT_PRIME(obj); } - // delay namespace loading to optimize for unmodified classes - ns = rb_current_namespace(); - if (!ns || NAMESPACE_BUILTIN_P(ns)) { + // delay determining the current box to optimize for unmodified classes + box = rb_current_box(); + if (BOX_ROOT_P(box)) { return RCLASS_EXT_PRIME(obj); } - return RCLASS_EXT_READABLE_LOOKUP(obj, ns); + return RCLASS_EXT_READABLE_LOOKUP(obj, box); } static inline rb_classext_t * -RCLASS_EXT_WRITABLE_LOOKUP(VALUE obj, const rb_namespace_t *ns) +RCLASS_EXT_WRITABLE_LOOKUP(VALUE obj, const rb_box_t *box) { rb_classext_t *ext; int first_set = 0; - ext = RCLASS_EXT_TABLE_LOOKUP_INTERNAL(obj, ns); + ext = RCLASS_EXT_TABLE_LOOKUP_INTERNAL(obj, box); if (ext) return ext; - if (!rb_shape_obj_too_complex_p(obj)) { - rb_evict_ivars_to_hash(obj); // fallback to ivptr for ivars from shapes - } - - RB_VM_LOCK_ENTER(); - { + RB_VM_LOCKING() { // re-check the classext is not created to avoid the multi-thread race - ext = RCLASS_EXT_TABLE_LOOKUP_INTERNAL(obj, ns); + ext = RCLASS_EXT_TABLE_LOOKUP_INTERNAL(obj, box); if (!ext) { - ext = rb_class_duplicate_classext(RCLASS_EXT_PRIME(obj), obj, ns); - first_set = RCLASS_SET_NAMESPACE_CLASSEXT(obj, ns, ext); + ext = rb_class_duplicate_classext(RCLASS_EXT_PRIME(obj), obj, box); + first_set = RCLASS_SET_BOX_CLASSEXT(obj, box, ext); if (first_set) { + // TODO: are there any case that a class/module become non-writable after its birthtime? RCLASS_SET_PRIME_CLASSEXT_WRITABLE(obj, false); } } } - RB_VM_LOCK_LEAVE(); return ext; } static inline rb_classext_t * -RCLASS_EXT_WRITABLE_IN_NS(VALUE obj, const rb_namespace_t *ns) +RCLASS_EXT_WRITABLE_IN_BOX(VALUE obj, const rb_box_t *box) { - if (!ns - || NAMESPACE_BUILTIN_P(ns) + if (BOX_ROOT_P(box) || RCLASS_PRIME_CLASSEXT_WRITABLE_P(obj)) { return RCLASS_EXT_PRIME(obj); } - return RCLASS_EXT_WRITABLE_LOOKUP(obj, ns); + return RCLASS_EXT_WRITABLE_LOOKUP(obj, box); } static inline rb_classext_t * RCLASS_EXT_WRITABLE(VALUE obj) { - const rb_namespace_t *ns; - if (RCLASS_PRIME_CLASSEXT_WRITABLE_P(obj)) { + const rb_box_t *box; + if (LIKELY(RCLASS_PRIME_CLASSEXT_WRITABLE_P(obj))) { return RCLASS_EXT_PRIME(obj); } - // delay namespace loading to optimize for unmodified classes - ns = rb_current_namespace(); - if (!ns || NAMESPACE_BUILTIN_P(ns)) { - // If no namespace is specified, Ruby VM is in bootstrap - // and the clean class definition is under construction. + // delay determining the current box to optimize for unmodified classes + box = rb_current_box(); + if (BOX_ROOT_P(box)) { return RCLASS_EXT_PRIME(obj); } - return RCLASS_EXT_WRITABLE_LOOKUP(obj, ns); + return RCLASS_EXT_WRITABLE_LOOKUP(obj, box); } static inline void @@ -470,24 +485,18 @@ RCLASSEXT_SET_INCLUDER(rb_classext_t *ext, VALUE klass, VALUE includer) } /* class.c */ -typedef void rb_class_classext_foreach_callback_func(rb_classext_t *classext, bool is_prime, VALUE namespace, void *arg); +typedef void rb_class_classext_foreach_callback_func(rb_classext_t *classext, bool is_prime, VALUE box_value, void *arg); void rb_class_classext_foreach(VALUE klass, rb_class_classext_foreach_callback_func *func, void *arg); void rb_class_subclass_add(VALUE super, VALUE klass); -void rb_class_remove_from_super_subclasses(VALUE); -void rb_class_remove_from_module_subclasses(VALUE); -void rb_class_classext_free_subclasses(rb_classext_t *, VALUE); +void rb_class_classext_free_subclasses(rb_classext_t *, VALUE, bool); void rb_class_foreach_subclass(VALUE klass, void (*f)(VALUE, VALUE), VALUE); -void rb_class_detach_subclasses(VALUE); -void rb_class_detach_module_subclasses(VALUE); void rb_class_update_superclasses(VALUE); -size_t rb_class_superclasses_memsize(VALUE); -void rb_class_remove_subclass_head(VALUE); int rb_singleton_class_internal_p(VALUE sklass); VALUE rb_class_set_super(VALUE klass, VALUE super); VALUE rb_class_boot(VALUE); VALUE rb_class_s_alloc(VALUE klass); VALUE rb_module_s_alloc(VALUE klass); -void rb_module_set_initialized(VALUE module); +void rb_class_set_initialized(VALUE klass); void rb_module_check_initializable(VALUE module); VALUE rb_make_metaclass(VALUE, VALUE); VALUE rb_include_class_new(VALUE, VALUE); @@ -504,6 +513,10 @@ void rb_undef_methods_from(VALUE klass, VALUE super); VALUE rb_class_inherited(VALUE, VALUE); VALUE rb_keyword_error_new(const char *, VALUE); +rb_classext_t *rb_class_unlink_classext(VALUE klass, const rb_box_t *box); +void rb_class_classext_free(VALUE klass, rb_classext_t *ext, bool is_prime); +void rb_iclass_classext_free(VALUE klass, rb_classext_t *ext, bool is_prime); + RUBY_SYMBOL_EXPORT_BEGIN /* for objspace */ @@ -531,79 +544,66 @@ RCLASS_WRITE_SUPER(VALUE klass, VALUE super) RB_OBJ_WRITE(klass, &RCLASSEXT_SUPER(RCLASS_EXT_WRITABLE(klass)), super); } -static inline st_table * -RCLASS_FIELDS_HASH(VALUE obj) +static inline VALUE +RCLASS_WRITABLE_ENSURE_FIELDS_OBJ(VALUE obj) { RUBY_ASSERT(RB_TYPE_P(obj, RUBY_T_CLASS) || RB_TYPE_P(obj, RUBY_T_MODULE)); - RUBY_ASSERT(rb_shape_obj_too_complex_p(obj)); - return (st_table *)RCLASSEXT_FIELDS(RCLASS_EXT_READABLE(obj)); + rb_classext_t *ext = RCLASS_EXT_WRITABLE(obj); + if (!ext->fields_obj) { + RB_OBJ_WRITE(obj, &ext->fields_obj, rb_imemo_fields_new(obj, 1, true)); + } + return ext->fields_obj; } -static inline st_table * -RCLASS_WRITABLE_FIELDS_HASH(VALUE obj) +static inline VALUE +RCLASS_WRITABLE_FIELDS_OBJ(VALUE obj) { RUBY_ASSERT(RB_TYPE_P(obj, RUBY_T_CLASS) || RB_TYPE_P(obj, RUBY_T_MODULE)); - RUBY_ASSERT(rb_shape_obj_too_complex_p(obj)); - return (st_table *)RCLASSEXT_FIELDS(RCLASS_EXT_WRITABLE(obj)); + return RCLASSEXT_FIELDS_OBJ(RCLASS_EXT_WRITABLE(obj)); } static inline void -RCLASS_SET_FIELDS_HASH(VALUE obj, const st_table *tbl) +RCLASSEXT_SET_FIELDS_OBJ(VALUE obj, rb_classext_t *ext, VALUE fields_obj) { RUBY_ASSERT(RB_TYPE_P(obj, RUBY_T_CLASS) || RB_TYPE_P(obj, RUBY_T_MODULE)); - RUBY_ASSERT(rb_shape_obj_too_complex_p(obj)); - RCLASSEXT_FIELDS(RCLASS_EXT_PRIME(obj)) = (VALUE *)tbl; + + RB_OBJ_ATOMIC_WRITE(obj, &ext->fields_obj, fields_obj); } static inline void -RCLASS_WRITE_FIELDS_HASH(VALUE obj, const st_table *tbl) +RCLASS_WRITABLE_SET_FIELDS_OBJ(VALUE obj, VALUE fields_obj) { RUBY_ASSERT(RB_TYPE_P(obj, RUBY_T_CLASS) || RB_TYPE_P(obj, RUBY_T_MODULE)); - RUBY_ASSERT(rb_shape_obj_too_complex_p(obj)); - RCLASSEXT_FIELDS(RCLASS_EXT_WRITABLE(obj)) = (VALUE *)tbl; + + RCLASSEXT_SET_FIELDS_OBJ(obj, RCLASS_EXT_WRITABLE(obj), fields_obj); } static inline uint32_t RCLASS_FIELDS_COUNT(VALUE obj) { RUBY_ASSERT(RB_TYPE_P(obj, RUBY_T_CLASS) || RB_TYPE_P(obj, RUBY_T_MODULE)); - if (rb_shape_obj_too_complex_p(obj)) { - uint32_t count; - - // "Too complex" classes could have their IV hash mutated in - // parallel, so lets lock around getting the hash size. - RB_VM_LOCK_ENTER(); - { - count = (uint32_t)rb_st_table_size(RCLASS_FIELDS_HASH(obj)); - } - RB_VM_LOCK_LEAVE(); - return count; - } - else { - return RSHAPE(RCLASS_SHAPE_ID(obj))->next_field_index; + VALUE fields_obj = RCLASS_WRITABLE_FIELDS_OBJ(obj); + if (fields_obj) { + if (rb_shape_obj_too_complex_p(fields_obj)) { + return (uint32_t)rb_st_table_size(rb_imemo_fields_complex_tbl(fields_obj)); + } + else { + return RSHAPE_LEN(RBASIC_SHAPE_ID(fields_obj)); + } } + return 0; } -#define RCLASS_SET_M_TBL_EVEN_WHEN_PROMOTED(klass, table) RCLASS_SET_M_TBL_WORKAROUND(klass, table, false) -#define RCLASS_SET_M_TBL(klass, table) RCLASS_SET_M_TBL_WORKAROUND(klass, table, true) - static inline void -RCLASS_SET_M_TBL_WORKAROUND(VALUE klass, struct rb_id_table *table, bool check_promoted) +RCLASS_SET_M_TBL(VALUE klass, struct rb_id_table *table) { - RUBY_ASSERT(!check_promoted || !RB_OBJ_PROMOTED(klass)); RCLASSEXT_M_TBL(RCLASS_EXT_PRIME(klass)) = table; } -#define RCLASS_WRITE_M_TBL_EVEN_WHEN_PROMOTED(klass, table) RCLASS_WRITE_M_TBL_WORKAROUND(klass, table, false) -#define RCLASS_WRITE_M_TBL(klass, table) RCLASS_WRITE_M_TBL_WORKAROUND(klass, table, true) - static inline void -RCLASS_WRITE_M_TBL_WORKAROUND(VALUE klass, struct rb_id_table *table, bool check_promoted) +RCLASS_WRITE_M_TBL(VALUE klass, struct rb_id_table *table) { - RUBY_ASSERT(!check_promoted || !RB_OBJ_PROMOTED(klass)); - // TODO: add write barrier here to guard assigning m_tbl - // see commit 28a6e4ea9d9379a654a8f7c4b37fa33aa3ccd0b7 RCLASSEXT_M_TBL(RCLASS_EXT_WRITABLE(klass)) = table; } @@ -632,9 +632,9 @@ RCLASS_WRITE_CALLABLE_M_TBL(VALUE klass, struct rb_id_table *table) } static inline void -RCLASS_WRITE_CC_TBL(VALUE klass, struct rb_id_table *table) +RCLASS_WRITE_CC_TBL(VALUE klass, VALUE table) { - RCLASSEXT_CC_TBL(RCLASS_EXT_WRITABLE(klass)) = table; + RB_OBJ_ATOMIC_WRITE(klass, &RCLASSEXT_CC_TBL(RCLASS_EXT_WRITABLE(klass)), table); } static inline void @@ -658,7 +658,8 @@ RCLASS_SET_REFINED_CLASS(VALUE klass, VALUE refined) static inline rb_alloc_func_t RCLASS_ALLOCATOR(VALUE klass) { - if (RCLASS_SINGLETON_P(klass) || RB_TYPE_P(klass, T_ICLASS)) { + RBIMPL_ASSERT_TYPE(klass, T_CLASS); + if (RCLASS_SINGLETON_P(klass)) { return 0; } return RCLASS_EXT_PRIME(klass)->as.class.allocator; @@ -667,7 +668,8 @@ RCLASS_ALLOCATOR(VALUE klass) static inline void RCLASS_SET_ALLOCATOR(VALUE klass, rb_alloc_func_t allocator) { - assert(!RCLASS_SINGLETON_P(klass)); + RUBY_ASSERT(RB_TYPE_P(klass, T_CLASS)); + RUBY_ASSERT(!RCLASS_SINGLETON_P(klass)); RCLASS_EXT_PRIME(klass)->as.class.allocator = allocator; // Allocator is set only on the initial definition } @@ -714,14 +716,13 @@ RCLASS_SET_INCLUDER(VALUE iclass, VALUE klass) } static inline void -RCLASS_WRITE_SUPERCLASSES(VALUE klass, size_t depth, VALUE *superclasses, bool owns_it, bool with_self) +RCLASS_WRITE_SUPERCLASSES(VALUE klass, size_t depth, VALUE *superclasses, bool with_self) { RUBY_ASSERT(depth <= RCLASS_MAX_SUPERCLASS_DEPTH); - rb_classext_t *ext = RCLASS_EXT_WRITABLE(klass); + rb_classext_t *ext = RCLASS_EXT_PRIME(klass); RCLASSEXT_SUPERCLASS_DEPTH(ext) = depth; RCLASSEXT_SUPERCLASSES(ext) = superclasses; - RCLASSEXT_SUPERCLASSES_OWNER(ext) = owns_it; RCLASSEXT_SUPERCLASSES_WITH_SELF(ext) = with_self; } @@ -733,21 +734,21 @@ RCLASS_SET_SUBCLASSES(VALUE klass, struct rb_subclass_anchor *anchor) } static inline void -RCLASS_WRITE_NS_SUPER_SUBCLASSES(VALUE klass, rb_ns_subclasses_t *ns_subclasses) +RCLASS_WRITE_BOX_SUPER_SUBCLASSES(VALUE klass, rb_box_subclasses_t *box_subclasses) { rb_classext_t *ext = RCLASS_EXT_WRITABLE(klass); - if (RCLASSEXT_NS_SUPER_SUBCLASSES(ext)) - rb_ns_subclasses_ref_dec(RCLASSEXT_NS_SUPER_SUBCLASSES(ext)); - RCLASSEXT_NS_SUPER_SUBCLASSES(ext) = rb_ns_subclasses_ref_inc(ns_subclasses); + if (RCLASSEXT_BOX_SUPER_SUBCLASSES(ext)) + rb_box_subclasses_ref_dec(RCLASSEXT_BOX_SUPER_SUBCLASSES(ext)); + RCLASSEXT_BOX_SUPER_SUBCLASSES(ext) = rb_box_subclasses_ref_inc(box_subclasses); } static inline void -RCLASS_WRITE_NS_MODULE_SUBCLASSES(VALUE klass, rb_ns_subclasses_t *ns_subclasses) +RCLASS_WRITE_BOX_MODULE_SUBCLASSES(VALUE klass, rb_box_subclasses_t *box_subclasses) { rb_classext_t *ext = RCLASS_EXT_WRITABLE(klass); - if (RCLASSEXT_NS_MODULE_SUBCLASSES(ext)) - rb_ns_subclasses_ref_dec(RCLASSEXT_NS_MODULE_SUBCLASSES(ext)); - RCLASSEXT_NS_MODULE_SUBCLASSES(ext) = rb_ns_subclasses_ref_inc(ns_subclasses); + if (RCLASSEXT_BOX_MODULE_SUBCLASSES(ext)) + rb_box_subclasses_ref_dec(RCLASSEXT_BOX_MODULE_SUBCLASSES(ext)); + RCLASSEXT_BOX_MODULE_SUBCLASSES(ext) = rb_box_subclasses_ref_inc(box_subclasses); } static inline void @@ -756,6 +757,7 @@ RCLASS_SET_CLASSPATH(VALUE klass, VALUE classpath, bool permanent) rb_classext_t *ext = RCLASS_EXT_READABLE(klass); assert(BUILTIN_TYPE(klass) == T_CLASS || BUILTIN_TYPE(klass) == T_MODULE); assert(classpath == 0 || BUILTIN_TYPE(classpath) == T_STRING); + assert(FL_TEST_RAW(classpath, RUBY_FL_SHAREABLE)); RB_OBJ_WRITE(klass, &(RCLASSEXT_CLASSPATH(ext)), classpath); RCLASSEXT_PERMANENT_CLASSPATH(ext) = permanent; @@ -767,6 +769,7 @@ RCLASS_WRITE_CLASSPATH(VALUE klass, VALUE classpath, bool permanent) rb_classext_t *ext = RCLASS_EXT_WRITABLE(klass); assert(BUILTIN_TYPE(klass) == T_CLASS || BUILTIN_TYPE(klass) == T_MODULE); assert(classpath == 0 || BUILTIN_TYPE(classpath) == T_STRING); + assert(!RB_FL_ABLE(classpath) || FL_TEST_RAW(classpath, RUBY_FL_SHAREABLE)); RB_OBJ_WRITE(klass, &(RCLASSEXT_CLASSPATH(ext)), classpath); RCLASSEXT_PERMANENT_CLASSPATH(ext) = permanent; @@ -784,19 +787,20 @@ RCLASS_SET_ATTACHED_OBJECT(VALUE klass, VALUE attached_object) static inline void RCLASS_SET_MAX_IV_COUNT(VALUE klass, attr_index_t count) { - RCLASSEXT_MAX_IV_COUNT(RCLASS_EXT_PRIME(klass)) = count; + RCLASS_MAX_IV_COUNT(klass) = count; } static inline void -RCLASS_WRITE_MAX_IV_COUNT(VALUE klass, attr_index_t count) +RCLASS_SET_CLONED(VALUE klass, bool cloned) { - RCLASSEXT_MAX_IV_COUNT(RCLASS_EXT_WRITABLE(klass)) = count; + RCLASSEXT_CLONED(RCLASS_EXT_PRIME(klass)) = cloned; } -static inline void -RCLASS_SET_CLONED(VALUE klass, bool cloned) +static inline bool +RCLASS_INITIALIZED_P(VALUE klass) { - RCLASSEXT_CLONED(RCLASS_EXT_PRIME(klass)) = cloned; + VM_ASSERT(RB_TYPE_P(klass, T_CLASS) || RB_TYPE_P(klass, T_MODULE)); + return FL_TEST_RAW(klass, RCLASS_IS_INITIALIZED); } #endif /* INTERNAL_CLASS_H */ diff --git a/internal/cmdlineopt.h b/internal/cmdlineopt.h index 667fd6df2e..aed209e2a2 100644 --- a/internal/cmdlineopt.h +++ b/internal/cmdlineopt.h @@ -23,9 +23,6 @@ typedef struct ruby_cmdline_options { ruby_features_t warn; unsigned int dump; long backtrace_length_limit; -#if USE_ZJIT - void *zjit; -#endif const char *crash_report; @@ -42,6 +39,9 @@ typedef struct ruby_cmdline_options { #if USE_YJIT unsigned int yjit: 1; #endif +#if USE_ZJIT + unsigned int zjit: 1; +#endif } ruby_cmdline_options_t; struct ruby_opt_message { diff --git a/internal/concurrent_set.h b/internal/concurrent_set.h new file mode 100644 index 0000000000..76cbefab04 --- /dev/null +++ b/internal/concurrent_set.h @@ -0,0 +1,21 @@ +#ifndef RUBY_RACTOR_SAFE_TABLE_H +#define RUBY_RACTOR_SAFE_TABLE_H + +#include "ruby/atomic.h" +#include "ruby/ruby.h" + +struct rb_concurrent_set_funcs { + VALUE (*hash)(VALUE key); + bool (*cmp)(VALUE a, VALUE b); + VALUE (*create)(VALUE key, void *data); + void (*free)(VALUE key); +}; + +VALUE rb_concurrent_set_new(const struct rb_concurrent_set_funcs *funcs, int capacity); +rb_atomic_t rb_concurrent_set_size(VALUE set_obj); +VALUE rb_concurrent_set_find(VALUE *set_obj_ptr, VALUE key); +VALUE rb_concurrent_set_find_or_insert(VALUE *set_obj_ptr, VALUE key, void *data); +VALUE rb_concurrent_set_delete_by_identity(VALUE set_obj, VALUE key); +void rb_concurrent_set_foreach_with_replace(VALUE set_obj, int (*callback)(VALUE *key, void *data), void *data); + +#endif diff --git a/internal/cont.h b/internal/cont.h index 3c2528a02a..dcf6f820a3 100644 --- a/internal/cont.h +++ b/internal/cont.h @@ -31,5 +31,4 @@ VALUE rb_fiber_inherit_storage(struct rb_execution_context_struct *ec, struct rb VALUE rb_fiberptr_self(struct rb_fiber_struct *fiber); unsigned int rb_fiberptr_blocking(struct rb_fiber_struct *fiber); struct rb_execution_context_struct * rb_fiberptr_get_ec(struct rb_fiber_struct *fiber); - #endif /* INTERNAL_CONT_H */ diff --git a/internal/encoding.h b/internal/encoding.h index 29d7dc5c2d..38bf8fc9da 100644 --- a/internal/encoding.h +++ b/internal/encoding.h @@ -11,7 +11,6 @@ #include "ruby/ruby.h" /* for ID */ #include "ruby/encoding.h" /* for rb_encoding */ -#define rb_enc_autoload_p(enc) (!rb_enc_mbmaxlen(enc)) #define rb_is_usascii_enc(enc) ((enc) == rb_usascii_encoding()) #define rb_is_ascii8bit_enc(enc) ((enc) == rb_ascii8bit_encoding()) #define rb_is_locale_enc(enc) ((enc) == rb_locale_encoding()) @@ -24,11 +23,13 @@ rb_encoding *rb_enc_check_str(VALUE str1, VALUE str2); int rb_encdb_replicate(const char *alias, const char *orig); int rb_encdb_alias(const char *alias, const char *orig); int rb_enc_autoload(rb_encoding *enc); +bool rb_enc_autoload_p(rb_encoding *enc); int rb_encdb_dummy(const char *name); void rb_encdb_declare(const char *name); void rb_enc_set_base(const char *name, const char *orig); int rb_enc_set_dummy(int index); void rb_enc_raw_set(VALUE obj, rb_encoding *enc); +int rb_enc_registered(const char *name); PUREFUNC(int rb_data_is_encoding(VALUE obj)); diff --git a/internal/error.h b/internal/error.h index 5d53f96b8e..ae9a13fcec 100644 --- a/internal/error.h +++ b/internal/error.h @@ -75,11 +75,11 @@ PRINTF_ARGS(void rb_warn_deprecated_to_remove(const char *removal, const char *f PRINTF_ARGS(void rb_warn_reserved_name(const char *removal, const char *fmt, ...), 2, 3); #if RUBY_DEBUG # include "ruby/version.h" -# define RUBY_VERSION_SINCE(major, minor) (RUBY_API_VERSION_CODE >= (major * 10000) + (minor) * 100) -# define RUBY_VERSION_BEFORE(major, minor) (RUBY_API_VERSION_CODE < (major * 10000) + (minor) * 100) +# define RUBY_VERSION_SINCE(major, minor) (RUBY_API_VERSION_CODE >= (major) * 10000 + (minor) * 100) +# define RUBY_VERSION_BEFORE(major, minor) (RUBY_API_VERSION_CODE < (major) * 10000 + (minor) * 100) # if defined(RBIMPL_WARNING_PRAGMA0) # define RBIMPL_TODO0(x) RBIMPL_WARNING_PRAGMA0(message(x)) -# elif RBIMPL_COMPILER_SINCE(MSVC, 12, 0, 0) +# elif RBIMPL_COMPILER_IS(MSVC) # define RBIMPL_TODO0(x) __pragma(message(x)) # endif @@ -235,10 +235,18 @@ rb_key_err_raise(VALUE mesg, VALUE recv, VALUE name) rb_exc_raise(exc); } +RBIMPL_ATTR_NONNULL((2)) static inline bool rb_typeddata_is_instance_of_inline(VALUE obj, const rb_data_type_t *data_type) { - return RB_TYPE_P(obj, T_DATA) && RTYPEDDATA_P(obj) && (RTYPEDDATA_TYPE(obj) == data_type); + return rbimpl_obj_typeddata_p(obj) && (RTYPEDDATA_TYPE(obj) == data_type); } +typedef enum { + rb_stack_overflow_prevention = 0, // VM stack overflow or about to machine stack overflow + rb_stack_overflow_signal = 1, // machine stack overflow but may be recoverable + rb_stack_overflow_fatal = 2, // fatal machine stack overflow +} ruby_stack_overflow_critical_level; +NORETURN(void rb_ec_stack_overflow(struct rb_execution_context_struct *ec, ruby_stack_overflow_critical_level crit)); + #endif /* INTERNAL_ERROR_H */ diff --git a/internal/eval.h b/internal/eval.h index a8b649e509..17ade0a7f1 100644 --- a/internal/eval.h +++ b/internal/eval.h @@ -11,6 +11,7 @@ * header (related to this file, but not the same role). */ #include "ruby/ruby.h" /* for ID */ +#include "vm_core.h" /* for ID */ #define id_signo ruby_static_id_signo #define id_status ruby_static_id_status @@ -26,9 +27,11 @@ extern ID ruby_static_id_status; VALUE rb_refinement_module_get_refined_class(VALUE module); void rb_class_modify_check(VALUE); NORETURN(VALUE rb_f_raise(int argc, VALUE *argv)); +VALUE rb_exception_setup(int argc, VALUE *argv); void rb_refinement_setup(struct rb_refinements_data *data, VALUE module, VALUE klass); void rb_vm_using_module(VALUE module); VALUE rb_top_main_class(const char *method); +VALUE rb_ec_ensure(rb_execution_context_t *ec, VALUE (*b_proc)(VALUE), VALUE data1, VALUE (*e_proc)(VALUE), VALUE data2); /* eval_error.c */ VALUE rb_get_backtrace(VALUE info); diff --git a/internal/gc.h b/internal/gc.h index 06103ca25f..ee1f390e10 100644 --- a/internal/gc.h +++ b/internal/gc.h @@ -122,10 +122,12 @@ const char *rb_raw_obj_info(char *const buff, const size_t buff_size, VALUE obj) struct rb_execution_context_struct; /* in vm_core.h */ struct rb_objspace; /* in vm_core.h */ -#define NEWOBJ_OF(var, T, c, f, s, ec) \ +#define NEWOBJ_OF_WITH_SHAPE(var, T, c, f, shape_id, s, ec) \ T *(var) = (T *)(((f) & FL_WB_PROTECTED) ? \ - rb_wb_protected_newobj_of((ec ? ec : GET_EC()), (c), (f) & ~FL_WB_PROTECTED, s) : \ - rb_wb_unprotected_newobj_of((c), (f), s)) + rb_wb_protected_newobj_of((ec ? ec : GET_EC()), (c), (f) & ~FL_WB_PROTECTED, shape_id, s) : \ + rb_wb_unprotected_newobj_of((c), (f), shape_id, s)) + +#define NEWOBJ_OF(var, T, c, f, s, ec) NEWOBJ_OF_WITH_SHAPE(var, T, c, f, 0 /* ROOT_SHAPE_ID */, s, ec) #ifndef RB_GC_OBJECT_METADATA_ENTRY_DEFINED # define RB_GC_OBJECT_METADATA_ENTRY_DEFINED @@ -202,6 +204,7 @@ static inline void *ruby_sized_xrealloc_inlined(void *ptr, size_t new_size, size static inline void *ruby_sized_xrealloc2_inlined(void *ptr, size_t new_count, size_t elemsiz, size_t old_count) RUBY_ATTR_RETURNS_NONNULL RUBY_ATTR_ALLOC_SIZE((2, 3)); static inline void ruby_sized_xfree_inlined(void *ptr, size_t size); void rb_gc_obj_id_moved(VALUE obj); +void rb_gc_register_pinning_obj(VALUE obj); void *rb_gc_ractor_cache_alloc(rb_ractor_t *ractor); void rb_gc_ractor_cache_free(void *cache); @@ -212,8 +215,8 @@ size_t rb_gc_heap_id_for_size(size_t size); void rb_gc_mark_and_move(VALUE *ptr); -void rb_gc_mark_weak(VALUE *ptr); -void rb_gc_remove_weak(VALUE parent_obj, VALUE *ptr); +void rb_gc_declare_weak_references(VALUE obj); +bool rb_gc_handle_weak_references_alive_p(VALUE obj); void rb_gc_ref_update_table_values_only(st_table *tbl); @@ -248,8 +251,8 @@ VALUE rb_gc_disable_no_rest(void); /* gc.c (export) */ const char *rb_objspace_data_type_name(VALUE obj); -VALUE rb_wb_protected_newobj_of(struct rb_execution_context_struct *, VALUE, VALUE, size_t); -VALUE rb_wb_unprotected_newobj_of(VALUE, VALUE, size_t); +VALUE rb_wb_protected_newobj_of(struct rb_execution_context_struct *, VALUE, VALUE, uint32_t /* shape_id_t */, size_t); +VALUE rb_wb_unprotected_newobj_of(VALUE, VALUE, uint32_t /* shape_id_t */, size_t); size_t rb_obj_memsize_of(VALUE); struct rb_gc_object_metadata_entry *rb_gc_object_metadata(VALUE obj); void rb_gc_mark_values(long n, const VALUE *values); @@ -264,6 +267,26 @@ int rb_gc_modular_gc_loaded_p(void); RUBY_SYMBOL_EXPORT_END +static inline VALUE +rb_obj_atomic_write( + VALUE a, VALUE *slot, VALUE b, + RBIMPL_ATTR_MAYBE_UNUSED() + const char *filename, + RBIMPL_ATTR_MAYBE_UNUSED() + int line) +{ +#ifdef RGENGC_LOGGING_WRITE + RGENGC_LOGGING_WRITE(a, slot, b, filename, line); +#endif + + RUBY_ATOMIC_VALUE_SET(*slot, b); + + rb_obj_written(a, RUBY_Qundef /* ignore `oldv' now */, b, filename, line); + return a; +} +#define RB_OBJ_ATOMIC_WRITE(old, slot, young) \ + RBIMPL_CAST(rb_obj_atomic_write((VALUE)(old), (VALUE *)(slot), (VALUE)(young), __FILE__, __LINE__)) + int rb_ec_stack_check(struct rb_execution_context_struct *ec); void rb_gc_writebarrier_remember(VALUE obj); const char *rb_obj_info(VALUE obj); @@ -331,4 +354,8 @@ ruby_sized_realloc_n(void *ptr, size_t new_count, size_t element_size, size_t ol #define ruby_sized_xrealloc ruby_sized_xrealloc_inlined #define ruby_sized_xrealloc2 ruby_sized_xrealloc2_inlined #define ruby_sized_xfree ruby_sized_xfree_inlined + +void rb_gc_verify_shareable(VALUE); +bool rb_gc_checking_shareable(void); + #endif /* INTERNAL_GC_H */ diff --git a/internal/imemo.h b/internal/imemo.h index 305d12d240..31cc0be35a 100644 --- a/internal/imemo.h +++ b/internal/imemo.h @@ -37,11 +37,10 @@ enum imemo_type { imemo_ment = 6, imemo_iseq = 7, imemo_tmpbuf = 8, - imemo_ast = 9, // Obsolete due to the universal parser - imemo_parser_strterm = 10, - imemo_callinfo = 11, - imemo_callcache = 12, - imemo_constcache = 13, + imemo_callinfo = 10, + imemo_callcache = 11, + imemo_constcache = 12, + imemo_fields = 13, }; /* CREF (Class REFerence) is defined in method.h */ @@ -94,9 +93,7 @@ struct vm_ifunc { struct rb_imemo_tmpbuf_struct { VALUE flags; - VALUE reserved; VALUE *ptr; /* malloc'ed buffer */ - struct rb_imemo_tmpbuf_struct *next; /* next imemo */ size_t cnt; /* buffer size in VALUE */ }; @@ -117,7 +114,8 @@ struct MEMO { } u3; }; -#define IMEMO_NEW(T, type, v0) ((T *)rb_imemo_new((type), (v0), sizeof(T))) +#define IMEMO_NEW(T, type, v0) ((T *)rb_imemo_new((type), (v0), sizeof(T), false)) +#define SHAREABLE_IMEMO_NEW(T, type, v0) ((T *)rb_imemo_new((type), (v0), sizeof(T), true)) /* ment is in method.h */ @@ -134,42 +132,27 @@ struct MEMO { #ifndef RUBY_RUBYPARSER_H typedef struct rb_imemo_tmpbuf_struct rb_imemo_tmpbuf_t; #endif -rb_imemo_tmpbuf_t *rb_imemo_tmpbuf_parser_heap(void *buf, rb_imemo_tmpbuf_t *old_heap, size_t cnt); +VALUE rb_imemo_new(enum imemo_type type, VALUE v0, size_t size, bool is_shareable); +VALUE rb_imemo_tmpbuf_new(void); +struct MEMO *rb_imemo_memo_new(VALUE a, VALUE b, VALUE c); struct vm_ifunc *rb_vm_ifunc_new(rb_block_call_func_t func, const void *data, int min_argc, int max_argc); static inline enum imemo_type imemo_type(VALUE imemo); static inline int imemo_type_p(VALUE imemo, enum imemo_type imemo_type); static inline bool imemo_throw_data_p(VALUE imemo); static inline struct vm_ifunc *rb_vm_ifunc_proc_new(rb_block_call_func_t func, const void *data); -static inline VALUE rb_imemo_tmpbuf_auto_free_pointer(void); static inline void *RB_IMEMO_TMPBUF_PTR(VALUE v); static inline void *rb_imemo_tmpbuf_set_ptr(VALUE v, void *ptr); -static inline VALUE rb_imemo_tmpbuf_auto_free_pointer_new_from_an_RString(VALUE str); static inline void MEMO_V1_SET(struct MEMO *m, VALUE v); static inline void MEMO_V2_SET(struct MEMO *m, VALUE v); size_t rb_imemo_memsize(VALUE obj); -void rb_cc_table_mark(VALUE klass); void rb_imemo_mark_and_move(VALUE obj, bool reference_updating); -void rb_cc_table_free(VALUE klass); -void rb_cc_tbl_free(struct rb_id_table *cc_tbl, VALUE klass); void rb_imemo_free(VALUE obj); RUBY_SYMBOL_EXPORT_BEGIN -VALUE rb_imemo_new(enum imemo_type type, VALUE v0, size_t size); const char *rb_imemo_name(enum imemo_type type); RUBY_SYMBOL_EXPORT_END -static inline struct MEMO * -MEMO_NEW(VALUE a, VALUE b, VALUE c) -{ - struct MEMO *memo = IMEMO_NEW(struct MEMO, imemo_memo, 0); - *((VALUE *)&memo->v1) = a; - *((VALUE *)&memo->v2) = b; - *((VALUE *)&memo->u3.value) = c; - - return memo; -} - static inline enum imemo_type imemo_type(VALUE imemo) { @@ -205,12 +188,6 @@ rb_vm_ifunc_proc_new(rb_block_call_func_t func, const void *data) return rb_vm_ifunc_new(func, data, 0, UNLIMITED_ARGUMENTS); } -static inline VALUE -rb_imemo_tmpbuf_auto_free_pointer(void) -{ - return rb_imemo_new(imemo_tmpbuf, 0, sizeof(rb_imemo_tmpbuf_t)); -} - static inline void * RB_IMEMO_TMPBUF_PTR(VALUE v) { @@ -225,7 +202,7 @@ rb_imemo_tmpbuf_set_ptr(VALUE v, void *ptr) } static inline VALUE -rb_imemo_tmpbuf_auto_free_pointer_new_from_an_RString(VALUE str) +rb_imemo_tmpbuf_new_from_an_RString(VALUE str) { const void *src; VALUE imemo; @@ -235,7 +212,7 @@ rb_imemo_tmpbuf_auto_free_pointer_new_from_an_RString(VALUE str) StringValue(str); /* create tmpbuf to keep the pointer before xmalloc */ - imemo = rb_imemo_tmpbuf_auto_free_pointer(); + imemo = rb_imemo_tmpbuf_new(); tmpbuf = (rb_imemo_tmpbuf_t *)imemo; len = RSTRING_LEN(str); src = RSTRING_PTR(str); @@ -257,4 +234,79 @@ MEMO_V2_SET(struct MEMO *m, VALUE v) RB_OBJ_WRITE(m, &m->v2, v); } +struct rb_fields { + struct RBasic basic; + union { + struct { + VALUE fields[1]; + } embed; + struct { + VALUE *ptr; + } external; + struct { + // Note: the st_table could be embedded, but complex T_CLASS should be rare to + // non-existent, so not really worth the trouble. + st_table *table; + } complex; + } as; +}; + +// IMEMO/fields and T_OBJECT have exactly the same layout. +// This is useful for JIT and common codepaths. +#define OBJ_FIELD_HEAP ROBJECT_HEAP +STATIC_ASSERT(imemo_fields_flags, OBJ_FIELD_HEAP == IMEMO_FL_USER0); +STATIC_ASSERT(imemo_fields_embed_offset, offsetof(struct RObject, as.ary) == offsetof(struct rb_fields, as.embed.fields)); +STATIC_ASSERT(imemo_fields_external_offset, offsetof(struct RObject, as.heap.fields) == offsetof(struct rb_fields, as.external.ptr)); +STATIC_ASSERT(imemo_fields_complex_offset, offsetof(struct RObject, as.heap.fields) == offsetof(struct rb_fields, as.complex.table)); + +#define IMEMO_OBJ_FIELDS(fields) ((struct rb_fields *)fields) + +VALUE rb_imemo_fields_new(VALUE owner, size_t capa, bool shareable); +VALUE rb_imemo_fields_new_complex(VALUE owner, size_t capa, bool shareable); +VALUE rb_imemo_fields_new_complex_tbl(VALUE owner, st_table *tbl, bool shareable); +VALUE rb_imemo_fields_clone(VALUE fields_obj); +void rb_imemo_fields_clear(VALUE fields_obj); + +static inline VALUE +rb_imemo_fields_owner(VALUE fields_obj) +{ + RUBY_ASSERT(IMEMO_TYPE_P(fields_obj, imemo_fields)); + + return CLASS_OF(fields_obj); +} + +static inline VALUE * +rb_imemo_fields_ptr(VALUE fields_obj) +{ + if (!fields_obj) { + return NULL; + } + + RUBY_ASSERT(IMEMO_TYPE_P(fields_obj, imemo_fields) || RB_TYPE_P(fields_obj, T_OBJECT)); + + if (UNLIKELY(FL_TEST_RAW(fields_obj, OBJ_FIELD_HEAP))) { + return IMEMO_OBJ_FIELDS(fields_obj)->as.external.ptr; + } + else { + return IMEMO_OBJ_FIELDS(fields_obj)->as.embed.fields; + } +} + +static inline st_table * +rb_imemo_fields_complex_tbl(VALUE fields_obj) +{ + if (!fields_obj) { + return NULL; + } + + RUBY_ASSERT(IMEMO_TYPE_P(fields_obj, imemo_fields) || RB_TYPE_P(fields_obj, T_OBJECT)); + RUBY_ASSERT(FL_TEST_RAW(fields_obj, OBJ_FIELD_HEAP)); + + // Some codepaths unconditionally access the fields_ptr, and assume it can be used as st_table if the + // shape is too_complex. + RUBY_ASSERT((st_table *)rb_imemo_fields_ptr(fields_obj) == IMEMO_OBJ_FIELDS(fields_obj)->as.complex.table); + + return IMEMO_OBJ_FIELDS(fields_obj)->as.complex.table; +} + #endif /* INTERNAL_IMEMO_H */ diff --git a/internal/inits.h b/internal/inits.h index 03de289dd4..dee818285c 100644 --- a/internal/inits.h +++ b/internal/inits.h @@ -9,6 +9,10 @@ * @brief Internal header aggregating init functions. */ +/* box.c */ +void Init_enable_box(void); +void Init_root_box(void); + /* class.c */ void Init_class_hierarchy(void); diff --git a/internal/io.h b/internal/io.h index b1e9052b66..b81774e0a7 100644 --- a/internal/io.h +++ b/internal/io.h @@ -25,7 +25,7 @@ struct rb_io_blocking_operation { // The linked list data structure. struct ccan_list_node list; - // The execution context of the blocking operation: + // The execution context of the blocking operation. struct rb_execution_context_struct *ec; }; @@ -132,7 +132,7 @@ struct rb_io { struct rb_execution_context_struct *closing_ec; VALUE wakeup_mutex; - // The fork generation of the the blocking operations list. + // The fork generation of the blocking operations list. rb_serial_t fork_generation; }; diff --git a/internal/namespace.h b/internal/namespace.h deleted file mode 100644 index ad1507b50c..0000000000 --- a/internal/namespace.h +++ /dev/null @@ -1,78 +0,0 @@ -#ifndef INTERNAL_NAMESPACE_H /*-*-C-*-vi:se ft=c:*/ -#define INTERNAL_NAMESPACE_H - -#include "ruby/ruby.h" /* for VALUE */ - -/** - * @author Ruby developers <ruby-core@ruby-lang.org> - * @copyright This file is a part of the programming language Ruby. - * Permission is hereby granted, to either redistribute and/or - * modify this file, provided that the conditions mentioned in the - * file COPYING are met. Consult the file for details. - * @brief Internal header for Namespace. - */ -struct rb_namespace_struct { - /* - * To retrieve Namespace object that provides #require and so on. - * That is used from load.c, etc., that uses rb_namespace_t internally. - */ - VALUE ns_object; - long ns_id; // namespace id to generate ext filenames - - VALUE top_self; - - VALUE load_path; - VALUE load_path_snapshot; - VALUE load_path_check_cache; - VALUE expanded_load_path; - VALUE loaded_features; - VALUE loaded_features_snapshot; - VALUE loaded_features_realpaths; - VALUE loaded_features_realpath_map; - struct st_table *loaded_features_index; - struct st_table *loading_table; - VALUE ruby_dln_libmap; - - VALUE gvar_tbl; - - bool is_builtin; - bool is_user; - bool is_optional; -}; -typedef struct rb_namespace_struct rb_namespace_t; - -#define NAMESPACE_BUILTIN_P(ns) (ns && ns->is_builtin) -#define NAMESPACE_USER_P(ns) (ns && ns->is_user) -#define NAMESPACE_OPTIONAL_P(ns) (ns && ns->is_optional) -#define NAMESPACE_MAIN_P(ns) (ns && ns->is_user && !ns->is_optional) - -#define NAMESPACE_METHOD_DEFINITION(mdef) (mdef ? mdef->ns : NULL) -#define NAMESPACE_METHOD_ENTRY(me) (me ? NAMESPACE_METHOD_DEFINITION(me->def) : NULL) -#define NAMESPACE_CC(cc) (cc ? NAMESPACE_METHOD_ENTRY(cc->cme_) : NULL) -#define NAMESPACE_CC_ENTRIES(ccs) (ccs ? NAMESPACE_METHOD_ENTRY(ccs->cme) : NULL) - -int rb_namespace_available(void); -void rb_namespace_enable_builtin(void); -void rb_namespace_disable_builtin(void); -void rb_namespace_push_loading_namespace(const rb_namespace_t *); -void rb_namespace_pop_loading_namespace(const rb_namespace_t *); -rb_namespace_t * rb_root_namespace(void); -const rb_namespace_t *rb_builtin_namespace(void); -rb_namespace_t * rb_main_namespace(void); -const rb_namespace_t * rb_definition_namespace(void); -const rb_namespace_t * rb_loading_namespace(void); -const rb_namespace_t * rb_current_namespace(void); -VALUE rb_current_namespace_details(VALUE); - -void rb_namespace_entry_mark(void *); - -rb_namespace_t * rb_get_namespace_t(VALUE ns); -VALUE rb_get_namespace_object(rb_namespace_t *ns); -typedef VALUE namespace_exec_func(VALUE arg); -VALUE rb_namespace_exec(const rb_namespace_t *ns, namespace_exec_func *func, VALUE arg); - -VALUE rb_namespace_local_extension(VALUE namespace, VALUE fname, VALUE path); - -void rb_initialize_main_namespace(void); - -#endif /* INTERNAL_NAMESPACE_H */ diff --git a/internal/numeric.h b/internal/numeric.h index 6406cfc2fa..d3905f048c 100644 --- a/internal/numeric.h +++ b/internal/numeric.h @@ -85,6 +85,7 @@ VALUE rb_int_cmp(VALUE x, VALUE y); VALUE rb_int_equal(VALUE x, VALUE y); VALUE rb_int_divmod(VALUE x, VALUE y); VALUE rb_int_and(VALUE x, VALUE y); +VALUE rb_int_xor(VALUE x, VALUE y); VALUE rb_int_lshift(VALUE x, VALUE y); VALUE rb_int_rshift(VALUE x, VALUE y); VALUE rb_int_div(VALUE x, VALUE y); @@ -126,6 +127,54 @@ VALUE rb_int_bit_length(VALUE num); VALUE rb_int_uminus(VALUE num); VALUE rb_int_comp(VALUE num); +// Unified 128-bit integer structures that work with or without native support: +union rb_uint128 { +#ifdef WORDS_BIGENDIAN + struct { + uint64_t high; + uint64_t low; + } parts; +#else + struct { + uint64_t low; + uint64_t high; + } parts; +#endif +#ifdef HAVE_UINT128_T + uint128_t value; +#endif +}; +typedef union rb_uint128 rb_uint128_t; + +union rb_int128 { +#ifdef WORDS_BIGENDIAN + struct { + uint64_t high; + uint64_t low; + } parts; +#else + struct { + uint64_t low; + uint64_t high; + } parts; +#endif +#ifdef HAVE_UINT128_T + int128_t value; +#endif +}; +typedef union rb_int128 rb_int128_t; + +union uint128_int128_conversion { + rb_uint128_t uint128; + rb_int128_t int128; +}; + +// Conversion functions for 128-bit integers: +rb_uint128_t rb_numeric_to_uint128(VALUE x); +rb_int128_t rb_numeric_to_int128(VALUE x); +VALUE rb_uint128_to_numeric(rb_uint128_t n); +VALUE rb_int128_to_numeric(rb_int128_t n); + static inline bool INT_POSITIVE_P(VALUE num) { diff --git a/internal/range.h b/internal/range.h index 2394937bf8..80493ce13e 100644 --- a/internal/range.h +++ b/internal/range.h @@ -18,19 +18,19 @@ static inline VALUE RANGE_EXCL(VALUE r); static inline VALUE RANGE_BEG(VALUE r) { - return RSTRUCT(r)->as.ary[0]; + return RSTRUCT_GET_RAW(r, 0); } static inline VALUE RANGE_END(VALUE r) { - return RSTRUCT_GET(r, 1); + return RSTRUCT_GET_RAW(r, 1); } static inline VALUE RANGE_EXCL(VALUE r) { - return RSTRUCT_GET(r, 2); + return RSTRUCT_GET_RAW(r, 2); } VALUE diff --git a/internal/re.h b/internal/re.h index 2788f8b42a..593e5c464f 100644 --- a/internal/re.h +++ b/internal/re.h @@ -25,4 +25,9 @@ int rb_match_count(VALUE match); VALUE rb_reg_new_ary(VALUE ary, int options); VALUE rb_reg_last_defined(VALUE match); +#define ARG_REG_OPTION_MASK \ + (ONIG_OPTION_IGNORECASE|ONIG_OPTION_MULTILINE|ONIG_OPTION_EXTEND) +#define ARG_ENCODING_FIXED 16 +#define ARG_ENCODING_NONE 32 + #endif /* INTERNAL_RE_H */ diff --git a/internal/sanitizers.h b/internal/sanitizers.h index 279cbbe069..feafb4e616 100644 --- a/internal/sanitizers.h +++ b/internal/sanitizers.h @@ -29,6 +29,13 @@ # endif #endif +#ifdef HAVE_SANITIZER_TSAN_INTERFACE_H +# if __has_feature(thread_sanitizer) || defined(__SANITIZE_THREAD__) +# define RUBY_TSAN_ENABLED +# include <sanitizer/tsan_interface.h> +# endif +#endif + #include "ruby/internal/stdbool.h" /* for bool */ #include "ruby/ruby.h" /* for VALUE */ @@ -42,6 +49,9 @@ #elif defined(RUBY_MSAN_ENABLED) # define ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS(x) \ __attribute__((__no_sanitize__("memory"), __noinline__)) x +#elif defined(RUBY_TSAN_ENABLED) +# define ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS(x) \ + __attribute__((__no_sanitize__("thread"), __noinline__)) x #elif defined(NO_SANITIZE_ADDRESS) # define ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS(x) \ NO_SANITIZE_ADDRESS(NOINLINE(x)) @@ -127,6 +137,7 @@ asan_poison_memory_region(const volatile void *ptr, size_t size) #define asan_poison_object_if(ptr, obj) ((void)(ptr), (void)(obj)) #endif +#ifdef RUBY_ASAN_ENABLED RUBY_SYMBOL_EXPORT_BEGIN /** * This is a variant of asan_poison_memory_region that takes a VALUE. @@ -153,6 +164,11 @@ void *rb_asan_poisoned_object_p(VALUE obj); void rb_asan_unpoison_object(VALUE obj, bool newobj_p); RUBY_SYMBOL_EXPORT_END +#else +# define rb_asan_poison_object(obj) ((void)obj) +# define rb_asan_poisoned_object_p(obj) ((void)obj, NULL) +# define rb_asan_unpoison_object(obj, newobj_p) ((void)obj, (void)newobj_p) +#endif /** * This function asserts that a (formally poisoned) memory region from ptr to diff --git a/internal/set_table.h b/internal/set_table.h index 7c03de2060..3c29abb4f5 100644 --- a/internal/set_table.h +++ b/internal/set_table.h @@ -1,7 +1,7 @@ #ifndef INTERNAL_SET_TABLE_H #define INTERNAL_SET_TABLE_H -#include "include/ruby/st.h" +#include "ruby/st.h" struct set_table_entry; @@ -15,13 +15,16 @@ struct set_table { const struct st_hash_type *type; /* Number of entries currently in the table. */ st_index_t num_entries; - /* Array of bins used for access by keys. */ - st_index_t *bins; + /* Start and bound index of entries in array entries. entries_starts and entries_bound are in interval [0,allocated_entries]. */ st_index_t entries_start, entries_bound; - /* Array of size 2^entry_power. */ + + /** + * Array of size 2^entry_power. + * Followed by st_index_t *bins, Array of bins used for access by keys. + */ set_table_entry *entries; }; @@ -37,24 +40,26 @@ size_t rb_set_table_size(const struct set_table *tbl); set_table *rb_set_init_table_with_size(set_table *tab, const struct st_hash_type *, st_index_t); #define set_init_numtable rb_set_init_numtable set_table *rb_set_init_numtable(void); -#define set_delete rb_set_delete -int rb_set_delete(set_table *, st_data_t *); /* returns 0:notfound 1:deleted */ +#define set_init_numtable_with_size rb_set_init_numtable_with_size +set_table *rb_set_init_numtable_with_size(st_index_t size); +#define set_table_delete rb_set_table_delete +int rb_set_table_delete(set_table *, st_data_t *); /* returns 0:notfound 1:deleted */ #define set_insert rb_set_insert int rb_set_insert(set_table *, st_data_t); -#define set_lookup rb_set_lookup -int rb_set_lookup(set_table *, st_data_t); +#define set_table_lookup rb_set_table_lookup +int rb_set_table_lookup(set_table *, st_data_t); #define set_foreach_with_replace rb_set_foreach_with_replace int rb_set_foreach_with_replace(set_table *tab, set_foreach_check_callback_func *func, set_update_callback_func *replace, st_data_t arg); -#define set_foreach rb_set_foreach -int rb_set_foreach(set_table *, set_foreach_callback_func *, st_data_t); +#define set_table_foreach rb_set_table_foreach +int rb_set_table_foreach(set_table *, set_foreach_callback_func *, st_data_t); #define set_foreach_check rb_set_foreach_check int rb_set_foreach_check(set_table *, set_foreach_check_callback_func *, st_data_t, st_data_t); #define set_keys rb_set_keys st_index_t rb_set_keys(set_table *table, st_data_t *keys, st_index_t size); #define set_free_table rb_set_free_table void rb_set_free_table(set_table *); -#define set_clear rb_set_clear -void rb_set_clear(set_table *); +#define set_table_clear rb_set_table_clear +void rb_set_table_clear(set_table *); #define set_copy rb_set_copy set_table *rb_set_copy(set_table *new_table, set_table *old_table); #define set_memsize rb_set_memsize diff --git a/internal/signal.h b/internal/signal.h index 2363bf412c..904747e226 100644 --- a/internal/signal.h +++ b/internal/signal.h @@ -19,6 +19,7 @@ void (*ruby_posix_signal(int, void (*)(int)))(int); RUBY_SYMBOL_EXPORT_BEGIN /* signal.c (export) */ +void rb_signal_atfork(void); RUBY_SYMBOL_EXPORT_END #endif /* INTERNAL_SIGNAL_H */ diff --git a/internal/st.h b/internal/st.h index a26b224505..c220edd9f0 100644 --- a/internal/st.h +++ b/internal/st.h @@ -1,7 +1,7 @@ #ifndef INTERNAL_ST_H #define INTERNAL_ST_H -#include "include/ruby/st.h" +#include "ruby/st.h" st_table *rb_st_replace(st_table *new_tab, st_table *old_tab); #define st_replace rb_st_replace diff --git a/internal/string.h b/internal/string.h index 50561924f2..d6fea62061 100644 --- a/internal/string.h +++ b/internal/string.h @@ -30,6 +30,7 @@ enum ruby_rstring_private_flags { #endif /* string.c */ +VALUE rb_str_dup_m(VALUE str); VALUE rb_fstring(VALUE); VALUE rb_fstring_cstr(const char *str); VALUE rb_fstring_enc_new(const char *ptr, long len, rb_encoding *enc); diff --git a/internal/struct.h b/internal/struct.h index a8c773b730..d3c8157393 100644 --- a/internal/struct.h +++ b/internal/struct.h @@ -11,10 +11,23 @@ #include "ruby/internal/stdbool.h" /* for bool */ #include "ruby/ruby.h" /* for struct RBasic */ +/* Flags of RStruct + * + * 1-7: RSTRUCT_EMBED_LEN + * If non-zero, the struct is embedded (its contents follow the + * header, rather than being on a separately allocated buffer) and + * these bits are the length of the Struct. + * 8: RSTRUCT_GEN_FIELDS + * The struct is embedded and has no space left to store the + * IMEMO/fields reference. Any ivar this struct may have will be in + * the generic_fields_tbl. This flag doesn't imply the struct has + * ivars. + */ enum { RSTRUCT_EMBED_LEN_MASK = RUBY_FL_USER7 | RUBY_FL_USER6 | RUBY_FL_USER5 | RUBY_FL_USER4 | RUBY_FL_USER3 | RUBY_FL_USER2 | RUBY_FL_USER1, RSTRUCT_EMBED_LEN_SHIFT = (RUBY_FL_USHIFT+1), + RSTRUCT_GEN_FIELDS = RUBY_FL_USER8, }; struct RStruct { @@ -23,6 +36,7 @@ struct RStruct { struct { long len; const VALUE *ptr; + VALUE fields_obj; } heap; /* This is a length 1 array because: * 1. GCC has a bug that does not optimize C flexible array members @@ -35,36 +49,16 @@ struct RStruct { #define RSTRUCT(obj) ((struct RStruct *)(obj)) -#ifdef RSTRUCT_LEN -# undef RSTRUCT_LEN -#endif - -#ifdef RSTRUCT_PTR -# undef RSTRUCT_PTR -#endif - -#ifdef RSTRUCT_SET -# undef RSTRUCT_SET -#endif - -#ifdef RSTRUCT_GET -# undef RSTRUCT_GET -#endif - -#define RSTRUCT_LEN internal_RSTRUCT_LEN -#define RSTRUCT_SET internal_RSTRUCT_SET -#define RSTRUCT_GET internal_RSTRUCT_GET - /* struct.c */ VALUE rb_struct_init_copy(VALUE copy, VALUE s); VALUE rb_struct_lookup(VALUE s, VALUE idx); VALUE rb_struct_s_keyword_init(VALUE klass); static inline long RSTRUCT_EMBED_LEN(VALUE st); -static inline long RSTRUCT_LEN(VALUE st); +static inline long RSTRUCT_LEN_RAW(VALUE st); static inline int RSTRUCT_LENINT(VALUE st); static inline const VALUE *RSTRUCT_CONST_PTR(VALUE st); -static inline void RSTRUCT_SET(VALUE st, long k, VALUE v); -static inline VALUE RSTRUCT_GET(VALUE st, long k); +static inline void RSTRUCT_SET_RAW(VALUE st, long k, VALUE v); +static inline VALUE RSTRUCT_GET_RAW(VALUE st, long k); static inline long RSTRUCT_EMBED_LEN(VALUE st) @@ -75,7 +69,7 @@ RSTRUCT_EMBED_LEN(VALUE st) } static inline long -RSTRUCT_LEN(VALUE st) +RSTRUCT_LEN_RAW(VALUE st) { if (FL_TEST_RAW(st, RSTRUCT_EMBED_LEN_MASK)) { return RSTRUCT_EMBED_LEN(st); @@ -88,7 +82,7 @@ RSTRUCT_LEN(VALUE st) static inline int RSTRUCT_LENINT(VALUE st) { - return rb_long2int(RSTRUCT_LEN(st)); + return rb_long2int(RSTRUCT_LEN_RAW(st)); } static inline const VALUE * @@ -105,15 +99,42 @@ RSTRUCT_CONST_PTR(VALUE st) } static inline void -RSTRUCT_SET(VALUE st, long k, VALUE v) +RSTRUCT_SET_RAW(VALUE st, long k, VALUE v) { RB_OBJ_WRITE(st, &RSTRUCT_CONST_PTR(st)[k], v); } static inline VALUE -RSTRUCT_GET(VALUE st, long k) +RSTRUCT_GET_RAW(VALUE st, long k) { return RSTRUCT_CONST_PTR(st)[k]; } +static inline VALUE +RSTRUCT_FIELDS_OBJ(VALUE st) +{ + const long embed_len = RSTRUCT_EMBED_LEN(st); + VALUE fields_obj; + if (embed_len) { + RUBY_ASSERT(!FL_TEST_RAW(st, RSTRUCT_GEN_FIELDS)); + fields_obj = RSTRUCT_GET_RAW(st, embed_len); + } + else { + fields_obj = RSTRUCT(st)->as.heap.fields_obj; + } + return fields_obj; +} + +static inline void +RSTRUCT_SET_FIELDS_OBJ(VALUE st, VALUE fields_obj) +{ + const long embed_len = RSTRUCT_EMBED_LEN(st); + if (embed_len) { + RUBY_ASSERT(!FL_TEST_RAW(st, RSTRUCT_GEN_FIELDS)); + RSTRUCT_SET_RAW(st, embed_len, fields_obj); + } + else { + RB_OBJ_WRITE(st, &RSTRUCT(st)->as.heap.fields_obj, fields_obj); + } +} #endif /* INTERNAL_STRUCT_H */ diff --git a/internal/symbol.h b/internal/symbol.h index 1a066af0e7..b9109b1347 100644 --- a/internal/symbol.h +++ b/internal/symbol.h @@ -17,8 +17,7 @@ #endif /* symbol.c */ -void rb_sym_global_symbols_mark(void); -void rb_sym_global_symbols_update_references(void); +void rb_sym_global_symbols_mark_and_move(void); VALUE rb_to_symbol_type(VALUE obj); VALUE rb_sym_intern(const char *ptr, long len, rb_encoding *enc); VALUE rb_sym_intern_ascii(const char *ptr, long len); @@ -31,11 +30,11 @@ PUREFUNC(int rb_is_const_sym(VALUE sym)); PUREFUNC(int rb_is_attrset_sym(VALUE sym)); ID rb_make_internal_id(void); ID rb_make_temporary_id(size_t n); +bool rb_obj_is_symbol_table(VALUE obj); +void rb_sym_global_symbol_table_foreach_weak_reference(int (*callback)(VALUE *key, void *data), void *data); void rb_gc_free_dsymbol(VALUE); int rb_static_id_valid_p(ID id); - -/* vm.c */ -void rb_free_static_symid_str(void); +void rb_free_global_symbol_table(void); #if __has_builtin(__builtin_constant_p) #define rb_sym_intern_ascii_cstr(ptr) \ diff --git a/internal/thread.h b/internal/thread.h index 5406a617e4..ea891b4372 100644 --- a/internal/thread.h +++ b/internal/thread.h @@ -56,14 +56,16 @@ VALUE rb_mutex_owned_p(VALUE self); VALUE rb_exec_recursive_outer_mid(VALUE (*f)(VALUE g, VALUE h, int r), VALUE g, VALUE h, ID mid); void ruby_mn_threads_params(void); -int rb_thread_io_wait(struct rb_io *io, int events, struct timeval * timeout); -int rb_thread_wait_for_single_fd(int fd, int events, struct timeval * timeout); +int rb_thread_io_wait(struct rb_thread_struct *th, struct rb_io *io, int events, struct timeval * timeout); +int rb_thread_wait_for_single_fd(struct rb_thread_struct *th, int fd, int events, struct timeval * timeout); size_t rb_thread_io_close_interrupt(struct rb_io *); void rb_thread_io_close_wait(struct rb_io *); void rb_ec_check_ints(struct rb_execution_context_struct *ec); +void rb_thread_free_native_thread(void *th_ptr); + RUBY_SYMBOL_EXPORT_BEGIN void *rb_thread_prevent_fork(void *(*func)(void *), void *data); /* for ext/socket/raddrinfo.c */ @@ -72,6 +74,9 @@ void *rb_thread_prevent_fork(void *(*func)(void *), void *data); /* for ext/sock VALUE rb_thread_io_blocking_region(struct rb_io *io, rb_blocking_function_t *func, void *data1); VALUE rb_thread_io_blocking_call(struct rb_io *io, rb_blocking_function_t *func, void *data1, int events); +// Invoke the given function, with the specified argument, in a way that `IO#close` from another execution context can interrupt it. +VALUE rb_thread_io_blocking_operation(VALUE self, VALUE(*function)(VALUE), VALUE argument); + /* thread.c (export) */ int ruby_thread_has_gvl_p(void); /* for ext/fiddle/closure.c */ @@ -80,6 +85,8 @@ RUBY_SYMBOL_EXPORT_END int rb_threadptr_execute_interrupts(struct rb_thread_struct *th, int blocking_timing); bool rb_thread_mn_schedulable(VALUE thread); +bool rb_thread_resolve_unblock_function(rb_unblock_function_t **unblock_function, void **data2, struct rb_thread_struct *thread); + // interrupt exec typedef VALUE (rb_interrupt_exec_func_t)(void *data); @@ -87,6 +94,7 @@ typedef VALUE (rb_interrupt_exec_func_t)(void *data); enum rb_interrupt_exec_flag { rb_interrupt_exec_flag_none = 0x00, rb_interrupt_exec_flag_value_data = 0x01, + rb_interrupt_exec_flag_new_thread = 0x02, }; // interrupt the target_th and run func. diff --git a/internal/time.h b/internal/time.h index e21b3574f6..1f3505f5bc 100644 --- a/internal/time.h +++ b/internal/time.h @@ -27,11 +27,8 @@ struct timeval rb_time_timeval(VALUE); RUBY_SYMBOL_EXPORT_BEGIN /* time.c (export) */ -void ruby_reset_leap_second_info(void); -#ifdef RBIMPL_ATTR_DEPRECATED_INTERNAL_ONLY -RBIMPL_ATTR_DEPRECATED_INTERNAL_ONLY() -#endif -void ruby_reset_timezone(const char *); RUBY_SYMBOL_EXPORT_END +void ruby_reset_timezone(const char *); + #endif /* INTERNAL_TIME_H */ diff --git a/internal/variable.h b/internal/variable.h index d2432fe22e..ca5e189c90 100644 --- a/internal/variable.h +++ b/internal/variable.h @@ -13,23 +13,22 @@ #include "constant.h" /* for rb_const_entry_t */ #include "ruby/internal/stdbool.h" /* for bool */ #include "ruby/ruby.h" /* for VALUE */ -#include "shape.h" /* for rb_shape_t */ +#include "shape.h" /* for shape_id_t */ /* variable.c */ void rb_gc_mark_global_tbl(void); void rb_gc_update_global_tbl(void); -size_t rb_generic_ivar_memsize(VALUE); VALUE rb_search_class_path(VALUE); VALUE rb_attr_delete(VALUE, ID); void rb_autoload_str(VALUE mod, ID id, VALUE file); VALUE rb_autoload_at_p(VALUE, ID, int); -void rb_autoload_copy_table_for_namespace(st_table *, const rb_namespace_t *); +void rb_autoload_copy_table_for_box(st_table *, const rb_box_t *); NORETURN(VALUE rb_mod_const_missing(VALUE,VALUE)); rb_gvar_getter_t *rb_gvar_getter_function_of(ID); rb_gvar_setter_t *rb_gvar_setter_function_of(ID); void rb_gvar_readonly_setter(VALUE v, ID id, VALUE *_); void rb_gvar_ractor_local(const char *name); -void rb_gvar_namespace_ready(const char *name); +void rb_gvar_box_ready(const char *name); /** * Sets the name of a module. @@ -47,21 +46,21 @@ void rb_gvar_namespace_ready(const char *name); */ VALUE rb_mod_set_temporary_name(VALUE, VALUE); -struct gen_fields_tbl; -int rb_gen_fields_tbl_get(VALUE obj, ID id, struct gen_fields_tbl **fields_tbl); void rb_obj_copy_ivs_to_hash_table(VALUE obj, st_table *table); void rb_obj_init_too_complex(VALUE obj, st_table *table); void rb_evict_ivars_to_hash(VALUE obj); -void rb_evict_fields_to_hash(VALUE obj); VALUE rb_obj_field_get(VALUE obj, shape_id_t target_shape_id); void rb_ivar_set_internal(VALUE obj, ID id, VALUE val); -void rb_obj_field_set(VALUE obj, shape_id_t target_shape_id, VALUE val); +attr_index_t rb_ivar_set_index(VALUE obj, ID id, VALUE val); +attr_index_t rb_obj_field_set(VALUE obj, shape_id_t target_shape_id, ID field_name, VALUE val); +VALUE rb_ivar_get_at(VALUE obj, attr_index_t index, ID id); +VALUE rb_ivar_get_at_no_ractor_check(VALUE obj, attr_index_t index); RUBY_SYMBOL_EXPORT_BEGIN /* variable.c (export) */ void rb_mark_generic_ivar(VALUE obj); VALUE rb_const_missing(VALUE klass, VALUE name); -int rb_class_ivar_set(VALUE klass, ID vid, VALUE value); +bool rb_class_ivar_set(VALUE klass, ID vid, VALUE value); void rb_fields_tbl_copy(VALUE dst, VALUE src); RUBY_SYMBOL_EXPORT_END @@ -70,7 +69,6 @@ VALUE rb_gvar_get(ID); VALUE rb_gvar_set(ID, VALUE); VALUE rb_gvar_defined(ID); void rb_const_warn_if_deprecated(const rb_const_entry_t *, VALUE, ID); -void rb_ensure_iv_list_size(VALUE obj, uint32_t len, uint32_t newsize); -attr_index_t rb_obj_ivar_set(VALUE obj, ID id, VALUE val); +void rb_ensure_iv_list_size(VALUE obj, uint32_t current_len, uint32_t newsize); #endif /* INTERNAL_VARIABLE_H */ diff --git a/internal/vm.h b/internal/vm.h index d1ee437cdc..029b19d555 100644 --- a/internal/vm.h +++ b/internal/vm.h @@ -69,6 +69,7 @@ const char *rb_type_str(enum ruby_value_type type); VALUE rb_check_funcall_default(VALUE, ID, int, const VALUE *, VALUE); VALUE rb_check_funcall_basic_kw(VALUE, ID, VALUE, int, const VALUE*, int); VALUE rb_yield_1(VALUE val); +VALUE rb_ec_yield(struct rb_execution_context_struct *ec, VALUE val); VALUE rb_yield_force_blockarg(VALUE values); VALUE rb_lambda_call(VALUE obj, ID mid, int argc, const VALUE *argv, rb_block_call_func_t bl_proc, int min_argc, int max_argc, @@ -78,10 +79,11 @@ void rb_check_stack_overflow(void); VALUE rb_block_call2(VALUE obj, ID mid, int argc, const VALUE *argv, rb_block_call_func_t bl_proc, VALUE data2, long flags); struct vm_ifunc *rb_current_ifunc(void); VALUE rb_gccct_clear_table(VALUE); +VALUE rb_eval_cmd_call_kw(VALUE cmd, int argc, const VALUE *argv, int kw_splat); -#if USE_YJIT +#if USE_YJIT || USE_ZJIT /* vm_exec.c */ -extern uint64_t rb_vm_insns_count; +extern uint64_t rb_vm_insn_count; #endif extern bool rb_free_at_exit; @@ -99,8 +101,6 @@ const struct rb_callcache *rb_vm_search_method_slowpath(const struct rb_callinfo /* vm_method.c */ int rb_ec_obj_respond_to(struct rb_execution_context_struct *ec, VALUE obj, ID id, int priv); -void rb_clear_constant_cache(void); - /* vm_dump.c */ void rb_print_backtrace(FILE *); @@ -121,7 +121,6 @@ int rb_get_node_id_from_frame_info(VALUE obj); const struct rb_iseq_struct *rb_get_iseq_from_frame_info(VALUE obj); VALUE rb_ec_backtrace_object(const struct rb_execution_context_struct *ec); -void rb_backtrace_use_iseq_first_lineno_for_last_location(VALUE self); #define RUBY_DTRACE_CREATE_HOOK(name, arg) \ RUBY_DTRACE_HOOK(name##_CREATE, arg) |