diff options
Diffstat (limited to 'internal')
44 files changed, 2408 insertions, 580 deletions
diff --git a/internal/array.h b/internal/array.h index a0d16dec3f..3a689646fb 100644 --- a/internal/array.h +++ b/internal/array.h @@ -23,6 +23,7 @@ #define RARRAY_PTR_IN_USE_FLAG FL_USER14 /* array.c */ +VALUE rb_ary_hash_values(long len, const VALUE *elements); VALUE rb_ary_last(int, const VALUE *, VALUE); void rb_ary_set_len(VALUE, long); void rb_ary_delete_same(VALUE, VALUE); @@ -36,13 +37,11 @@ size_t rb_ary_size_as_embedded(VALUE ary); void rb_ary_make_embedded(VALUE ary); bool rb_ary_embeddable_p(VALUE ary); VALUE rb_ary_diff(VALUE ary1, VALUE ary2); +RUBY_EXTERN VALUE rb_cArray_empty_frozen; static inline VALUE rb_ary_entry_internal(VALUE ary, long offset); static inline bool ARY_PTR_USING_P(VALUE ary); -static inline void RARY_TRANSIENT_SET(VALUE ary); -static inline void RARY_TRANSIENT_UNSET(VALUE ary); -MJIT_SYMBOL_EXPORT_BEGIN VALUE rb_ary_tmp_new_from_values(VALUE, long, const VALUE *); VALUE rb_check_to_array(VALUE ary); VALUE rb_ary_behead(VALUE, long); @@ -50,14 +49,13 @@ VALUE rb_ary_aref1(VALUE ary, VALUE i); struct rb_execution_context_struct; VALUE rb_ec_ary_new_from_values(struct rb_execution_context_struct *ec, long n, const VALUE *elts); -MJIT_SYMBOL_EXPORT_END // YJIT needs this function to never allocate and never raise static inline VALUE rb_ary_entry_internal(VALUE ary, long offset) { long len = RARRAY_LEN(ary); - const VALUE *ptr = RARRAY_CONST_PTR_TRANSIENT(ary); + const VALUE *ptr = RARRAY_CONST_PTR(ary); if (len == 0) return Qnil; if (offset < 0) { offset += len; @@ -119,22 +117,6 @@ ARY_SHARED_ROOT_REFCNT(VALUE ary) return RARRAY(ary)->as.heap.aux.capa; } -static inline void -RARY_TRANSIENT_SET(VALUE ary) -{ -#if USE_TRANSIENT_HEAP - FL_SET_RAW(ary, RARRAY_TRANSIENT_FLAG); -#endif -} - -static inline void -RARY_TRANSIENT_UNSET(VALUE ary) -{ -#if USE_TRANSIENT_HEAP - FL_UNSET_RAW(ary, RARRAY_TRANSIENT_FLAG); -#endif -} - #undef rb_ary_new_from_args #if RBIMPL_HAS_WARNING("-Wgnu-zero-variadic-macro-arguments") # /* Skip it; clang -pedantic doesn't like the following */ @@ -155,9 +137,18 @@ RBIMPL_ATTR_ARTIFICIAL() static inline VALUE RARRAY_AREF(VALUE ary, long i) { + VALUE val; RBIMPL_ASSERT_TYPE(ary, RUBY_T_ARRAY); - return RARRAY_CONST_PTR_TRANSIENT(ary)[i]; + RUBY_ASSERT(i < RARRAY_LEN(ary)); + + RBIMPL_WARNING_PUSH(); +#if defined(__GNUC__) && !defined(__clang__) && __GNUC__ == 13 + RBIMPL_WARNING_IGNORED(-Warray-bounds); +#endif + val = RARRAY_CONST_PTR(ary)[i]; + RBIMPL_WARNING_POP(); + return val; } #endif /* INTERNAL_ARRAY_H */ diff --git a/internal/basic_operators.h b/internal/basic_operators.h index 2cd9f50073..493d2fa7f7 100644 --- a/internal/basic_operators.h +++ b/internal/basic_operators.h @@ -24,6 +24,7 @@ enum ruby_basic_operators { BOP_SUCC, BOP_GT, BOP_GE, + BOP_GTGT, BOP_NOT, BOP_NEQ, BOP_MATCH, @@ -31,18 +32,19 @@ enum ruby_basic_operators { BOP_UMINUS, BOP_MAX, BOP_MIN, + BOP_HASH, BOP_CALL, BOP_AND, BOP_OR, BOP_CMP, BOP_DEFAULT, + BOP_PACK, + BOP_INCLUDE_P, BOP_LAST_ }; -MJIT_SYMBOL_EXPORT_BEGIN RUBY_EXTERN short ruby_vm_redefined_flag[BOP_LAST_]; -MJIT_SYMBOL_EXPORT_END /* optimize insn */ #define INTEGER_REDEFINED_OP_FLAG (1 << 0) diff --git a/internal/bignum.h b/internal/bignum.h index 5cd35ede8a..0692bafed3 100644 --- a/internal/bignum.h +++ b/internal/bignum.h @@ -9,6 +9,7 @@ * @brief Internal header for Bignums. */ #include "ruby/internal/config.h" /* for HAVE_LIBGMP */ +#include "internal/compilers.h" /* for FLEX_ARY_LEN */ #include <stddef.h> /* for size_t */ #ifdef HAVE_SYS_TYPES_H @@ -76,18 +77,17 @@ #define RBIGNUM(obj) ((struct RBignum *)(obj)) #define BIGNUM_SIGN_BIT FL_USER1 #define BIGNUM_EMBED_FLAG ((VALUE)FL_USER2) -#define BIGNUM_EMBED_LEN_NUMBITS 3 + +/* This is likely more bits than we need today and will also need adjustment if + * we change GC slot sizes. + */ +#define BIGNUM_EMBED_LEN_NUMBITS 9 #define BIGNUM_EMBED_LEN_MASK \ - (~(~(VALUE)0U << BIGNUM_EMBED_LEN_NUMBITS) << BIGNUM_EMBED_LEN_SHIFT) + (RUBY_FL_USER11 | RUBY_FL_USER10 | RUBY_FL_USER9 | RUBY_FL_USER8 | RUBY_FL_USER7 | \ + RUBY_FL_USER6 | RUBY_FL_USER5 | RUBY_FL_USER4 | RUBY_FL_USER3) #define BIGNUM_EMBED_LEN_SHIFT \ (FL_USHIFT+3) /* bit offset of BIGNUM_EMBED_LEN_MASK */ -#ifndef BIGNUM_EMBED_LEN_MAX -# if (SIZEOF_VALUE*RBIMPL_RVALUE_EMBED_LEN_MAX/SIZEOF_ACTUAL_BDIGIT) < (1 << BIGNUM_EMBED_LEN_NUMBITS)-1 -# define BIGNUM_EMBED_LEN_MAX (SIZEOF_VALUE*RBIMPL_RVALUE_EMBED_LEN_MAX/SIZEOF_ACTUAL_BDIGIT) -# else -# define BIGNUM_EMBED_LEN_MAX ((1 << BIGNUM_EMBED_LEN_NUMBITS)-1) -# endif -#endif +#define BIGNUM_EMBED_LEN_MAX (BIGNUM_EMBED_LEN_MASK >> BIGNUM_EMBED_LEN_SHIFT) enum rb_int_parse_flags { RB_INT_PARSE_SIGN = 0x01, @@ -104,7 +104,12 @@ struct RBignum { size_t len; BDIGIT *digits; } heap; - BDIGIT ary[BIGNUM_EMBED_LEN_MAX]; + /* This is a length 1 array because: + * 1. GCC has a bug that does not optimize C flexible array members + * (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=102452) + * 2. Zero length arrays are not supported by all compilers + */ + BDIGIT ary[1]; } as; }; @@ -121,6 +126,7 @@ VALUE rb_integer_float_eq(VALUE x, VALUE y); VALUE rb_str_convert_to_inum(VALUE str, int base, int badcheck, int raise_exception); VALUE rb_big_comp(VALUE x); VALUE rb_big_aref(VALUE x, VALUE y); +VALUE rb_big_aref2(VALUE num, VALUE beg, VALUE len); VALUE rb_big_abs(VALUE x); VALUE rb_big_size_m(VALUE big); VALUE rb_big_bit_length(VALUE big); @@ -163,11 +169,15 @@ VALUE rb_str2big_gmp(VALUE arg, int base, int badcheck); VALUE rb_int_parse_cstr(const char *str, ssize_t len, char **endp, size_t *ndigits, int base, int flags); RUBY_SYMBOL_EXPORT_END -MJIT_SYMBOL_EXPORT_BEGIN +#if HAVE_LONG_LONG +VALUE rb_ull2big(unsigned LONG_LONG n); +VALUE rb_ll2big(LONG_LONG n); +#endif + #if defined(HAVE_INT128_T) +VALUE rb_uint128t2big(uint128_t n); VALUE rb_int128t2big(int128_t n); #endif -MJIT_SYMBOL_EXPORT_END /* sign: positive:1, negative:0 */ static inline bool diff --git a/internal/bits.h b/internal/bits.h index 2602ff7a31..698ab3e219 100644 --- a/internal/bits.h +++ b/internal/bits.h @@ -30,14 +30,13 @@ #include <stdint.h> /* for uintptr_t */ #include "internal/compilers.h" /* for MSC_VERSION_SINCE */ -#if MSC_VERSION_SINCE(1310) +#ifdef _MSC_VER # include <stdlib.h> /* for _byteswap_uint64 */ #endif -#if defined(HAVE_X86INTRIN_H) && ! defined(MJIT_HEADER) -# /* Rule out MJIT_HEADER, which does not interface well with <immintrin.h> */ +#if defined(HAVE_X86INTRIN_H) # include <x86intrin.h> /* for _lzcnt_u64 */ -#elif MSC_VERSION_SINCE(1310) +#elif defined(_MSC_VER) # include <intrin.h> /* for the following intrinsics */ #endif @@ -51,16 +50,13 @@ # pragma intrinsic(__lzcnt64) #endif -#if MSC_VERSION_SINCE(1310) +#if defined(_MSC_VER) # pragma intrinsic(_rotl) # pragma intrinsic(_rotr) # ifdef _WIN64 # pragma intrinsic(_rotl64) # pragma intrinsic(_rotr64) # endif -#endif - -#if MSC_VERSION_SINCE(1400) # pragma intrinsic(_BitScanForward) # pragma intrinsic(_BitScanReverse) # ifdef _WIN64 @@ -91,6 +87,7 @@ #define UNSIGNED_INTEGER_MAX(T) ((T)~(T)0) +#ifndef MUL_OVERFLOW_SIGNED_INTEGER_P #if __has_builtin(__builtin_mul_overflow_p) # define MUL_OVERFLOW_P(a, b) \ __builtin_mul_overflow_p((a), (b), (__typeof__(a * b))0) @@ -119,15 +116,100 @@ MUL_OVERFLOW_SIGNED_INTEGER_P(a, b, FIXNUM_MIN, FIXNUM_MAX) #endif -#ifdef MUL_OVERFLOW_P +#if defined(MUL_OVERFLOW_P) && defined(USE___BUILTIN_MUL_OVERFLOW_LONG_LONG) # define MUL_OVERFLOW_LONG_LONG_P(a, b) MUL_OVERFLOW_P(a, b) +#else +# define MUL_OVERFLOW_LONG_LONG_P(a, b) MUL_OVERFLOW_SIGNED_INTEGER_P(a, b, LLONG_MIN, LLONG_MAX) +#endif + +#ifdef MUL_OVERFLOW_P # define MUL_OVERFLOW_LONG_P(a, b) MUL_OVERFLOW_P(a, b) # define MUL_OVERFLOW_INT_P(a, b) MUL_OVERFLOW_P(a, b) #else -# define MUL_OVERFLOW_LONG_LONG_P(a, b) MUL_OVERFLOW_SIGNED_INTEGER_P(a, b, LLONG_MIN, LLONG_MAX) # define MUL_OVERFLOW_LONG_P(a, b) MUL_OVERFLOW_SIGNED_INTEGER_P(a, b, LONG_MIN, LONG_MAX) # define MUL_OVERFLOW_INT_P(a, b) MUL_OVERFLOW_SIGNED_INTEGER_P(a, b, INT_MIN, INT_MAX) #endif +#endif + +#ifndef ADD_OVERFLOW_SIGNED_INTEGER_P +#if __has_builtin(__builtin_add_overflow_p) +# define ADD_OVERFLOW_P(a, b) \ + __builtin_add_overflow_p((a), (b), (__typeof__(a * b))0) +#elif __has_builtin(__builtin_add_overflow) +# define ADD_OVERFLOW_P(a, b) \ + __extension__ ({ __typeof__(a) c; __builtin_add_overflow((a), (b), &c); }) +#endif + +#define ADD_OVERFLOW_SIGNED_INTEGER_P(a, b, min, max) ( \ + (a) > 0 ? (b) > (max) - (a) : (b) < (min) - (a)) + +#if __has_builtin(__builtin_add_overflow_p) +/* __builtin_add_overflow_p can take bitfield */ +/* and GCC permits bitfields for integers other than int */ +# define ADD_OVERFLOW_FIXNUM_P(a, b) \ + __extension__ ({ \ + struct { long fixnum : sizeof(long) * CHAR_BIT - 1; } c = { 0 }; \ + __builtin_add_overflow_p((a), (b), c.fixnum); \ + }) +#else +# define ADD_OVERFLOW_FIXNUM_P(a, b) \ + ADD_OVERFLOW_SIGNED_INTEGER_P(a, b, FIXNUM_MIN, FIXNUM_MAX) +#endif + +#if defined(ADD_OVERFLOW_P) && defined(USE___BUILTIN_ADD_OVERFLOW_LONG_LONG) +# define ADD_OVERFLOW_LONG_LONG_P(a, b) ADD_OVERFLOW_P(a, b) +#else +# define ADD_OVERFLOW_LONG_LONG_P(a, b) ADD_OVERFLOW_SIGNED_INTEGER_P(a, b, LLONG_MIN, LLONG_MAX) +#endif + +#ifdef ADD_OVERFLOW_P +# define ADD_OVERFLOW_LONG_P(a, b) ADD_OVERFLOW_P(a, b) +# define ADD_OVERFLOW_INT_P(a, b) ADD_OVERFLOW_P(a, b) +#else +# define ADD_OVERFLOW_LONG_P(a, b) ADD_OVERFLOW_SIGNED_INTEGER_P(a, b, LONG_MIN, LONG_MAX) +# define ADD_OVERFLOW_INT_P(a, b) ADD_OVERFLOW_SIGNED_INTEGER_P(a, b, INT_MIN, INT_MAX) +#endif +#endif + +#ifndef SUB_OVERFLOW_SIGNED_INTEGER_P +#if __has_builtin(__builtin_sub_overflow_p) +# define SUB_OVERFLOW_P(a, b) \ + __builtin_sub_overflow_p((a), (b), (__typeof__(a * b))0) +#elif __has_builtin(__builtin_sub_overflow) +# define SUB_OVERFLOW_P(a, b) \ + __extension__ ({ __typeof__(a) c; __builtin_sub_overflow((a), (b), &c); }) +#endif + +#define SUB_OVERFLOW_SIGNED_INTEGER_P(a, b, min, max) ( \ + (b) > 0 ? (a) < (min) + (b) : (a) > (max) + (b)) + +#if __has_builtin(__builtin_sub_overflow_p) +/* __builtin_sub_overflow_p can take bitfield */ +/* and GCC permits bitfields for integers other than int */ +# define SUB_OVERFLOW_FIXNUM_P(a, b) \ + __extension__ ({ \ + struct { long fixnum : sizeof(long) * CHAR_BIT - 1; } c = { 0 }; \ + __builtin_sub_overflow_p((a), (b), c.fixnum); \ + }) +#else +# define SUB_OVERFLOW_FIXNUM_P(a, b) \ + SUB_OVERFLOW_SIGNED_INTEGER_P(a, b, FIXNUM_MIN, FIXNUM_MAX) +#endif + +#if defined(SUB_OVERFLOW_P) && defined(USE___BUILTIN_SUB_OVERFLOW_LONG_LONG) +# define SUB_OVERFLOW_LONG_LONG_P(a, b) SUB_OVERFLOW_P(a, b) +#else +# define SUB_OVERFLOW_LONG_LONG_P(a, b) SUB_OVERFLOW_SIGNED_INTEGER_P(a, b, LLONG_MIN, LLONG_MAX) +#endif + +#ifdef SUB_OVERFLOW_P +# define SUB_OVERFLOW_LONG_P(a, b) SUB_OVERFLOW_P(a, b) +# define SUB_OVERFLOW_INT_P(a, b) SUB_OVERFLOW_P(a, b) +#else +# define SUB_OVERFLOW_LONG_P(a, b) SUB_OVERFLOW_SIGNED_INTEGER_P(a, b, LONG_MIN, LONG_MAX) +# define SUB_OVERFLOW_INT_P(a, b) SUB_OVERFLOW_SIGNED_INTEGER_P(a, b, INT_MIN, INT_MAX) +#endif +#endif #ifdef HAVE_UINT128_T # define bit_length(x) \ @@ -181,7 +263,7 @@ ruby_swap16(uint16_t x) #if __has_builtin(__builtin_bswap16) return __builtin_bswap16(x); -#elif MSC_VERSION_SINCE(1310) +#elif defined(_MSC_VER) return _byteswap_ushort(x); #else @@ -196,7 +278,7 @@ ruby_swap32(uint32_t x) #if __has_builtin(__builtin_bswap32) return __builtin_bswap32(x); -#elif MSC_VERSION_SINCE(1310) +#elif defined(_MSC_VER) return _byteswap_ulong(x); #else @@ -213,7 +295,7 @@ ruby_swap64(uint64_t x) #if __has_builtin(__builtin_bswap64) return __builtin_bswap64(x); -#elif MSC_VERSION_SINCE(1310) +#elif defined(_MSC_VER) return _byteswap_uint64(x); #else @@ -235,10 +317,10 @@ nlz_int32(uint32_t x) * safety. */ return (unsigned int)__lzcnt(x); -#elif defined(__x86_64__) && defined(__LZCNT__) && ! defined(MJIT_HEADER) +#elif defined(__x86_64__) && defined(__LZCNT__) return (unsigned int)_lzcnt_u32(x); -#elif MSC_VERSION_SINCE(1400) /* &&! defined(__AVX2__) */ +#elif defined(_MSC_VER) /* &&! defined(__AVX2__) */ unsigned long r; return _BitScanReverse(&r, x) ? (31 - (int)r) : 32; @@ -264,10 +346,10 @@ nlz_int64(uint64_t x) #if defined(_MSC_VER) && defined(__AVX2__) return (unsigned int)__lzcnt64(x); -#elif defined(__x86_64__) && defined(__LZCNT__) && ! defined(MJIT_HEADER) +#elif defined(__x86_64__) && defined(__LZCNT__) return (unsigned int)_lzcnt_u64(x); -#elif defined(_WIN64) && MSC_VERSION_SINCE(1400) /* &&! defined(__AVX2__) */ +#elif defined(_WIN64) && defined(_MSC_VER) /* &&! defined(__AVX2__) */ unsigned long r; return _BitScanReverse64(&r, x) ? (63u - (unsigned int)r) : 64; @@ -395,9 +477,9 @@ rb_popcount32(uint32_t x) #else x = (x & 0x55555555) + (x >> 1 & 0x55555555); x = (x & 0x33333333) + (x >> 2 & 0x33333333); - x = (x & 0x0f0f0f0f) + (x >> 4 & 0x0f0f0f0f); - x = (x & 0x001f001f) + (x >> 8 & 0x001f001f); - x = (x & 0x0000003f) + (x >>16 & 0x0000003f); + x = (x & 0x07070707) + (x >> 4 & 0x07070707); + x = (x & 0x000f000f) + (x >> 8 & 0x000f000f); + x = (x & 0x0000001f) + (x >>16 & 0x0000001f); return (unsigned int)x; #endif @@ -425,9 +507,9 @@ rb_popcount64(uint64_t x) x = (x & 0x5555555555555555) + (x >> 1 & 0x5555555555555555); x = (x & 0x3333333333333333) + (x >> 2 & 0x3333333333333333); x = (x & 0x0707070707070707) + (x >> 4 & 0x0707070707070707); - x = (x & 0x001f001f001f001f) + (x >> 8 & 0x001f001f001f001f); - x = (x & 0x0000003f0000003f) + (x >>16 & 0x0000003f0000003f); - x = (x & 0x000000000000007f) + (x >>32 & 0x000000000000007f); + x = (x & 0x000f000f000f000f) + (x >> 8 & 0x000f000f000f000f); + x = (x & 0x0000001f0000001f) + (x >>16 & 0x0000001f0000001f); + x = (x & 0x000000000000003f) + (x >>32 & 0x000000000000003f); return (unsigned int)x; #endif @@ -450,10 +532,10 @@ rb_popcount_intptr(uintptr_t x) static inline int ntz_int32(uint32_t x) { -#if defined(__x86_64__) && defined(__BMI__) && ! defined(MJIT_HEADER) +#if defined(__x86_64__) && defined(__BMI__) return (unsigned)_tzcnt_u32(x); -#elif MSC_VERSION_SINCE(1400) +#elif defined(_MSC_VER) /* :FIXME: Is there any way to issue TZCNT instead of BSF, apart from using * assembly? Because issuing LZCNT seems possible (see nlz.h). */ unsigned long r; @@ -472,10 +554,10 @@ ntz_int32(uint32_t x) static inline int ntz_int64(uint64_t x) { -#if defined(__x86_64__) && defined(__BMI__) && ! defined(MJIT_HEADER) +#if defined(__x86_64__) && defined(__BMI__) return (unsigned)_tzcnt_u64(x); -#elif defined(_WIN64) && MSC_VERSION_SINCE(1400) +#elif defined(_WIN64) && defined(_MSC_VER) unsigned long r; return _BitScanForward64(&r, x) ? (int)r : 64; @@ -523,10 +605,10 @@ RUBY_BIT_ROTL(VALUE v, int n) #elif __has_builtin(__builtin_rotateleft64) && (SIZEOF_VALUE * CHAR_BIT == 64) return __builtin_rotateleft64(v, n); -#elif MSC_VERSION_SINCE(1310) && (SIZEOF_VALUE * CHAR_BIT == 32) +#elif defined(_MSC_VER) && (SIZEOF_VALUE * CHAR_BIT == 32) return _rotl(v, n); -#elif MSC_VERSION_SINCE(1310) && (SIZEOF_VALUE * CHAR_BIT == 64) +#elif defined(_MSC_VER) && (SIZEOF_VALUE * CHAR_BIT == 64) return _rotl64(v, n); #elif defined(_lrotl) && (SIZEOF_VALUE == SIZEOF_LONG) @@ -547,10 +629,10 @@ RUBY_BIT_ROTR(VALUE v, int n) #elif __has_builtin(__builtin_rotateright64) && (SIZEOF_VALUE * CHAR_BIT == 64) return __builtin_rotateright64(v, n); -#elif MSC_VERSION_SINCE(1310) && (SIZEOF_VALUE * CHAR_BIT == 32) +#elif defined(_MSC_VER) && (SIZEOF_VALUE * CHAR_BIT == 32) return _rotr(v, n); -#elif MSC_VERSION_SINCE(1310) && (SIZEOF_VALUE * CHAR_BIT == 64) +#elif defined(_MSC_VER) && (SIZEOF_VALUE * CHAR_BIT == 64) return _rotr64(v, n); #elif defined(_lrotr) && (SIZEOF_VALUE == SIZEOF_LONG) diff --git a/internal/box.h b/internal/box.h new file mode 100644 index 0000000000..b62b6a9bc9 --- /dev/null +++ b/internal/box.h @@ -0,0 +1,83 @@ +#ifndef INTERNAL_BOX_H /*-*-C-*-vi:se ft=c:*/ +#define INTERNAL_BOX_H + +#include "ruby/ruby.h" /* for VALUE */ + +/** + * @author Ruby developers <ruby-core@ruby-lang.org> + * @copyright This file is a part of the programming language Ruby. + * Permission is hereby granted, to either redistribute and/or + * modify this file, provided that the conditions mentioned in the + * file COPYING are met. Consult the file for details. + * @brief Internal header for Ruby Box. + */ +struct rb_box_struct { + /* + * To retrieve Ruby::Box object that provides #require and so on. + * That is used from load.c, etc., that uses rb_box_t internally. + */ + VALUE box_object; + long box_id; // box_id to generate ext filenames + + VALUE top_self; + + VALUE load_path; + VALUE load_path_snapshot; + VALUE load_path_check_cache; + VALUE expanded_load_path; + VALUE loaded_features; + VALUE loaded_features_snapshot; + VALUE loaded_features_realpaths; + VALUE loaded_features_realpath_map; + struct st_table *loaded_features_index; + struct st_table *loading_table; + VALUE ruby_dln_libmap; + + VALUE gvar_tbl; + struct st_table *classext_cow_classes; + + bool is_user; + bool is_optional; +}; +typedef struct rb_box_struct rb_box_t; + +#define BOX_OBJ_P(obj) (rb_obj_class(obj) == rb_cBox) + +#define BOX_ROOT_P(box) (box && !box->is_user) +#define BOX_USER_P(box) (box && box->is_user) +#define BOX_OPTIONAL_P(box) (box && box->is_optional) +#define BOX_MAIN_P(box) (box && box->is_user && !box->is_optional) + +#define BOX_METHOD_DEFINITION(mdef) (mdef ? mdef->ns : NULL) +#define BOX_METHOD_ENTRY(me) (me ? BOX_METHOD_DEFINITION(me->def) : NULL) +#define BOX_CC(cc) (cc ? BOX_METHOD_ENTRY(cc->cme_) : NULL) +#define BOX_CC_ENTRIES(ccs) (ccs ? BOX_METHOD_ENTRY(ccs->cme) : NULL) + +RUBY_EXTERN bool ruby_box_enabled; +RUBY_EXTERN bool ruby_box_init_done; +RUBY_EXTERN bool ruby_box_crashed; + +static inline bool +rb_box_available(void) +{ + return ruby_box_enabled; +} + +const rb_box_t * rb_root_box(void); +const rb_box_t * rb_main_box(void); +const rb_box_t * rb_current_box(void); +const rb_box_t * rb_loading_box(void); +const rb_box_t * rb_current_box_in_crash_report(void); + +void rb_box_entry_mark(void *); +void rb_box_gc_update_references(void *ptr); + +rb_box_t * rb_get_box_t(VALUE ns); +VALUE rb_get_box_object(rb_box_t *ns); + +VALUE rb_box_local_extension(VALUE box, VALUE fname, VALUE path, VALUE *cleanup); +void rb_box_cleanup_local_extension(VALUE cleanup); + +void rb_initialize_main_box(void); +void rb_box_init_done(void); +#endif /* INTERNAL_BOX_H */ diff --git a/internal/class.h b/internal/class.h index 63917e867f..ea68b07fc2 100644 --- a/internal/class.h +++ b/internal/class.h @@ -8,26 +8,66 @@ * file COPYING are met. Consult the file for details. * @brief Internal header for Class. */ +#include "id.h" #include "id_table.h" /* for struct rb_id_table */ -#include "internal/gc.h" /* for RB_OBJ_WRITE */ +#include "internal/box.h" #include "internal/serial.h" /* for rb_serial_t */ +#include "internal/static_assert.h" +#include "internal/variable.h" /* for rb_class_ivar_set */ #include "ruby/internal/stdbool.h" /* for bool */ #include "ruby/intern.h" /* for rb_alloc_func_t */ #include "ruby/ruby.h" /* for struct RBasic */ #include "shape.h" #include "ruby_assert.h" #include "vm_core.h" +#include "vm_sync.h" #include "method.h" /* for rb_cref_t */ #ifdef RCLASS_SUPER # undef RCLASS_SUPER #endif +struct rb_box_subclasses { + long refcount; + struct st_table *tbl; +}; +typedef struct rb_box_subclasses rb_box_subclasses_t; + +static inline long +rb_box_subclasses_ref_count(rb_box_subclasses_t *box_sub) +{ + return box_sub->refcount; +} + +static inline rb_box_subclasses_t * +rb_box_subclasses_ref_inc(rb_box_subclasses_t *box_sub) +{ + box_sub->refcount++; + return box_sub; +} + +static inline void +rb_box_subclasses_ref_dec(rb_box_subclasses_t *box_sub) +{ + box_sub->refcount--; + if (box_sub->refcount == 0) { + st_free_table(box_sub->tbl); + xfree(box_sub); + } +} + +struct rb_subclass_anchor { + rb_box_subclasses_t *box_subclasses; + struct rb_subclass_entry *head; +}; +typedef struct rb_subclass_anchor rb_subclass_anchor_t; + struct rb_subclass_entry { VALUE klass; struct rb_subclass_entry *next; struct rb_subclass_entry *prev; }; +typedef struct rb_subclass_entry rb_subclass_entry_t; struct rb_cvar_class_tbl_entry { uint32_t index; @@ -37,88 +77,429 @@ struct rb_cvar_class_tbl_entry { }; struct rb_classext_struct { - VALUE *iv_ptr; + const rb_box_t *box; + VALUE super; + VALUE fields_obj; // Fields are either ivar or other internal properties stored inline + struct rb_id_table *m_tbl; struct rb_id_table *const_tbl; struct rb_id_table *callable_m_tbl; - struct rb_id_table *cc_tbl; /* ID -> [[ci, cc1], cc2, ...] */ + VALUE cc_tbl; /* { ID => { cme, [cc1, cc2, ...] }, ... } */ struct rb_id_table *cvc_tbl; - size_t superclass_depth; VALUE *superclasses; - struct rb_subclass_entry *subclasses; - struct rb_subclass_entry *subclass_entry; /** - * In the case that this is an `ICLASS`, `module_subclasses` points to the link + * The head of subclasses is a blank (w/o klass) entry to be referred from anchor (and be never deleted). + * (anchor -> head -> 1st-entry) + */ + struct rb_subclass_anchor *subclasses; + /** + * The `box_super_subclasses` points the `box_subclasses` struct to retreive the subclasses + * of the super class in a specific box. + * In compaction GCs, collecting a classext should trigger the deletion of a rb_subclass_entry + * from the super's subclasses. But it may be prevented by the read barrier. + * Fetching the super's subclasses for a ns is to avoid the read barrier in that process. + */ + rb_box_subclasses_t *box_super_subclasses; + /** + * In the case that this is an `ICLASS`, `box_module_subclasses` points to the link * in the module's `subclasses` list that indicates that the klass has been * included. Hopefully that makes sense. */ - struct rb_subclass_entry *module_subclass_entry; + rb_box_subclasses_t *box_module_subclasses; + const VALUE origin_; const VALUE refined_class; - rb_alloc_func_t allocator; - const VALUE includer; - uint32_t max_iv_count; - uint32_t variation_count; -#if !SHAPE_IN_BASIC_FLAGS - shape_id_t shape_id; -#endif + union { + struct { + rb_alloc_func_t allocator; + } class; + struct { + VALUE attached_object; + } singleton_class; + struct { + const VALUE includer; + } iclass; + } as; + attr_index_t max_iv_count; + uint16_t superclass_depth; + unsigned char variation_count; + bool permanent_classpath : 1; + bool cloned : 1; + bool shared_const_tbl : 1; + bool iclass_is_origin : 1; + bool iclass_origin_shared_mtbl : 1; + bool superclasses_with_self : 1; + VALUE classpath; }; +typedef struct rb_classext_struct rb_classext_t; + +STATIC_ASSERT(shape_max_variations, SHAPE_MAX_VARIATIONS < (1 << (sizeof(((rb_classext_t *)0)->variation_count) * CHAR_BIT))); struct RClass { struct RBasic basic; - VALUE super; - struct rb_id_table *m_tbl; -#if SIZE_POOL_COUNT == 1 - struct rb_classext_struct *ptr; -#endif + VALUE object_id; + /* + * If box_classext_tbl is NULL, then the prime classext is readable (because no other classext exists). + * For the check whether writable or not, check flag RCLASS_PRIME_CLASSEXT_WRITABLE + */ }; -typedef struct rb_subclass_entry rb_subclass_entry_t; -typedef struct rb_classext_struct rb_classext_t; +struct RClass_and_rb_classext_t { + struct RClass rclass; + rb_classext_t classext; +}; -#if RCLASS_EXT_EMBEDDED -# define RCLASS_EXT(c) ((rb_classext_t *)((char *)(c) + sizeof(struct RClass))) -#else -# define RCLASS_EXT(c) (RCLASS(c)->ptr) +#if SIZEOF_VALUE >= SIZEOF_LONG_LONG +// Assert that classes can be embedded in heaps[2] (which has 160B slot size) +// On 32bit platforms there is no variable width allocation so it doesn't matter. +STATIC_ASSERT(sizeof_rb_classext_t, sizeof(struct RClass_and_rb_classext_t) <= 4 * RVALUE_SIZE); #endif -#define RCLASS_CONST_TBL(c) (RCLASS_EXT(c)->const_tbl) -#define RCLASS_M_TBL(c) (RCLASS(c)->m_tbl) -#define RCLASS_IVPTR(c) (RCLASS_EXT(c)->iv_ptr) -#define RCLASS_CALLABLE_M_TBL(c) (RCLASS_EXT(c)->callable_m_tbl) -#define RCLASS_CC_TBL(c) (RCLASS_EXT(c)->cc_tbl) -#define RCLASS_CVC_TBL(c) (RCLASS_EXT(c)->cvc_tbl) -#define RCLASS_ORIGIN(c) (RCLASS_EXT(c)->origin_) -#define RCLASS_REFINED_CLASS(c) (RCLASS_EXT(c)->refined_class) -#define RCLASS_INCLUDER(c) (RCLASS_EXT(c)->includer) -#define RCLASS_SUBCLASS_ENTRY(c) (RCLASS_EXT(c)->subclass_entry) -#define RCLASS_MODULE_SUBCLASS_ENTRY(c) (RCLASS_EXT(c)->module_subclass_entry) -#define RCLASS_ALLOCATOR(c) (RCLASS_EXT(c)->allocator) -#define RCLASS_SUBCLASSES(c) (RCLASS_EXT(c)->subclasses) -#define RCLASS_SUPERCLASS_DEPTH(c) (RCLASS_EXT(c)->superclass_depth) -#define RCLASS_SUPERCLASSES(c) (RCLASS_EXT(c)->superclasses) - -#define RICLASS_IS_ORIGIN FL_USER0 -#define RCLASS_CLONED FL_USER1 -#define RCLASS_SUPERCLASSES_INCLUDE_SELF FL_USER2 -#define RICLASS_ORIGIN_SHARED_MTBL FL_USER3 + +struct RClass_boxable { + struct RClass_and_rb_classext_t base; + st_table *box_classext_tbl; // box_object -> (rb_classext_t *) +}; + +static const uint16_t RCLASS_MAX_SUPERCLASS_DEPTH = ((uint16_t)-1); + +static inline bool RCLASS_SINGLETON_P(VALUE klass); + +static inline bool RCLASS_PRIME_CLASSEXT_READABLE_P(VALUE obj); +static inline bool RCLASS_PRIME_CLASSEXT_WRITABLE_P(VALUE obj); +static inline void RCLASS_SET_PRIME_CLASSEXT_WRITABLE(VALUE obj, bool writable); + +#define RCLASS_EXT_PRIME(c) (&((struct RClass_and_rb_classext_t*)(c))->classext) +#define RCLASS_EXT_PRIME_P(ext, c) (&((struct RClass_and_rb_classext_t*)(c))->classext == ext) + +static inline rb_classext_t * RCLASS_EXT_READABLE_IN_BOX(VALUE obj, const rb_box_t *box); +static inline rb_classext_t * RCLASS_EXT_READABLE(VALUE obj); +static inline rb_classext_t * RCLASS_EXT_WRITABLE_IN_BOX(VALUE obj, const rb_box_t *box); +static inline rb_classext_t * RCLASS_EXT_WRITABLE(VALUE obj); + +// Raw accessor +#define RCLASSEXT_BOX(ext) (ext->box) +#define RCLASSEXT_SUPER(ext) (ext->super) +#define RCLASSEXT_FIELDS(ext) (ext->fields_obj ? ROBJECT_FIELDS(ext->fields_obj) : NULL) +#define RCLASSEXT_FIELDS_OBJ(ext) (ext->fields_obj) +#define RCLASSEXT_M_TBL(ext) (ext->m_tbl) +#define RCLASSEXT_CONST_TBL(ext) (ext->const_tbl) +#define RCLASSEXT_CALLABLE_M_TBL(ext) (ext->callable_m_tbl) +#define RCLASSEXT_CC_TBL(ext) (ext->cc_tbl) +#define RCLASSEXT_CVC_TBL(ext) (ext->cvc_tbl) +#define RCLASSEXT_SUPERCLASS_DEPTH(ext) (ext->superclass_depth) +#define RCLASSEXT_SUPERCLASSES(ext) (ext->superclasses) +#define RCLASSEXT_SUBCLASSES(ext) (ext->subclasses) +#define RCLASSEXT_BOX_SUPER_SUBCLASSES(ext) (ext->box_super_subclasses) +#define RCLASSEXT_BOX_MODULE_SUBCLASSES(ext) (ext->box_module_subclasses) +#define RCLASSEXT_ORIGIN(ext) (ext->origin_) +#define RCLASSEXT_REFINED_CLASS(ext) (ext->refined_class) +// class.allocator/singleton_class.attached_object are not accessed directly via RCLASSEXT_* +#define RCLASSEXT_INCLUDER(ext) (ext->as.iclass.includer) +#define RCLASSEXT_PERMANENT_CLASSPATH(ext) (ext->permanent_classpath) +#define RCLASSEXT_CLONED(ext) (ext->cloned) +#define RCLASSEXT_SHARED_CONST_TBL(ext) (ext->shared_const_tbl) +#define RCLASSEXT_ICLASS_IS_ORIGIN(ext) (ext->iclass_is_origin) +#define RCLASSEXT_ICLASS_ORIGIN_SHARED_MTBL(ext) (ext->iclass_origin_shared_mtbl) +#define RCLASSEXT_SUPERCLASSES_WITH_SELF(ext) (ext->superclasses_with_self) +#define RCLASSEXT_CLASSPATH(ext) (ext->classpath) + +static inline void RCLASSEXT_SET_ORIGIN(rb_classext_t *ext, VALUE klass, VALUE origin); +static inline void RCLASSEXT_SET_INCLUDER(rb_classext_t *ext, VALUE klass, VALUE includer); + +/* Prime classext entry accessor for very specific reason */ +#define RCLASS_PRIME_BOX(c) (RCLASS_EXT_PRIME(c)->box) +// To invalidate CC by inserting&invalidating method entry into tables containing the target cme +// See clear_method_cache_by_id_in_class() +#define RCLASS_PRIME_FIELDS_OBJ(c) (RCLASS_EXT_PRIME(c)->fields_obj) +#define RCLASS_PRIME_M_TBL(c) (RCLASS_EXT_PRIME(c)->m_tbl) +#define RCLASS_PRIME_CONST_TBL(c) (RCLASS_EXT_PRIME(c)->const_tbl) +#define RCLASS_PRIME_CALLABLE_M_TBL(c) (RCLASS_EXT_PRIME(c)->callable_m_tbl) +#define RCLASS_PRIME_CC_TBL(c) (RCLASS_EXT_PRIME(c)->cc_tbl) +#define RCLASS_M_TBL_NOT_PRIME_P(c, tbl) (RCLASS_EXT_PRIME(c)->m_tbl != tbl) +#define RCLASS_CALLABLE_M_TBL_NOT_PRIME_P(c, tbl) (RCLASS_EXT_PRIME(c)->callable_m_tbl != tbl) +#define RCLASS_CC_TBL_NOT_PRIME_P(c, tbl) (RCLASS_EXT_PRIME(c)->cc_tbl != tbl) + +// Read accessor, regarding box +#define RCLASS_SUPER(c) (RCLASS_EXT_READABLE(c)->super) +#define RCLASS_M_TBL(c) (RCLASS_EXT_READABLE(c)->m_tbl) +#define RCLASS_CONST_TBL(c) (RCLASS_EXT_READABLE(c)->const_tbl) +/* + * Both cc_tbl/callable_m_tbl are cache-like and always be changed when referreed, + * so always those should be writable. + */ +#define RCLASS_CVC_TBL(c) (RCLASS_EXT_READABLE(c)->cvc_tbl) +#define RCLASS_SUBCLASSES_X(c) (RCLASS_EXT_READABLE(c)->subclasses) +#define RCLASS_SUBCLASSES_FIRST(c) (RCLASS_EXT_READABLE(c)->subclasses->head->next) +#define RCLASS_ORIGIN(c) (RCLASS_EXT_READABLE(c)->origin_) +#define RICLASS_IS_ORIGIN_P(c) (RCLASS_EXT_READABLE(c)->iclass_is_origin) +#define RCLASS_PERMANENT_CLASSPATH_P(c) (RCLASS_EXT_READABLE(c)->permanent_classpath) +#define RCLASS_CLONED_P(c) (RCLASS_EXT_READABLE(c)->cloned) +#define RCLASS_CLASSPATH(c) (RCLASS_EXT_READABLE(c)->classpath) + +// Superclasses can't be changed after initialization +#define RCLASS_SUPERCLASS_DEPTH(c) (RCLASS_EXT_PRIME(c)->superclass_depth) +#define RCLASS_SUPERCLASSES(c) (RCLASS_EXT_PRIME(c)->superclasses) +#define RCLASS_SUPERCLASSES_WITH_SELF_P(c) (RCLASS_EXT_PRIME(c)->superclasses_with_self) + +// Ruby Box doesn't make changes on these refined_class/attached_object/includer +#define RCLASS_REFINED_CLASS(c) (RCLASS_EXT_PRIME(c)->refined_class) +#define RCLASS_ATTACHED_OBJECT(c) (RCLASS_EXT_PRIME(c)->as.singleton_class.attached_object) +#define RCLASS_INCLUDER(c) (RCLASS_EXT_PRIME(c)->as.iclass.includer) + +// max IV count and variation count are just hints, so they don't need to be per-box +#define RCLASS_MAX_IV_COUNT(ext) (RCLASS_EXT_PRIME(ext)->max_iv_count) +#define RCLASS_VARIATION_COUNT(ext) (RCLASS_EXT_PRIME(ext)->variation_count) + +// Writable classext entries (instead of RCLASS_SET_*) because member data will be operated directly +#define RCLASS_WRITABLE_M_TBL(c) (RCLASS_EXT_WRITABLE(c)->m_tbl) +#define RCLASS_WRITABLE_CONST_TBL(c) (RCLASS_EXT_WRITABLE(c)->const_tbl) +#define RCLASS_WRITABLE_CALLABLE_M_TBL(c) (RCLASS_EXT_WRITABLE(c)->callable_m_tbl) +#define RCLASS_WRITABLE_CC_TBL(c) (RCLASS_EXT_WRITABLE(c)->cc_tbl) +#define RCLASS_WRITABLE_CVC_TBL(c) (RCLASS_EXT_WRITABLE(c)->cvc_tbl) +#define RCLASS_WRITABLE_SUBCLASSES(c) (RCLASS_EXT_WRITABLE(c)->subclasses) + +static inline void RCLASS_SET_SUPER(VALUE klass, VALUE super); +static inline void RCLASS_WRITE_SUPER(VALUE klass, VALUE super); +static inline void RCLASS_SET_CONST_TBL(VALUE klass, struct rb_id_table *table, bool shared); +static inline void RCLASS_WRITE_CONST_TBL(VALUE klass, struct rb_id_table *table, bool shared); +static inline void RCLASS_WRITE_CALLABLE_M_TBL(VALUE klass, struct rb_id_table *table); +static inline void RCLASS_WRITE_CC_TBL(VALUE klass, VALUE table); +static inline void RCLASS_SET_CVC_TBL(VALUE klass, struct rb_id_table *table); +static inline void RCLASS_WRITE_CVC_TBL(VALUE klass, struct rb_id_table *table); + +static inline void RCLASS_WRITE_SUPERCLASSES(VALUE klass, size_t depth, VALUE *superclasses, bool with_self); +static inline void RCLASS_SET_SUBCLASSES(VALUE klass, rb_subclass_anchor_t *anchor); +static inline void RCLASS_WRITE_BOX_SUPER_SUBCLASSES(VALUE klass, rb_box_subclasses_t *box_subclasses); +static inline void RCLASS_WRITE_BOX_MODULE_SUBCLASSES(VALUE klass, rb_box_subclasses_t *box_subclasses); + +static inline void RCLASS_SET_ORIGIN(VALUE klass, VALUE origin); +static inline void RCLASS_WRITE_ORIGIN(VALUE klass, VALUE origin); +static inline void RICLASS_SET_ORIGIN_SHARED_MTBL(VALUE iclass); +static inline void RICLASS_WRITE_ORIGIN_SHARED_MTBL(VALUE iclass); +static inline bool RICLASS_OWNS_M_TBL_P(VALUE iclass); + +static inline void RCLASS_SET_REFINED_CLASS(VALUE klass, VALUE refined); +static inline rb_alloc_func_t RCLASS_ALLOCATOR(VALUE klass); +static inline void RCLASS_SET_ALLOCATOR(VALUE klass, rb_alloc_func_t allocator); +static inline VALUE RCLASS_SET_ATTACHED_OBJECT(VALUE klass, VALUE attached_object); + +static inline void RCLASS_SET_INCLUDER(VALUE iclass, VALUE klass); +static inline void RCLASS_SET_MAX_IV_COUNT(VALUE klass, attr_index_t count); +static inline void RCLASS_SET_CLONED(VALUE klass, bool cloned); +static inline void RCLASS_SET_CLASSPATH(VALUE klass, VALUE classpath, bool permanent); +static inline void RCLASS_WRITE_CLASSPATH(VALUE klass, VALUE classpath, bool permanent); + +#define RCLASS_IS_ROOT FL_USER0 +// 1 is for RUBY_FL_SINGLETON or RMODULE_IS_REFINEMENT +#define RCLASS_PRIME_CLASSEXT_WRITABLE FL_USER2 +#define RCLASS_IS_INITIALIZED FL_USER3 +// 3 is RMODULE_IS_REFINEMENT for RMODULE +#define RCLASS_BOXABLE FL_USER4 + +static inline st_table * +RCLASS_CLASSEXT_TBL(VALUE klass) +{ + if (FL_TEST_RAW(klass, RCLASS_BOXABLE)) { + struct RClass_boxable *box_klass = (struct RClass_boxable *)klass; + return box_klass->box_classext_tbl; + } + return NULL; +} + +static inline void +RCLASS_SET_CLASSEXT_TBL(VALUE klass, st_table *tbl) +{ + RUBY_ASSERT(FL_TEST_RAW(klass, RCLASS_BOXABLE)); + struct RClass_boxable *box_klass = (struct RClass_boxable *)klass; + box_klass->box_classext_tbl = tbl; +} /* class.c */ +rb_classext_t * rb_class_duplicate_classext(rb_classext_t *orig, VALUE obj, const rb_box_t *box); +void rb_class_ensure_writable(VALUE obj); + +void rb_class_set_box_classext(VALUE obj, const rb_box_t *box, rb_classext_t *ext); + +static inline int +RCLASS_SET_BOX_CLASSEXT(VALUE obj, const rb_box_t *box, rb_classext_t *ext) +{ + int first_set = 0; + st_table *tbl = RCLASS_CLASSEXT_TBL(obj); + VM_ASSERT(BOX_USER_P(box)); // non-prime classext is only for user box, with box_object + VM_ASSERT(box->box_object); + VM_ASSERT(RCLASSEXT_BOX(ext) == box); + if (!tbl) { + tbl = st_init_numtable_with_size(1); + RCLASS_SET_CLASSEXT_TBL(obj, tbl); + } + if (rb_st_table_size(tbl) == 0) { + first_set = 1; + } + + rb_class_set_box_classext(obj, box, ext); + + return first_set; +} + +#define VM_ASSERT_BOXABLE_TYPE(klass) \ + VM_ASSERT(RB_TYPE_P(klass, T_CLASS) || RB_TYPE_P(klass, T_MODULE) || RB_TYPE_P(klass, T_ICLASS), "%s is not boxable type", rb_type_str(BUILTIN_TYPE(klass))) + +static inline bool +RCLASS_PRIME_CLASSEXT_READABLE_P(VALUE klass) +{ + VM_ASSERT(klass != 0, "klass should be a valid object"); + VM_ASSERT_BOXABLE_TYPE(klass); + // if the lookup table exists, then it means the prime classext is NOT directly readable. + return !FL_TEST_RAW(klass, RCLASS_BOXABLE) || RCLASS_CLASSEXT_TBL(klass) == NULL; +} + +static inline bool +RCLASS_PRIME_CLASSEXT_WRITABLE_P(VALUE klass) +{ + VM_ASSERT(klass != 0, "klass should be a valid object"); + VM_ASSERT_BOXABLE_TYPE(klass); + return FL_TEST(klass, RCLASS_PRIME_CLASSEXT_WRITABLE); +} + +static inline void +RCLASS_SET_PRIME_CLASSEXT_WRITABLE(VALUE klass, bool writable) +{ + VM_ASSERT(klass != 0, "klass should be a valid object"); + VM_ASSERT_BOXABLE_TYPE(klass); + if (writable) { + FL_SET(klass, RCLASS_PRIME_CLASSEXT_WRITABLE); + } + else { + FL_UNSET(klass, RCLASS_PRIME_CLASSEXT_WRITABLE); + } +} + +static inline rb_classext_t * +RCLASS_EXT_TABLE_LOOKUP_INTERNAL(VALUE obj, const rb_box_t *box) +{ + st_data_t classext_ptr; + st_table *classext_tbl = RCLASS_CLASSEXT_TBL(obj); + if (classext_tbl) { + if (rb_st_lookup(classext_tbl, (st_data_t)box->box_object, &classext_ptr)) { + return (rb_classext_t *)classext_ptr; + } + } + return NULL; +} + +static inline rb_classext_t * +RCLASS_EXT_READABLE_LOOKUP(VALUE obj, const rb_box_t *box) +{ + rb_classext_t *ext = RCLASS_EXT_TABLE_LOOKUP_INTERNAL(obj, box); + if (ext) + return ext; + // Classext for the ns not found. Refer the prime one instead. + return RCLASS_EXT_PRIME(obj); +} + +static inline rb_classext_t * +RCLASS_EXT_READABLE_IN_BOX(VALUE obj, const rb_box_t *box) +{ + if (BOX_ROOT_P(box) + || RCLASS_PRIME_CLASSEXT_READABLE_P(obj)) { + return RCLASS_EXT_PRIME(obj); + } + return RCLASS_EXT_READABLE_LOOKUP(obj, box); +} + +static inline rb_classext_t * +RCLASS_EXT_READABLE(VALUE obj) +{ + const rb_box_t *box; + if (RCLASS_PRIME_CLASSEXT_READABLE_P(obj)) { + return RCLASS_EXT_PRIME(obj); + } + // delay determining the current box to optimize for unmodified classes + box = rb_current_box(); + if (BOX_ROOT_P(box)) { + return RCLASS_EXT_PRIME(obj); + } + return RCLASS_EXT_READABLE_LOOKUP(obj, box); +} + +static inline rb_classext_t * +RCLASS_EXT_WRITABLE_LOOKUP(VALUE obj, const rb_box_t *box) +{ + rb_classext_t *ext; + int first_set = 0; + + ext = RCLASS_EXT_TABLE_LOOKUP_INTERNAL(obj, box); + if (ext) + return ext; + + RB_VM_LOCKING() { + // re-check the classext is not created to avoid the multi-thread race + ext = RCLASS_EXT_TABLE_LOOKUP_INTERNAL(obj, box); + if (!ext) { + ext = rb_class_duplicate_classext(RCLASS_EXT_PRIME(obj), obj, box); + first_set = RCLASS_SET_BOX_CLASSEXT(obj, box, ext); + if (first_set) { + // TODO: are there any case that a class/module become non-writable after its birthtime? + RCLASS_SET_PRIME_CLASSEXT_WRITABLE(obj, false); + } + } + } + return ext; +} + +static inline rb_classext_t * +RCLASS_EXT_WRITABLE_IN_BOX(VALUE obj, const rb_box_t *box) +{ + if (BOX_ROOT_P(box) + || RCLASS_PRIME_CLASSEXT_WRITABLE_P(obj)) { + return RCLASS_EXT_PRIME(obj); + } + return RCLASS_EXT_WRITABLE_LOOKUP(obj, box); +} + +static inline rb_classext_t * +RCLASS_EXT_WRITABLE(VALUE obj) +{ + const rb_box_t *box; + if (LIKELY(RCLASS_PRIME_CLASSEXT_WRITABLE_P(obj))) { + return RCLASS_EXT_PRIME(obj); + } + // delay determining the current box to optimize for unmodified classes + box = rb_current_box(); + if (BOX_ROOT_P(box)) { + return RCLASS_EXT_PRIME(obj); + } + return RCLASS_EXT_WRITABLE_LOOKUP(obj, box); +} + +static inline void +RCLASSEXT_SET_ORIGIN(rb_classext_t *ext, VALUE klass, VALUE origin) +{ + RB_OBJ_WRITE(klass, &(RCLASSEXT_ORIGIN(ext)), origin); +} + +static inline void +RCLASSEXT_SET_INCLUDER(rb_classext_t *ext, VALUE klass, VALUE includer) +{ + RUBY_ASSERT(RB_TYPE_P(klass, T_ICLASS)); + RB_OBJ_WRITE(klass, &(RCLASSEXT_INCLUDER(ext)), includer); +} + +/* class.c */ +typedef void rb_class_classext_foreach_callback_func(rb_classext_t *classext, bool is_prime, VALUE box_value, void *arg); +void rb_class_classext_foreach(VALUE klass, rb_class_classext_foreach_callback_func *func, void *arg); void rb_class_subclass_add(VALUE super, VALUE klass); -void rb_class_remove_from_super_subclasses(VALUE); +void rb_class_classext_free_subclasses(rb_classext_t *, VALUE, bool); +void rb_class_foreach_subclass(VALUE klass, void (*f)(VALUE, VALUE), VALUE); void rb_class_update_superclasses(VALUE); -size_t rb_class_superclasses_memsize(VALUE); -void rb_class_remove_subclass_head(VALUE); int rb_singleton_class_internal_p(VALUE sklass); +VALUE rb_class_set_super(VALUE klass, VALUE super); VALUE rb_class_boot(VALUE); VALUE rb_class_s_alloc(VALUE klass); VALUE rb_module_s_alloc(VALUE klass); -void rb_module_set_initialized(VALUE module); +void rb_class_set_initialized(VALUE klass); void rb_module_check_initializable(VALUE module); VALUE rb_make_metaclass(VALUE, VALUE); VALUE rb_include_class_new(VALUE, VALUE); -void rb_class_foreach_subclass(VALUE klass, void (*f)(VALUE, VALUE), VALUE); -void rb_class_detach_subclasses(VALUE); -void rb_class_detach_module_subclasses(VALUE); -void rb_class_remove_from_module_subclasses(VALUE); VALUE rb_define_class_id_under_no_pin(VALUE outer, ID id, VALUE super); VALUE rb_obj_methods(int argc, const VALUE *argv, VALUE obj); VALUE rb_obj_protected_methods(int argc, const VALUE *argv, VALUE obj); @@ -129,59 +510,297 @@ VALUE rb_special_singleton_class(VALUE); VALUE rb_singleton_class_clone_and_attach(VALUE obj, VALUE attach); VALUE rb_singleton_class_get(VALUE obj); void rb_undef_methods_from(VALUE klass, VALUE super); - -static inline void RCLASS_SET_ORIGIN(VALUE klass, VALUE origin); -static inline void RICLASS_SET_ORIGIN_SHARED_MTBL(VALUE iclass); -static inline VALUE RCLASS_SUPER(VALUE klass); -static inline VALUE RCLASS_SET_SUPER(VALUE klass, VALUE super); -static inline void RCLASS_SET_INCLUDER(VALUE iclass, VALUE klass); - -MJIT_SYMBOL_EXPORT_BEGIN VALUE rb_class_inherited(VALUE, VALUE); VALUE rb_keyword_error_new(const char *, VALUE); -MJIT_SYMBOL_EXPORT_END + +rb_classext_t *rb_class_unlink_classext(VALUE klass, const rb_box_t *box); +void rb_class_classext_free(VALUE klass, rb_classext_t *ext, bool is_prime); +void rb_iclass_classext_free(VALUE klass, rb_classext_t *ext, bool is_prime); + +RUBY_SYMBOL_EXPORT_BEGIN + +/* for objspace */ +VALUE rb_class_super_of(VALUE klass); +VALUE rb_class_singleton_p(VALUE klass); +unsigned char rb_class_variation_count(VALUE klass); + +RUBY_SYMBOL_EXPORT_END + +static inline bool +RCLASS_SINGLETON_P(VALUE klass) +{ + return RB_TYPE_P(klass, T_CLASS) && FL_TEST_RAW(klass, FL_SINGLETON); +} + +static inline void +RCLASS_SET_SUPER(VALUE klass, VALUE super) +{ + RB_OBJ_WRITE(klass, &RCLASSEXT_SUPER(RCLASS_EXT_PRIME(klass)), super); +} + +static inline void +RCLASS_WRITE_SUPER(VALUE klass, VALUE super) +{ + RB_OBJ_WRITE(klass, &RCLASSEXT_SUPER(RCLASS_EXT_WRITABLE(klass)), super); +} + +static inline VALUE +RCLASS_WRITABLE_ENSURE_FIELDS_OBJ(VALUE obj) +{ + RUBY_ASSERT(RB_TYPE_P(obj, RUBY_T_CLASS) || RB_TYPE_P(obj, RUBY_T_MODULE)); + rb_classext_t *ext = RCLASS_EXT_WRITABLE(obj); + if (!ext->fields_obj) { + RB_OBJ_WRITE(obj, &ext->fields_obj, rb_imemo_fields_new(obj, 1, true)); + } + return ext->fields_obj; +} + +static inline VALUE +RCLASS_WRITABLE_FIELDS_OBJ(VALUE obj) +{ + RUBY_ASSERT(RB_TYPE_P(obj, RUBY_T_CLASS) || RB_TYPE_P(obj, RUBY_T_MODULE)); + return RCLASSEXT_FIELDS_OBJ(RCLASS_EXT_WRITABLE(obj)); +} + +static inline void +RCLASSEXT_SET_FIELDS_OBJ(VALUE obj, rb_classext_t *ext, VALUE fields_obj) +{ + RUBY_ASSERT(RB_TYPE_P(obj, RUBY_T_CLASS) || RB_TYPE_P(obj, RUBY_T_MODULE)); + + RB_OBJ_ATOMIC_WRITE(obj, &ext->fields_obj, fields_obj); +} + +static inline void +RCLASS_WRITABLE_SET_FIELDS_OBJ(VALUE obj, VALUE fields_obj) +{ + RUBY_ASSERT(RB_TYPE_P(obj, RUBY_T_CLASS) || RB_TYPE_P(obj, RUBY_T_MODULE)); + + RCLASSEXT_SET_FIELDS_OBJ(obj, RCLASS_EXT_WRITABLE(obj), fields_obj); +} + +static inline uint32_t +RCLASS_FIELDS_COUNT(VALUE obj) +{ + RUBY_ASSERT(RB_TYPE_P(obj, RUBY_T_CLASS) || RB_TYPE_P(obj, RUBY_T_MODULE)); + + VALUE fields_obj = RCLASS_WRITABLE_FIELDS_OBJ(obj); + if (fields_obj) { + if (rb_shape_obj_too_complex_p(fields_obj)) { + return (uint32_t)rb_st_table_size(rb_imemo_fields_complex_tbl(fields_obj)); + } + else { + return RSHAPE_LEN(RBASIC_SHAPE_ID(fields_obj)); + } + } + return 0; +} + +static inline void +RCLASS_SET_M_TBL(VALUE klass, struct rb_id_table *table) +{ + RCLASSEXT_M_TBL(RCLASS_EXT_PRIME(klass)) = table; +} + +static inline void +RCLASS_WRITE_M_TBL(VALUE klass, struct rb_id_table *table) +{ + RCLASSEXT_M_TBL(RCLASS_EXT_WRITABLE(klass)) = table; +} + +static inline void +RCLASS_SET_CONST_TBL(VALUE klass, struct rb_id_table *table, bool shared) +{ + rb_classext_t *ext = RCLASS_EXT_PRIME(klass); + RCLASSEXT_CONST_TBL(ext) = table; + if (shared) + RCLASSEXT_SHARED_CONST_TBL(ext) = true; +} + +static inline void +RCLASS_WRITE_CONST_TBL(VALUE klass, struct rb_id_table *table, bool shared) +{ + rb_classext_t *ext = RCLASS_EXT_WRITABLE(klass); + RCLASSEXT_CONST_TBL(ext) = table; + if (shared) + RCLASSEXT_SHARED_CONST_TBL(ext) = true; +} + +static inline void +RCLASS_WRITE_CALLABLE_M_TBL(VALUE klass, struct rb_id_table *table) +{ + RCLASSEXT_CALLABLE_M_TBL(RCLASS_EXT_WRITABLE(klass)) = table; +} + +static inline void +RCLASS_WRITE_CC_TBL(VALUE klass, VALUE table) +{ + RB_OBJ_ATOMIC_WRITE(klass, &RCLASSEXT_CC_TBL(RCLASS_EXT_WRITABLE(klass)), table); +} + +static inline void +RCLASS_SET_CVC_TBL(VALUE klass, struct rb_id_table *table) +{ + RCLASSEXT_CVC_TBL(RCLASS_EXT_PRIME(klass)) = table; +} + +static inline void +RCLASS_WRITE_CVC_TBL(VALUE klass, struct rb_id_table *table) +{ + RCLASSEXT_CVC_TBL(RCLASS_EXT_WRITABLE(klass)) = table; +} + +static inline void +RCLASS_SET_REFINED_CLASS(VALUE klass, VALUE refined) +{ + RB_OBJ_WRITE(klass, &RCLASSEXT_REFINED_CLASS(RCLASS_EXT_PRIME(klass)), refined); +} + +static inline rb_alloc_func_t +RCLASS_ALLOCATOR(VALUE klass) +{ + RBIMPL_ASSERT_TYPE(klass, T_CLASS); + if (RCLASS_SINGLETON_P(klass)) { + return 0; + } + return RCLASS_EXT_PRIME(klass)->as.class.allocator; +} + +static inline void +RCLASS_SET_ALLOCATOR(VALUE klass, rb_alloc_func_t allocator) +{ + RUBY_ASSERT(RB_TYPE_P(klass, T_CLASS)); + RUBY_ASSERT(!RCLASS_SINGLETON_P(klass)); + RCLASS_EXT_PRIME(klass)->as.class.allocator = allocator; // Allocator is set only on the initial definition +} static inline void RCLASS_SET_ORIGIN(VALUE klass, VALUE origin) { - RB_OBJ_WRITE(klass, &RCLASS_ORIGIN(klass), origin); - if (klass != origin) FL_SET(origin, RICLASS_IS_ORIGIN); + rb_classext_t *ext = RCLASS_EXT_PRIME(klass); + RB_OBJ_WRITE(klass, &RCLASSEXT_ORIGIN(ext), origin); + if (klass != origin) RCLASSEXT_ICLASS_IS_ORIGIN(RCLASS_EXT_WRITABLE(origin)) = true; +} + +static inline void +RCLASS_WRITE_ORIGIN(VALUE klass, VALUE origin) +{ + rb_classext_t *ext = RCLASS_EXT_WRITABLE(klass); + RB_OBJ_WRITE(klass, &RCLASSEXT_ORIGIN(ext), origin); + if (klass != origin) RCLASSEXT_ICLASS_IS_ORIGIN(RCLASS_EXT_WRITABLE(origin)) = true; } static inline void RICLASS_SET_ORIGIN_SHARED_MTBL(VALUE iclass) { - FL_SET(iclass, RICLASS_ORIGIN_SHARED_MTBL); + RCLASSEXT_ICLASS_ORIGIN_SHARED_MTBL(RCLASS_EXT_PRIME(iclass)) = true; +} + +static inline void +RICLASS_WRITE_ORIGIN_SHARED_MTBL(VALUE iclass) +{ + RCLASSEXT_ICLASS_ORIGIN_SHARED_MTBL(RCLASS_EXT_WRITABLE(iclass)) = true; } static inline bool RICLASS_OWNS_M_TBL_P(VALUE iclass) { - return FL_TEST_RAW(iclass, RICLASS_IS_ORIGIN | RICLASS_ORIGIN_SHARED_MTBL) == RICLASS_IS_ORIGIN; + rb_classext_t *ext = RCLASS_EXT_READABLE(iclass); + return RCLASSEXT_ICLASS_IS_ORIGIN(ext) && !RCLASSEXT_ICLASS_ORIGIN_SHARED_MTBL(ext); } static inline void RCLASS_SET_INCLUDER(VALUE iclass, VALUE klass) { + RUBY_ASSERT(RB_TYPE_P(iclass, T_ICLASS)); RB_OBJ_WRITE(iclass, &RCLASS_INCLUDER(iclass), klass); } -static inline VALUE -RCLASS_SUPER(VALUE klass) +static inline void +RCLASS_WRITE_SUPERCLASSES(VALUE klass, size_t depth, VALUE *superclasses, bool with_self) +{ + RUBY_ASSERT(depth <= RCLASS_MAX_SUPERCLASS_DEPTH); + + rb_classext_t *ext = RCLASS_EXT_PRIME(klass); + RCLASSEXT_SUPERCLASS_DEPTH(ext) = depth; + RCLASSEXT_SUPERCLASSES(ext) = superclasses; + RCLASSEXT_SUPERCLASSES_WITH_SELF(ext) = with_self; +} + +static inline void +RCLASS_SET_SUBCLASSES(VALUE klass, struct rb_subclass_anchor *anchor) { - return RCLASS(klass)->super; + rb_classext_t *ext = RCLASS_EXT_PRIME(klass); + RCLASSEXT_SUBCLASSES(ext) = anchor; +} + +static inline void +RCLASS_WRITE_BOX_SUPER_SUBCLASSES(VALUE klass, rb_box_subclasses_t *box_subclasses) +{ + rb_classext_t *ext = RCLASS_EXT_WRITABLE(klass); + if (RCLASSEXT_BOX_SUPER_SUBCLASSES(ext)) + rb_box_subclasses_ref_dec(RCLASSEXT_BOX_SUPER_SUBCLASSES(ext)); + RCLASSEXT_BOX_SUPER_SUBCLASSES(ext) = rb_box_subclasses_ref_inc(box_subclasses); +} + +static inline void +RCLASS_WRITE_BOX_MODULE_SUBCLASSES(VALUE klass, rb_box_subclasses_t *box_subclasses) +{ + rb_classext_t *ext = RCLASS_EXT_WRITABLE(klass); + if (RCLASSEXT_BOX_MODULE_SUBCLASSES(ext)) + rb_box_subclasses_ref_dec(RCLASSEXT_BOX_MODULE_SUBCLASSES(ext)); + RCLASSEXT_BOX_MODULE_SUBCLASSES(ext) = rb_box_subclasses_ref_inc(box_subclasses); +} + +static inline void +RCLASS_SET_CLASSPATH(VALUE klass, VALUE classpath, bool permanent) +{ + rb_classext_t *ext = RCLASS_EXT_READABLE(klass); + assert(BUILTIN_TYPE(klass) == T_CLASS || BUILTIN_TYPE(klass) == T_MODULE); + assert(classpath == 0 || BUILTIN_TYPE(classpath) == T_STRING); + assert(FL_TEST_RAW(classpath, RUBY_FL_SHAREABLE)); + + RB_OBJ_WRITE(klass, &(RCLASSEXT_CLASSPATH(ext)), classpath); + RCLASSEXT_PERMANENT_CLASSPATH(ext) = permanent; +} + +static inline void +RCLASS_WRITE_CLASSPATH(VALUE klass, VALUE classpath, bool permanent) +{ + rb_classext_t *ext = RCLASS_EXT_WRITABLE(klass); + assert(BUILTIN_TYPE(klass) == T_CLASS || BUILTIN_TYPE(klass) == T_MODULE); + assert(classpath == 0 || BUILTIN_TYPE(classpath) == T_STRING); + assert(!RB_FL_ABLE(classpath) || FL_TEST_RAW(classpath, RUBY_FL_SHAREABLE)); + + RB_OBJ_WRITE(klass, &(RCLASSEXT_CLASSPATH(ext)), classpath); + RCLASSEXT_PERMANENT_CLASSPATH(ext) = permanent; } static inline VALUE -RCLASS_SET_SUPER(VALUE klass, VALUE super) +RCLASS_SET_ATTACHED_OBJECT(VALUE klass, VALUE attached_object) { - if (super) { - rb_class_remove_from_super_subclasses(klass); - rb_class_subclass_add(super, klass); - } - RB_OBJ_WRITE(klass, &RCLASS(klass)->super, super); - rb_class_update_superclasses(klass); - return super; + assert(RCLASS_SINGLETON_P(klass)); + + RB_OBJ_WRITE(klass, &RCLASS_EXT_PRIME(klass)->as.singleton_class.attached_object, attached_object); + return attached_object; +} + +static inline void +RCLASS_SET_MAX_IV_COUNT(VALUE klass, attr_index_t count) +{ + RCLASS_MAX_IV_COUNT(klass) = count; +} + +static inline void +RCLASS_SET_CLONED(VALUE klass, bool cloned) +{ + RCLASSEXT_CLONED(RCLASS_EXT_PRIME(klass)) = cloned; +} + +static inline bool +RCLASS_INITIALIZED_P(VALUE klass) +{ + VM_ASSERT(RB_TYPE_P(klass, T_CLASS) || RB_TYPE_P(klass, T_MODULE)); + return FL_TEST_RAW(klass, RCLASS_IS_INITIALIZED); } #endif /* INTERNAL_CLASS_H */ diff --git a/internal/cmdlineopt.h b/internal/cmdlineopt.h index bf52f1214b..aed209e2a2 100644 --- a/internal/cmdlineopt.h +++ b/internal/cmdlineopt.h @@ -1,7 +1,6 @@ #ifndef INTERNAL_CMDLINEOPT_H /*-*-C-*-vi:se ft=c:*/ #define INTERNAL_CMDLINEOPT_H -#include "mjit.h" #include "yjit.h" typedef struct { @@ -23,11 +22,12 @@ typedef struct ruby_cmdline_options { ruby_features_t features; ruby_features_t warn; unsigned int dump; -#if USE_MJIT - struct mjit_options mjit; -#endif + long backtrace_length_limit; + + const char *crash_report; - int sflag, xflag; + signed int sflag: 2; + unsigned int xflag: 1; unsigned int warning: 1; unsigned int verbose: 1; unsigned int do_loop: 1; @@ -39,6 +39,9 @@ typedef struct ruby_cmdline_options { #if USE_YJIT unsigned int yjit: 1; #endif +#if USE_ZJIT + unsigned int zjit: 1; +#endif } ruby_cmdline_options_t; struct ruby_opt_message { diff --git a/internal/compile.h b/internal/compile.h index d32c2233c9..2ece5396f6 100644 --- a/internal/compile.h +++ b/internal/compile.h @@ -17,6 +17,7 @@ struct rb_iseq_struct; /* in vm_core.h */ /* compile.c */ int rb_dvar_defined(ID, const struct rb_iseq_struct *); int rb_local_defined(ID, const struct rb_iseq_struct *); +int rb_insn_len(VALUE insn); const char *rb_insns_name(int i); VALUE rb_insns_name_array(void); int rb_iseq_cdhash_cmp(VALUE val, VALUE lit); @@ -27,9 +28,7 @@ int rb_vm_insn_addr2insn(const void *); int rb_vm_insn_decode(const VALUE encoded); extern bool ruby_vm_keep_script_lines; -MJIT_SYMBOL_EXPORT_BEGIN /* iseq.c (export) */ rb_event_flag_t rb_iseq_event_flags(const struct rb_iseq_struct *iseq, size_t pos); -MJIT_SYMBOL_EXPORT_END #endif /* INTERNAL_COMPILE_H */ diff --git a/internal/concurrent_set.h b/internal/concurrent_set.h new file mode 100644 index 0000000000..76cbefab04 --- /dev/null +++ b/internal/concurrent_set.h @@ -0,0 +1,21 @@ +#ifndef RUBY_RACTOR_SAFE_TABLE_H +#define RUBY_RACTOR_SAFE_TABLE_H + +#include "ruby/atomic.h" +#include "ruby/ruby.h" + +struct rb_concurrent_set_funcs { + VALUE (*hash)(VALUE key); + bool (*cmp)(VALUE a, VALUE b); + VALUE (*create)(VALUE key, void *data); + void (*free)(VALUE key); +}; + +VALUE rb_concurrent_set_new(const struct rb_concurrent_set_funcs *funcs, int capacity); +rb_atomic_t rb_concurrent_set_size(VALUE set_obj); +VALUE rb_concurrent_set_find(VALUE *set_obj_ptr, VALUE key); +VALUE rb_concurrent_set_find_or_insert(VALUE *set_obj_ptr, VALUE key, void *data); +VALUE rb_concurrent_set_delete_by_identity(VALUE set_obj, VALUE key); +void rb_concurrent_set_foreach_with_replace(VALUE set_obj, int (*callback)(VALUE *key, void *data), void *data); + +#endif diff --git a/internal/cont.h b/internal/cont.h index c3b091668a..dcf6f820a3 100644 --- a/internal/cont.h +++ b/internal/cont.h @@ -22,11 +22,13 @@ void rb_jit_cont_init(void); void rb_jit_cont_each_iseq(rb_iseq_callback callback, void *data); void rb_jit_cont_finish(void); +/* vm.c */ +void rb_free_shared_fiber_pool(void); + // Copy locals from the current execution to the specified fiber. VALUE rb_fiber_inherit_storage(struct rb_execution_context_struct *ec, struct rb_fiber_struct *fiber); VALUE rb_fiberptr_self(struct rb_fiber_struct *fiber); unsigned int rb_fiberptr_blocking(struct rb_fiber_struct *fiber); struct rb_execution_context_struct * rb_fiberptr_get_ec(struct rb_fiber_struct *fiber); - #endif /* INTERNAL_CONT_H */ diff --git a/internal/encoding.h b/internal/encoding.h index a3b81bd388..38bf8fc9da 100644 --- a/internal/encoding.h +++ b/internal/encoding.h @@ -11,22 +11,29 @@ #include "ruby/ruby.h" /* for ID */ #include "ruby/encoding.h" /* for rb_encoding */ -#define rb_enc_autoload_p(enc) (!rb_enc_mbmaxlen(enc)) #define rb_is_usascii_enc(enc) ((enc) == rb_usascii_encoding()) #define rb_is_ascii8bit_enc(enc) ((enc) == rb_ascii8bit_encoding()) #define rb_is_locale_enc(enc) ((enc) == rb_locale_encoding()) /* encoding.c */ ID rb_id_encoding(void); +const char * rb_enc_inspect_name(rb_encoding *enc); rb_encoding *rb_enc_get_from_index(int index); rb_encoding *rb_enc_check_str(VALUE str1, VALUE str2); int rb_encdb_replicate(const char *alias, const char *orig); int rb_encdb_alias(const char *alias, const char *orig); int rb_enc_autoload(rb_encoding *enc); +bool rb_enc_autoload_p(rb_encoding *enc); int rb_encdb_dummy(const char *name); void rb_encdb_declare(const char *name); void rb_enc_set_base(const char *name, const char *orig); int rb_enc_set_dummy(int index); +void rb_enc_raw_set(VALUE obj, rb_encoding *enc); +int rb_enc_registered(const char *name); + PUREFUNC(int rb_data_is_encoding(VALUE obj)); +/* vm.c */ +void rb_free_global_enc_table(void); + #endif /* INTERNAL_ENCODING_H */ diff --git a/internal/error.h b/internal/error.h index 11601858f4..ae9a13fcec 100644 --- a/internal/error.h +++ b/internal/error.h @@ -29,15 +29,37 @@ #define rb_raise_static(e, m) \ rb_raise_cstr_i((e), rb_str_new_static((m), rb_strlen_lit(m))) #ifdef RUBY_FUNCTION_NAME_STRING -# define rb_sys_fail_path(path) rb_sys_fail_path_in(RUBY_FUNCTION_NAME_STRING, path) # define rb_syserr_fail_path(err, path) rb_syserr_fail_path_in(RUBY_FUNCTION_NAME_STRING, (err), (path)) # define rb_syserr_new_path(err, path) rb_syserr_new_path_in(RUBY_FUNCTION_NAME_STRING, (err), (path)) #else -# define rb_sys_fail_path(path) rb_sys_fail_str(path) # define rb_syserr_fail_path(err, path) rb_syserr_fail_str((err), (path)) # define rb_syserr_new_path(err, path) rb_syserr_new_str((err), (path)) #endif +#define rb_sys_fail(mesg) \ +do { \ + int errno_to_fail = errno; \ + rb_syserr_fail(errno_to_fail, (mesg)); \ +} while (0) + +#define rb_sys_fail_str(mesg) \ +do { \ + int errno_to_fail = errno; \ + rb_syserr_fail_str(errno_to_fail, (mesg)); \ +} while (0) + +#define rb_sys_fail_path(path) \ +do { \ + int errno_to_fail = errno; \ + rb_syserr_fail_path(errno_to_fail, (path)); \ +} while (0) + +#define rb_sys_fail_sprintf(...) \ +do { \ + int errno_to_fail = errno; \ + rb_syserr_fail_str(errno_to_fail, rb_sprintf("" __VA_ARGS__)); \ +} while (0) + /* error.c */ extern long rb_backtrace_length_limit; extern VALUE rb_eEAGAIN; @@ -50,13 +72,14 @@ const char *rb_builtin_type_name(int t); const char *rb_builtin_class_name(VALUE x); PRINTF_ARGS(void rb_warn_deprecated(const char *fmt, const char *suggest, ...), 1, 3); PRINTF_ARGS(void rb_warn_deprecated_to_remove(const char *removal, const char *fmt, const char *suggest, ...), 2, 4); +PRINTF_ARGS(void rb_warn_reserved_name(const char *removal, const char *fmt, ...), 2, 3); #if RUBY_DEBUG # include "ruby/version.h" -# define RUBY_VERSION_SINCE(major, minor) (RUBY_API_VERSION_CODE >= (major * 10000) + (minor) * 100) -# define RUBY_VERSION_BEFORE(major, minor) (RUBY_API_VERSION_CODE < (major * 10000) + (minor) * 100) +# define RUBY_VERSION_SINCE(major, minor) (RUBY_API_VERSION_CODE >= (major) * 10000 + (minor) * 100) +# define RUBY_VERSION_BEFORE(major, minor) (RUBY_API_VERSION_CODE < (major) * 10000 + (minor) * 100) # if defined(RBIMPL_WARNING_PRAGMA0) # define RBIMPL_TODO0(x) RBIMPL_WARNING_PRAGMA0(message(x)) -# elif RBIMPL_COMPILER_SINCE(MSVC, 12, 0, 0) +# elif RBIMPL_COMPILER_IS(MSVC) # define RBIMPL_TODO0(x) __pragma(message(x)) # endif @@ -88,6 +111,14 @@ rb_deprecated_method_to_be_removed(const char *removal) RBIMPL_ATTR_DIAGNOSE_IF(RUBY_VERSION_STRING_SINCE(removal), "deprecated method to be removed", "error") { } + +RBIMPL_ATTR_FORCEINLINE() +static void +rb_diagnose_reserved_name_at(const char *coming) + RBIMPL_ATTR_DIAGNOSE_IF(!RUBY_VERSION_isdigit(coming[0]), "malformed version number", "error") + RBIMPL_ATTR_DIAGNOSE_IF(RUBY_VERSION_STRING_SINCE(coming), "reserved name already in use", "error") +{ +} # else RBIMPL_ATTR_ERROR(("deprecated")) void rb_deprecated_method_to_be_removed(const char *); @@ -95,16 +126,32 @@ void rb_deprecated_method_to_be_removed(const char *); (sizeof(char[1-2*(!RUBY_VERSION_isdigit(removal[0]) || RUBY_VERSION_STRING_SINCE(removal))])!=1 ? \ rb_deprecated_method_to_be_removed(removal) : \ RBIMPL_ASSERT_NOTHING) + +RBIMPL_ATTR_ERROR(("deprecated")) +void rb_diagnose_reserved_name_at(const char *); +# define rb_diagnose_reserved_name_at(coming) \ + (sizeof(char[1-2*(!RUBY_VERSION_isdigit(coming[0]) || RUBY_VERSION_STRING_SINCE(coming))])!=1 ? \ + rb_diagnose_reserved_name_at(coming) : \ + RBIMPL_ASSERT_NOTHING) + # endif # define rb_warn_deprecated_to_remove_at(removal, ...) \ (rb_deprecated_method_to_be_removed(#removal), \ rb_warn_deprecated_to_remove(#removal, __VA_ARGS__)) + +# define rb_warn_reserved_name_at(coming, ...) \ + (rb_diagnose_reserved_name_at(#coming), \ + rb_warn_reserved_name(#coming, __VA_ARGS__)) # endif #endif #ifndef rb_warn_deprecated_to_remove_at # define rb_warn_deprecated_to_remove_at(removal, ...) \ rb_warn_deprecated_to_remove(#removal, __VA_ARGS__) #endif +#ifndef rb_warn_reserved_name_at +# define rb_warn_reserved_name_at(removal, ...) \ + rb_warn_reserved_name(#removal, __VA_ARGS__) +#endif #ifndef RUBY_VERSION_SINCE # define RUBY_VERSION_SINCE(major, minor) 0 #endif @@ -120,6 +167,8 @@ VALUE rb_syntax_error_append(VALUE, VALUE, int, int, rb_encoding*, const char*, PRINTF_ARGS(void rb_enc_warn(rb_encoding *enc, const char *fmt, ...), 2, 3); PRINTF_ARGS(void rb_sys_enc_warning(rb_encoding *enc, const char *fmt, ...), 2, 3); PRINTF_ARGS(void rb_syserr_enc_warning(int err, rb_encoding *enc, const char *fmt, ...), 3, 4); +PRINTF_ARGS(void rb_enc_compile_warning(rb_encoding *enc, const char *file, int line, const char *fmt, ...), 4, 5); +PRINTF_ARGS(void rb_enc_compile_warn(rb_encoding *enc, const char *file, int line, const char *fmt, ...), 4, 5); rb_warning_category_t rb_warning_category_from_name(VALUE category); bool rb_warning_category_enabled_p(rb_warning_category_t category); VALUE rb_name_err_new(VALUE mesg, VALUE recv, VALUE method); @@ -136,6 +185,7 @@ NORETURN(static inline void rb_key_err_raise(VALUE mesg, VALUE recv, VALUE name) static inline void Check_Type(VALUE v, enum ruby_value_type t); static inline bool rb_typeddata_is_instance_of_inline(VALUE obj, const rb_data_type_t *data_type); #define rb_typeddata_is_instance_of rb_typeddata_is_instance_of_inline +void rb_bug_without_die(const char *fmt, ...); RUBY_SYMBOL_EXPORT_BEGIN /* error.c (export) */ @@ -147,6 +197,9 @@ VALUE rb_syserr_new_path_in(const char *func_name, int n, VALUE path); #endif RUBY_SYMBOL_EXPORT_END +/* vm.c */ +void rb_free_warning(void); + static inline void rb_raise_cstr_i(VALUE etype, VALUE mesg) { @@ -182,10 +235,18 @@ rb_key_err_raise(VALUE mesg, VALUE recv, VALUE name) rb_exc_raise(exc); } +RBIMPL_ATTR_NONNULL((2)) static inline bool rb_typeddata_is_instance_of_inline(VALUE obj, const rb_data_type_t *data_type) { - return RB_TYPE_P(obj, T_DATA) && RTYPEDDATA_P(obj) && (RTYPEDDATA_TYPE(obj) == data_type); + return rbimpl_obj_typeddata_p(obj) && (RTYPEDDATA_TYPE(obj) == data_type); } +typedef enum { + rb_stack_overflow_prevention = 0, // VM stack overflow or about to machine stack overflow + rb_stack_overflow_signal = 1, // machine stack overflow but may be recoverable + rb_stack_overflow_fatal = 2, // fatal machine stack overflow +} ruby_stack_overflow_critical_level; +NORETURN(void rb_ec_stack_overflow(struct rb_execution_context_struct *ec, ruby_stack_overflow_critical_level crit)); + #endif /* INTERNAL_ERROR_H */ diff --git a/internal/eval.h b/internal/eval.h index e594d8516d..17ade0a7f1 100644 --- a/internal/eval.h +++ b/internal/eval.h @@ -11,17 +11,27 @@ * header (related to this file, but not the same role). */ #include "ruby/ruby.h" /* for ID */ +#include "vm_core.h" /* for ID */ #define id_signo ruby_static_id_signo #define id_status ruby_static_id_status /* eval.c */ +struct rb_refinements_data { + VALUE refinement; + VALUE refinements; +}; + extern ID ruby_static_id_signo; extern ID ruby_static_id_status; VALUE rb_refinement_module_get_refined_class(VALUE module); void rb_class_modify_check(VALUE); NORETURN(VALUE rb_f_raise(int argc, VALUE *argv)); +VALUE rb_exception_setup(int argc, VALUE *argv); +void rb_refinement_setup(struct rb_refinements_data *data, VALUE module, VALUE klass); +void rb_vm_using_module(VALUE module); VALUE rb_top_main_class(const char *method); +VALUE rb_ec_ensure(rb_execution_context_t *ec, VALUE (*b_proc)(VALUE), VALUE data1, VALUE (*e_proc)(VALUE), VALUE data2); /* eval_error.c */ VALUE rb_get_backtrace(VALUE info); diff --git a/internal/fixnum.h b/internal/fixnum.h index 8c251adef1..b78e31460e 100644 --- a/internal/fixnum.h +++ b/internal/fixnum.h @@ -10,6 +10,7 @@ */ #include "ruby/internal/config.h" /* for HAVE_LONG_LONG */ #include <limits.h> /* for CHAR_BIT */ +#include "internal/bits.h" /* for MUL_OVERFLOW_FIXNUM_P */ #include "internal/compilers.h" /* for __has_builtin */ #include "ruby/internal/stdbool.h" /* for bool */ #include "ruby/intern.h" /* for rb_big_mul */ diff --git a/internal/gc.h b/internal/gc.h index e54a5dce9d..ee1f390e10 100644 --- a/internal/gc.h +++ b/internal/gc.h @@ -14,36 +14,128 @@ #include "internal/compilers.h" /* for __has_attribute */ #include "ruby/ruby.h" /* for rb_event_flag_t */ +#include "vm_core.h" /* for GET_EC() */ -struct rb_execution_context_struct; /* in vm_core.h */ -struct rb_objspace; /* in vm_core.h */ +#ifndef USE_MODULAR_GC +# define USE_MODULAR_GC 0 +#endif + +#if defined(__x86_64__) && !defined(_ILP32) && defined(__GNUC__) +#define SET_MACHINE_STACK_END(p) __asm__ __volatile__ ("movq\t%%rsp, %0" : "=r" (*(p))) +#elif defined(__i386) && defined(__GNUC__) +#define SET_MACHINE_STACK_END(p) __asm__ __volatile__ ("movl\t%%esp, %0" : "=r" (*(p))) +#elif (defined(__powerpc__) || defined(__powerpc64__)) && defined(__GNUC__) && !defined(_AIX) && !defined(__APPLE__) // Not Apple is NEEDED to unbreak ppc64 build on Darwin. Don't ask. +#define SET_MACHINE_STACK_END(p) __asm__ __volatile__ ("mr\t%0, %%r1" : "=r" (*(p))) +#elif (defined(__powerpc__) || defined(__powerpc64__)) && defined(__GNUC__) && defined(_AIX) +#define SET_MACHINE_STACK_END(p) __asm__ __volatile__ ("mr %0,1" : "=r" (*(p))) +#elif defined(__POWERPC__) && defined(__APPLE__) // Darwin ppc and ppc64 +#define SET_MACHINE_STACK_END(p) __asm__ volatile("mr %0, r1" : "=r" (*(p))) +#elif defined(__aarch64__) && defined(__GNUC__) +#define SET_MACHINE_STACK_END(p) __asm__ __volatile__ ("mov\t%0, sp" : "=r" (*(p))) +#else +NOINLINE(void rb_gc_set_stack_end(VALUE **stack_end_p)); +#define SET_MACHINE_STACK_END(p) rb_gc_set_stack_end(p) +#define USE_CONSERVATIVE_STACK_END +#endif + +/* for GC debug */ -#ifdef NEWOBJ_OF -# undef NEWOBJ_OF -# undef RB_NEWOBJ_OF -# undef RB_OBJ_WRITE +#ifndef RUBY_MARK_FREE_DEBUG +#define RUBY_MARK_FREE_DEBUG 0 #endif -#define RVALUE_SIZE (sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX])) +#if RUBY_MARK_FREE_DEBUG +extern int ruby_gc_debug_indent; + +static inline void +rb_gc_debug_indent(void) +{ + ruby_debug_printf("%*s", ruby_gc_debug_indent, ""); +} + +static inline void +rb_gc_debug_body(const char *mode, const char *msg, int st, void *ptr) +{ + if (st == 0) { + ruby_gc_debug_indent--; + } + rb_gc_debug_indent(); + ruby_debug_printf("%s: %s %s (%p)\n", mode, st ? "->" : "<-", msg, ptr); + + if (st) { + ruby_gc_debug_indent++; + } -#define RB_RVARGC_NEWOBJ_OF(var, T, c, f, s) \ - T *(var) = (T *)(((f) & FL_WB_PROTECTED) ? \ - rb_wb_protected_newobj_of((c), (f) & ~FL_WB_PROTECTED, s) : \ - rb_wb_unprotected_newobj_of((c), (f), s)) + fflush(stdout); +} -#define RB_RVARGC_EC_NEWOBJ_OF(ec, var, T, c, f, s) \ - T *(var) = (T *)(((f) & FL_WB_PROTECTED) ? \ - rb_ec_wb_protected_newobj_of((ec), (c), (f) & ~FL_WB_PROTECTED, s) : \ - rb_wb_unprotected_newobj_of((c), (f), s)) +#define RUBY_MARK_ENTER(msg) rb_gc_debug_body("mark", (msg), 1, ptr) +#define RUBY_MARK_LEAVE(msg) rb_gc_debug_body("mark", (msg), 0, ptr) +#define RUBY_FREE_ENTER(msg) rb_gc_debug_body("free", (msg), 1, ptr) +#define RUBY_FREE_LEAVE(msg) rb_gc_debug_body("free", (msg), 0, ptr) +#define RUBY_GC_INFO rb_gc_debug_indent(), ruby_debug_printf -/* optimized version of NEWOBJ() */ -#define RB_NEWOBJ_OF(var, T, c, f) RB_RVARGC_NEWOBJ_OF(var, T, c, f, RVALUE_SIZE) +#else +#define RUBY_MARK_ENTER(msg) +#define RUBY_MARK_LEAVE(msg) +#define RUBY_FREE_ENTER(msg) +#define RUBY_FREE_LEAVE(msg) +#define RUBY_GC_INFO if(0)printf +#endif -#define RB_EC_NEWOBJ_OF(ec, var, T, c, f) RB_RVARGC_EC_NEWOBJ_OF(ec, var, T, c, f, RVALUE_SIZE) +#define RUBY_FREE_UNLESS_NULL(ptr) if(ptr){ruby_xfree(ptr);(ptr)=NULL;} -#define NEWOBJ_OF(var, T, c, f) RB_NEWOBJ_OF((var), T, (c), (f)) -#define RVARGC_NEWOBJ_OF(var, T, c, f, s) RB_RVARGC_NEWOBJ_OF((var), T, (c), (f), (s)) -#define RB_OBJ_GC_FLAGS_MAX 6 /* used in ext/objspace */ +#if STACK_GROW_DIRECTION > 0 +# define STACK_UPPER(x, a, b) (a) +#elif STACK_GROW_DIRECTION < 0 +# define STACK_UPPER(x, a, b) (b) +#else +RUBY_EXTERN int ruby_stack_grow_direction; +int ruby_get_stack_grow_direction(volatile VALUE *addr); +# define stack_growup_p(x) ( \ + (ruby_stack_grow_direction ? \ + ruby_stack_grow_direction : \ + ruby_get_stack_grow_direction(x)) > 0) +# define STACK_UPPER(x, a, b) (stack_growup_p(x) ? (a) : (b)) +#endif + +/* + STACK_GROW_DIR_DETECTION is used with STACK_DIR_UPPER. + + On most normal systems, stacks grow from high address to lower address. In + this case, STACK_DIR_UPPER(a, b) will return (b), but on exotic systems where + the stack grows UP (from low address to high address), it will return (a). +*/ + +#if STACK_GROW_DIRECTION +#define STACK_GROW_DIR_DETECTION +#define STACK_DIR_UPPER(a,b) STACK_UPPER(0, (a), (b)) +#else +#define STACK_GROW_DIR_DETECTION VALUE stack_grow_dir_detection +#define STACK_DIR_UPPER(a,b) STACK_UPPER(&stack_grow_dir_detection, (a), (b)) +#endif +#define IS_STACK_DIR_UPPER() STACK_DIR_UPPER(1,0) + +const char *rb_obj_info(VALUE obj); +const char *rb_raw_obj_info(char *const buff, const size_t buff_size, VALUE obj); + +struct rb_execution_context_struct; /* in vm_core.h */ +struct rb_objspace; /* in vm_core.h */ + +#define NEWOBJ_OF_WITH_SHAPE(var, T, c, f, shape_id, s, ec) \ + T *(var) = (T *)(((f) & FL_WB_PROTECTED) ? \ + rb_wb_protected_newobj_of((ec ? ec : GET_EC()), (c), (f) & ~FL_WB_PROTECTED, shape_id, s) : \ + rb_wb_unprotected_newobj_of((c), (f), shape_id, s)) + +#define NEWOBJ_OF(var, T, c, f, s, ec) NEWOBJ_OF_WITH_SHAPE(var, T, c, f, 0 /* ROOT_SHAPE_ID */, s, ec) + +#ifndef RB_GC_OBJECT_METADATA_ENTRY_DEFINED +# define RB_GC_OBJECT_METADATA_ENTRY_DEFINED +struct rb_gc_object_metadata_entry { + ID name; + VALUE val; +}; +#endif #ifndef USE_UNALIGNED_MEMBER_ACCESS # define UNALIGNED_MEMBER_ACCESS(expr) (expr) @@ -60,49 +152,49 @@ struct rb_objspace; /* in vm_core.h */ COMPILER_WARNING_POP; \ unaligned_member_access_result; \ }) -#endif -#define UNALIGNED_MEMBER_PTR(ptr, mem) UNALIGNED_MEMBER_ACCESS(&(ptr)->mem) -#define RB_OBJ_WRITE(a, slot, b) \ - rb_obj_write((VALUE)(a), UNALIGNED_MEMBER_ACCESS((VALUE *)(slot)), \ - (VALUE)(b), __FILE__, __LINE__) +# define UNALIGNED_MEMBER_PTR(ptr, mem) __extension__({ \ + COMPILER_WARNING_PUSH; \ + COMPILER_WARNING_IGNORED(-Waddress-of-packed-member); \ + const volatile void *unaligned_member_ptr_result = &(ptr)->mem; \ + COMPILER_WARNING_POP; \ + (__typeof__((ptr)->mem) *)unaligned_member_ptr_result; \ +}) +#endif -// We use SIZE_POOL_COUNT number of shape IDs for transitions out of different size pools -// The next available shapd ID will be the SPECIAL_CONST_SHAPE_ID -#if USE_RVARGC && (SIZEOF_UINT64_T == SIZEOF_VALUE) -# define SIZE_POOL_COUNT 5 -#else -# define SIZE_POOL_COUNT 1 +#ifndef UNALIGNED_MEMBER_PTR +# define UNALIGNED_MEMBER_PTR(ptr, mem) UNALIGNED_MEMBER_ACCESS(&(ptr)->mem) #endif -#define RCLASS_EXT_EMBEDDED (SIZE_POOL_COUNT > 1) +#define RB_OBJ_WRITE_UNALIGNED(old, slot, young) do { \ + VALUE *_slot = UNALIGNED_MEMBER_ACCESS(slot); \ + RB_OBJ_WRITE(old, _slot, young); \ +} while (0) -typedef struct ractor_newobj_size_pool_cache { - struct RVALUE *freelist; - struct heap_page *using_page; -} rb_ractor_newobj_size_pool_cache_t; +/* Used in places that could malloc during, which can cause the GC to run. We + * need to temporarily disable the GC to allow the malloc to happen. + * Allocating memory during GC is a bad idea, so use this only when absolutely + * necessary. */ +#define DURING_GC_COULD_MALLOC_REGION_START() \ + assert(rb_during_gc()); \ + VALUE _already_disabled = rb_gc_disable_no_rest() -typedef struct ractor_newobj_cache { - size_t incremental_mark_step_allocated_slots; - rb_ractor_newobj_size_pool_cache_t size_pool_caches[SIZE_POOL_COUNT]; -} rb_ractor_newobj_cache_t; +#define DURING_GC_COULD_MALLOC_REGION_END() \ + if (_already_disabled == Qfalse) rb_gc_enable() /* gc.c */ -extern VALUE *ruby_initial_gc_stress_ptr; -extern int ruby_disable_gc; RUBY_ATTR_MALLOC void *ruby_mimmalloc(size_t size); +RUBY_ATTR_MALLOC void *ruby_mimcalloc(size_t num, size_t size); void ruby_mimfree(void *ptr); +void rb_gc_prepare_heap(void); void rb_objspace_set_event_hook(const rb_event_flag_t event); -VALUE rb_objspace_gc_enable(struct rb_objspace *); -VALUE rb_objspace_gc_disable(struct rb_objspace *); +VALUE rb_objspace_gc_enable(void *objspace); +VALUE rb_objspace_gc_disable(void *objspace); void ruby_gc_set_params(void); -void rb_copy_wb_protected_attribute(VALUE dest, VALUE obj); -#if __has_attribute(alloc_align) -__attribute__((__alloc_align__(1))) -#endif -RUBY_ATTR_MALLOC void *rb_aligned_malloc(size_t, size_t) RUBY_ATTR_ALLOC_SIZE((2)); +void rb_gc_copy_attributes(VALUE dest, VALUE obj); size_t rb_size_mul_or_raise(size_t, size_t, VALUE); /* used in compile.c */ size_t rb_size_mul_add_or_raise(size_t, size_t, size_t, VALUE); /* used in iseq.h */ +size_t rb_malloc_grow_capa(size_t current_capacity, size_t type_size); RUBY_ATTR_MALLOC void *rb_xmalloc_mul_add(size_t, size_t, size_t); RUBY_ATTR_MALLOC void *rb_xcalloc_mul_add(size_t, size_t, size_t); void *rb_xrealloc_mul_add(const void *, size_t, size_t, size_t); @@ -111,33 +203,94 @@ RUBY_ATTR_MALLOC void *rb_xcalloc_mul_add_mul(size_t, size_t, size_t, size_t); static inline void *ruby_sized_xrealloc_inlined(void *ptr, size_t new_size, size_t old_size) RUBY_ATTR_RETURNS_NONNULL RUBY_ATTR_ALLOC_SIZE((2)); static inline void *ruby_sized_xrealloc2_inlined(void *ptr, size_t new_count, size_t elemsiz, size_t old_count) RUBY_ATTR_RETURNS_NONNULL RUBY_ATTR_ALLOC_SIZE((2, 3)); static inline void ruby_sized_xfree_inlined(void *ptr, size_t size); -VALUE rb_class_allocate_instance(VALUE klass); -void rb_gc_ractor_newobj_cache_clear(rb_ractor_newobj_cache_t *newobj_cache); -size_t rb_gc_obj_slot_size(VALUE obj); +void rb_gc_obj_id_moved(VALUE obj); +void rb_gc_register_pinning_obj(VALUE obj); + +void *rb_gc_ractor_cache_alloc(rb_ractor_t *ractor); +void rb_gc_ractor_cache_free(void *cache); + bool rb_gc_size_allocatable_p(size_t size); -int rb_objspace_garbage_object_p(VALUE obj); +size_t *rb_gc_heap_sizes(void); +size_t rb_gc_heap_id_for_size(size_t size); + +void rb_gc_mark_and_move(VALUE *ptr); + +void rb_gc_declare_weak_references(VALUE obj); +bool rb_gc_handle_weak_references_alive_p(VALUE obj); + +void rb_gc_ref_update_table_values_only(st_table *tbl); + +void rb_gc_initial_stress_set(VALUE flag); + +void rb_gc_before_fork(void); +void rb_gc_after_fork(rb_pid_t pid); + +#define rb_gc_mark_and_move_ptr(ptr) do { \ + VALUE _obj = (VALUE)*(ptr); \ + rb_gc_mark_and_move(&_obj); \ + if (_obj != (VALUE)*(ptr)) *(ptr) = (void *)_obj; \ +} while (0) RUBY_SYMBOL_EXPORT_BEGIN +/* exports for objspace module */ +void rb_objspace_reachable_objects_from(VALUE obj, void (func)(VALUE, void *), void *data); +void rb_objspace_reachable_objects_from_root(void (func)(const char *category, VALUE, void *), void *data); +int rb_objspace_internal_object_p(VALUE obj); +int rb_objspace_garbage_object_p(VALUE obj); +bool rb_gc_pointer_to_heap_p(VALUE obj); + +void rb_objspace_each_objects( + int (*callback)(void *start, void *end, size_t stride, void *data), + void *data); + +size_t rb_gc_obj_slot_size(VALUE obj); + +VALUE rb_gc_disable_no_rest(void); + +#define RB_GC_MAX_NAME_LEN 20 + /* gc.c (export) */ const char *rb_objspace_data_type_name(VALUE obj); -VALUE rb_wb_protected_newobj_of(VALUE, VALUE, size_t); -VALUE rb_wb_unprotected_newobj_of(VALUE, VALUE, size_t); -VALUE rb_ec_wb_protected_newobj_of(struct rb_execution_context_struct *ec, VALUE klass, VALUE flags, size_t); +VALUE rb_wb_protected_newobj_of(struct rb_execution_context_struct *, VALUE, VALUE, uint32_t /* shape_id_t */, size_t); +VALUE rb_wb_unprotected_newobj_of(VALUE, VALUE, uint32_t /* shape_id_t */, size_t); size_t rb_obj_memsize_of(VALUE); -void rb_gc_verify_internal_consistency(void); -size_t rb_obj_gc_flags(VALUE, ID[], size_t); +struct rb_gc_object_metadata_entry *rb_gc_object_metadata(VALUE obj); void rb_gc_mark_values(long n, const VALUE *values); void rb_gc_mark_vm_stack_values(long n, const VALUE *values); +void rb_gc_update_values(long n, VALUE *values); void *ruby_sized_xrealloc(void *ptr, size_t new_size, size_t old_size) RUBY_ATTR_RETURNS_NONNULL RUBY_ATTR_ALLOC_SIZE((2)); void *ruby_sized_xrealloc2(void *ptr, size_t new_count, size_t element_size, size_t old_count) RUBY_ATTR_RETURNS_NONNULL RUBY_ATTR_ALLOC_SIZE((2, 3)); void ruby_sized_xfree(void *x, size_t size); + +const char *rb_gc_active_gc_name(void); +int rb_gc_modular_gc_loaded_p(void); + RUBY_SYMBOL_EXPORT_END -MJIT_SYMBOL_EXPORT_BEGIN +static inline VALUE +rb_obj_atomic_write( + VALUE a, VALUE *slot, VALUE b, + RBIMPL_ATTR_MAYBE_UNUSED() + const char *filename, + RBIMPL_ATTR_MAYBE_UNUSED() + int line) +{ +#ifdef RGENGC_LOGGING_WRITE + RGENGC_LOGGING_WRITE(a, slot, b, filename, line); +#endif + + RUBY_ATOMIC_VALUE_SET(*slot, b); + + rb_obj_written(a, RUBY_Qundef /* ignore `oldv' now */, b, filename, line); + return a; +} +#define RB_OBJ_ATOMIC_WRITE(old, slot, young) \ + RBIMPL_CAST(rb_obj_atomic_write((VALUE)(old), (VALUE *)(slot), (VALUE)(young), __FILE__, __LINE__)) + int rb_ec_stack_check(struct rb_execution_context_struct *ec); void rb_gc_writebarrier_remember(VALUE obj); const char *rb_obj_info(VALUE obj); -MJIT_SYMBOL_EXPORT_END +void ruby_annotate_mmap(const void *addr, unsigned long size, const char *name); #if defined(HAVE_MALLOC_USABLE_SIZE) || defined(HAVE_MALLOC_SIZE) || defined(_WIN32) @@ -161,6 +314,12 @@ ruby_sized_xfree_inlined(void *ptr, size_t size) # define SIZED_REALLOC_N(x, y, z, w) REALLOC_N(x, y, z) +static inline void * +ruby_sized_realloc_n(void *ptr, size_t new_count, size_t element_size, size_t old_count) +{ + return ruby_xrealloc2(ptr, new_count, element_size); +} + #else static inline void * @@ -184,9 +343,19 @@ ruby_sized_xfree_inlined(void *ptr, size_t size) # define SIZED_REALLOC_N(v, T, m, n) \ ((v) = (T *)ruby_sized_xrealloc2((void *)(v), (m), sizeof(T), (n))) +static inline void * +ruby_sized_realloc_n(void *ptr, size_t new_count, size_t element_size, size_t old_count) +{ + return ruby_sized_xrealloc2(ptr, new_count, element_size, old_count); +} + #endif /* HAVE_MALLOC_USABLE_SIZE */ #define ruby_sized_xrealloc ruby_sized_xrealloc_inlined #define ruby_sized_xrealloc2 ruby_sized_xrealloc2_inlined #define ruby_sized_xfree ruby_sized_xfree_inlined + +void rb_gc_verify_shareable(VALUE); +bool rb_gc_checking_shareable(void); + #endif /* INTERNAL_GC_H */ diff --git a/internal/hash.h b/internal/hash.h index 657e5eff3c..03cd830506 100644 --- a/internal/hash.h +++ b/internal/hash.h @@ -28,10 +28,6 @@ enum ruby_rhash_flags { RHASH_AR_TABLE_BOUND_MASK = (FL_USER8|FL_USER9|FL_USER10|FL_USER11), /* FL 8..11 */ RHASH_AR_TABLE_BOUND_SHIFT = (FL_USHIFT+8), -#if USE_TRANSIENT_HEAP - RHASH_TRANSIENT_FLAG = FL_USER12, /* FL 12 */ -#endif - // we can not put it in "enum" because it can exceed "int" range. #define RHASH_LEV_MASK (FL_USER13 | FL_USER14 | FL_USER15 | /* FL 13..19 */ \ FL_USER16 | FL_USER17 | FL_USER18 | FL_USER19) @@ -40,17 +36,23 @@ enum ruby_rhash_flags { RHASH_LEV_MAX = 127, /* 7 bits */ }; -struct RHash { - struct RBasic basic; - union { - st_table *st; - struct ar_table_struct *ar; /* possibly 0 */ - } as; - const VALUE ifnone; +typedef struct ar_table_pair_struct { + VALUE key; + VALUE val; +} ar_table_pair; + +typedef struct ar_table_struct { union { ar_hint_t ary[RHASH_AR_TABLE_MAX_SIZE]; VALUE word; } ar_hint; + /* 64bit CPU: 8B * 2 * 8 = 128B */ + ar_table_pair pairs[RHASH_AR_TABLE_MAX_SIZE]; +} ar_table; + +struct RHash { + struct RBasic basic; + const VALUE ifnone; }; #define RHASH(obj) ((struct RHash *)(obj)) @@ -70,9 +72,12 @@ struct RHash { /* hash.c */ void rb_hash_st_table_set(VALUE hash, st_table *st); VALUE rb_hash_default_value(VALUE hash, VALUE key); +VALUE rb_hash_set_default(VALUE hash, VALUE ifnone); VALUE rb_hash_set_default_proc(VALUE hash, VALUE proc); long rb_dbl_long_hash(double d); st_table *rb_init_identtable(void); +st_index_t rb_any_hash(VALUE a); +int rb_any_cmp(VALUE a, VALUE b); VALUE rb_to_hash_type(VALUE obj); VALUE rb_hash_key_str(VALUE); VALUE rb_hash_values(VALUE hash); @@ -82,8 +87,10 @@ VALUE rb_hash_set_pair(VALUE hash, VALUE pair); int rb_hash_stlike_delete(VALUE hash, st_data_t *pkey, st_data_t *pval); int rb_hash_stlike_foreach_with_replace(VALUE hash, st_foreach_check_callback_func *func, st_update_callback_func *replace, st_data_t arg); int rb_hash_stlike_update(VALUE hash, st_data_t key, st_update_callback_func *func, st_data_t arg); -extern st_table *rb_hash_st_table(VALUE hash); +bool rb_hash_default_unredefined(VALUE hash); VALUE rb_ident_hash_new_with_size(st_index_t size); +void rb_hash_free(VALUE hash); +RUBY_EXTERN VALUE rb_cHash_empty_frozen; static inline unsigned RHASH_AR_TABLE_SIZE_RAW(VALUE h); static inline VALUE RHASH_IFNONE(VALUE h); @@ -95,9 +102,6 @@ static inline struct ar_table_struct *RHASH_AR_TABLE(VALUE h); static inline st_table *RHASH_ST_TABLE(VALUE h); static inline size_t RHASH_ST_SIZE(VALUE h); static inline void RHASH_ST_CLEAR(VALUE h); -static inline bool RHASH_TRANSIENT_P(VALUE h); -static inline void RHASH_SET_TRANSIENT_FLAG(VALUE h); -static inline void RHASH_UNSET_TRANSIENT_FLAG(VALUE h); RUBY_SYMBOL_EXPORT_BEGIN /* hash.c (export) */ @@ -106,7 +110,6 @@ VALUE rb_ident_hash_new(void); int rb_hash_stlike_foreach(VALUE hash, st_foreach_callback_func *func, st_data_t arg); RUBY_SYMBOL_EXPORT_END -MJIT_SYMBOL_EXPORT_BEGIN VALUE rb_hash_new_with_size(st_index_t size); VALUE rb_hash_resurrect(VALUE hash); int rb_hash_stlike_lookup(VALUE hash, st_data_t key, st_data_t *pval); @@ -116,54 +119,29 @@ VALUE rb_hash_compare_by_id_p(VALUE hash); st_table *rb_hash_tbl_raw(VALUE hash, const char *file, int line); #define RHASH_TBL_RAW(h) rb_hash_tbl_raw(h, __FILE__, __LINE__) -MJIT_SYMBOL_EXPORT_END VALUE rb_hash_compare_by_id(VALUE hash); -#if 0 /* for debug */ - -static inline bool -RHASH_AR_TABLE_P(VALUE h) -{ - extern int rb_hash_ar_table_p(VALUE hash); - return rb_hash_ar_table_p(h) -} - -static inline struct ar_table_struct * -RHASH_AR_TABLE(VALUE h) -{ - extern struct ar_table_struct *rb_hash_ar_table(VALUE hash); - return rb_hash_ar_table(h) -} - -static inline st_table * -RHASH_ST_TABLE(VALUE h) -{ - return rb_hash_st_table(h) -} - -#else - static inline bool RHASH_AR_TABLE_P(VALUE h) { return ! FL_TEST_RAW(h, RHASH_ST_TABLE_FLAG); } +RBIMPL_ATTR_RETURNS_NONNULL() static inline struct ar_table_struct * RHASH_AR_TABLE(VALUE h) { - return RHASH(h)->as.ar; + return (struct ar_table_struct *)((uintptr_t)h + sizeof(struct RHash)); } +RBIMPL_ATTR_RETURNS_NONNULL() static inline st_table * RHASH_ST_TABLE(VALUE h) { - return RHASH(h)->as.st; + return (st_table *)((uintptr_t)h + sizeof(struct RHash)); } -#endif - static inline VALUE RHASH_IFNONE(VALUE h) { @@ -202,8 +180,7 @@ RHASH_ST_SIZE(VALUE h) static inline void RHASH_ST_CLEAR(VALUE h) { - FL_UNSET_RAW(h, RHASH_ST_TABLE_FLAG); - RHASH(h)->as.ar = NULL; + memset(RHASH_ST_TABLE(h), 0, sizeof(st_table)); } static inline unsigned @@ -214,30 +191,4 @@ RHASH_AR_TABLE_SIZE_RAW(VALUE h) return (unsigned)ret; } -static inline bool -RHASH_TRANSIENT_P(VALUE h) -{ -#if USE_TRANSIENT_HEAP - return FL_TEST_RAW(h, RHASH_TRANSIENT_FLAG); -#else - return false; -#endif -} - -static inline void -RHASH_SET_TRANSIENT_FLAG(VALUE h) -{ -#if USE_TRANSIENT_HEAP - FL_SET_RAW(h, RHASH_TRANSIENT_FLAG); -#endif -} - -static inline void -RHASH_UNSET_TRANSIENT_FLAG(VALUE h) -{ -#if USE_TRANSIENT_HEAP - FL_UNSET_RAW(h, RHASH_TRANSIENT_FLAG); -#endif -} - #endif /* INTERNAL_HASH_H */ diff --git a/internal/imemo.h b/internal/imemo.h index 91b524e0a6..31cc0be35a 100644 --- a/internal/imemo.h +++ b/internal/imemo.h @@ -10,15 +10,11 @@ */ #include "ruby/internal/config.h" #include <stddef.h> /* for size_t */ +#include "id_table.h" #include "internal/array.h" /* for rb_ary_hidden_new_fill */ -#include "internal/gc.h" /* for RB_OBJ_WRITE */ #include "ruby/internal/stdbool.h" /* for bool */ #include "ruby/ruby.h" /* for rb_block_call_func_t */ -#ifndef IMEMO_DEBUG -# define IMEMO_DEBUG 0 -#endif - #define IMEMO_MASK 0x0f /* FL_USER0 to FL_USER3 is for type */ @@ -29,6 +25,7 @@ #define IMEMO_FL_USER3 FL_USER7 #define IMEMO_FL_USER4 FL_USER8 #define IMEMO_FL_USER5 FL_USER9 +#define IMEMO_FL_USER6 FL_USER10 enum imemo_type { imemo_env = 0, @@ -40,11 +37,10 @@ enum imemo_type { imemo_ment = 6, imemo_iseq = 7, imemo_tmpbuf = 8, - imemo_ast = 9, - imemo_parser_strterm = 10, - imemo_callinfo = 11, - imemo_callcache = 12, - imemo_constcache = 13, + imemo_callinfo = 10, + imemo_callcache = 11, + imemo_constcache = 12, + imemo_fields = 13, }; /* CREF (Class REFerence) is defined in method.h */ @@ -80,20 +76,24 @@ struct vm_ifunc_argc { #endif }; -/*! IFUNC (Internal FUNCtion) */ +/*! IFUNC (Internal FUNCtion) + * + * Bookkeeping for converting a C function and some closed-over data into a + * block passable to methods. Like Ruby Proc, but not directly accessible at + * Ruby level since this is an imemo. See rb_block_call() and friends. + */ struct vm_ifunc { VALUE flags; - VALUE reserved; + VALUE *svar_lep; rb_block_call_func_t func; const void *data; struct vm_ifunc_argc argc; }; +#define IFUNC_YIELD_OPTIMIZABLE IMEMO_FL_USER0 struct rb_imemo_tmpbuf_struct { VALUE flags; - VALUE reserved; VALUE *ptr; /* malloc'ed buffer */ - struct rb_imemo_tmpbuf_struct *next; /* next imemo */ size_t cnt; /* buffer size in VALUE */ }; @@ -114,11 +114,13 @@ struct MEMO { } u3; }; +#define IMEMO_NEW(T, type, v0) ((T *)rb_imemo_new((type), (v0), sizeof(T), false)) +#define SHAREABLE_IMEMO_NEW(T, type, v0) ((T *)rb_imemo_new((type), (v0), sizeof(T), true)) + /* ment is in method.h */ #define THROW_DATA_P(err) imemo_throw_data_p((VALUE)err) #define MEMO_CAST(m) ((struct MEMO *)(m)) -#define MEMO_NEW(a, b, c) ((struct MEMO *)rb_imemo_new(imemo_memo, (VALUE)(a), (VALUE)(b), (VALUE)(c), 0)) #define MEMO_FOR(type, value) ((type *)RARRAY_PTR(value)) #define NEW_MEMO_FOR(type, value) \ ((value) = rb_ary_hidden_new_fill(type_roomof(type, VALUE)), MEMO_FOR(type, value)) @@ -127,28 +129,27 @@ struct MEMO { rb_ary_set_len((value), offsetof(type, member) / sizeof(VALUE)), \ MEMO_FOR(type, value)) +#ifndef RUBY_RUBYPARSER_H typedef struct rb_imemo_tmpbuf_struct rb_imemo_tmpbuf_t; -rb_imemo_tmpbuf_t *rb_imemo_tmpbuf_parser_heap(void *buf, rb_imemo_tmpbuf_t *old_heap, size_t cnt); +#endif +VALUE rb_imemo_new(enum imemo_type type, VALUE v0, size_t size, bool is_shareable); +VALUE rb_imemo_tmpbuf_new(void); +struct MEMO *rb_imemo_memo_new(VALUE a, VALUE b, VALUE c); struct vm_ifunc *rb_vm_ifunc_new(rb_block_call_func_t func, const void *data, int min_argc, int max_argc); -void rb_strterm_mark(VALUE obj); static inline enum imemo_type imemo_type(VALUE imemo); static inline int imemo_type_p(VALUE imemo, enum imemo_type imemo_type); static inline bool imemo_throw_data_p(VALUE imemo); static inline struct vm_ifunc *rb_vm_ifunc_proc_new(rb_block_call_func_t func, const void *data); -static inline VALUE rb_imemo_tmpbuf_auto_free_pointer(void); static inline void *RB_IMEMO_TMPBUF_PTR(VALUE v); static inline void *rb_imemo_tmpbuf_set_ptr(VALUE v, void *ptr); -static inline VALUE rb_imemo_tmpbuf_auto_free_pointer_new_from_an_RString(VALUE str); static inline void MEMO_V1_SET(struct MEMO *m, VALUE v); static inline void MEMO_V2_SET(struct MEMO *m, VALUE v); +size_t rb_imemo_memsize(VALUE obj); +void rb_imemo_mark_and_move(VALUE obj, bool reference_updating); +void rb_imemo_free(VALUE obj); + RUBY_SYMBOL_EXPORT_BEGIN -#if IMEMO_DEBUG -VALUE rb_imemo_new_debug(enum imemo_type type, VALUE v1, VALUE v2, VALUE v3, VALUE v0, const char *file, int line); -#define rb_imemo_new(type, v1, v2, v3, v0) rb_imemo_new_debug(type, v1, v2, v3, v0, __FILE__, __LINE__) -#else -VALUE rb_imemo_new(enum imemo_type type, VALUE v1, VALUE v2, VALUE v3, VALUE v0); -#endif const char *rb_imemo_name(enum imemo_type type); RUBY_SYMBOL_EXPORT_END @@ -173,7 +174,7 @@ imemo_type_p(VALUE imemo, enum imemo_type imemo_type) } } -#define IMEMO_TYPE_P(v, t) imemo_type_p((VALUE)v, t) +#define IMEMO_TYPE_P(v, t) imemo_type_p((VALUE)(v), t) static inline bool imemo_throw_data_p(VALUE imemo) @@ -187,12 +188,6 @@ rb_vm_ifunc_proc_new(rb_block_call_func_t func, const void *data) return rb_vm_ifunc_new(func, data, 0, UNLIMITED_ARGUMENTS); } -static inline VALUE -rb_imemo_tmpbuf_auto_free_pointer(void) -{ - return rb_imemo_new(imemo_tmpbuf, 0, 0, 0, 0); -} - static inline void * RB_IMEMO_TMPBUF_PTR(VALUE v) { @@ -207,7 +202,7 @@ rb_imemo_tmpbuf_set_ptr(VALUE v, void *ptr) } static inline VALUE -rb_imemo_tmpbuf_auto_free_pointer_new_from_an_RString(VALUE str) +rb_imemo_tmpbuf_new_from_an_RString(VALUE str) { const void *src; VALUE imemo; @@ -215,9 +210,9 @@ rb_imemo_tmpbuf_auto_free_pointer_new_from_an_RString(VALUE str) void *dst; size_t len; - SafeStringValue(str); + StringValue(str); /* create tmpbuf to keep the pointer before xmalloc */ - imemo = rb_imemo_tmpbuf_auto_free_pointer(); + imemo = rb_imemo_tmpbuf_new(); tmpbuf = (rb_imemo_tmpbuf_t *)imemo; len = RSTRING_LEN(str); src = RSTRING_PTR(str); @@ -239,4 +234,79 @@ MEMO_V2_SET(struct MEMO *m, VALUE v) RB_OBJ_WRITE(m, &m->v2, v); } +struct rb_fields { + struct RBasic basic; + union { + struct { + VALUE fields[1]; + } embed; + struct { + VALUE *ptr; + } external; + struct { + // Note: the st_table could be embedded, but complex T_CLASS should be rare to + // non-existent, so not really worth the trouble. + st_table *table; + } complex; + } as; +}; + +// IMEMO/fields and T_OBJECT have exactly the same layout. +// This is useful for JIT and common codepaths. +#define OBJ_FIELD_HEAP ROBJECT_HEAP +STATIC_ASSERT(imemo_fields_flags, OBJ_FIELD_HEAP == IMEMO_FL_USER0); +STATIC_ASSERT(imemo_fields_embed_offset, offsetof(struct RObject, as.ary) == offsetof(struct rb_fields, as.embed.fields)); +STATIC_ASSERT(imemo_fields_external_offset, offsetof(struct RObject, as.heap.fields) == offsetof(struct rb_fields, as.external.ptr)); +STATIC_ASSERT(imemo_fields_complex_offset, offsetof(struct RObject, as.heap.fields) == offsetof(struct rb_fields, as.complex.table)); + +#define IMEMO_OBJ_FIELDS(fields) ((struct rb_fields *)fields) + +VALUE rb_imemo_fields_new(VALUE owner, size_t capa, bool shareable); +VALUE rb_imemo_fields_new_complex(VALUE owner, size_t capa, bool shareable); +VALUE rb_imemo_fields_new_complex_tbl(VALUE owner, st_table *tbl, bool shareable); +VALUE rb_imemo_fields_clone(VALUE fields_obj); +void rb_imemo_fields_clear(VALUE fields_obj); + +static inline VALUE +rb_imemo_fields_owner(VALUE fields_obj) +{ + RUBY_ASSERT(IMEMO_TYPE_P(fields_obj, imemo_fields)); + + return CLASS_OF(fields_obj); +} + +static inline VALUE * +rb_imemo_fields_ptr(VALUE fields_obj) +{ + if (!fields_obj) { + return NULL; + } + + RUBY_ASSERT(IMEMO_TYPE_P(fields_obj, imemo_fields) || RB_TYPE_P(fields_obj, T_OBJECT)); + + if (UNLIKELY(FL_TEST_RAW(fields_obj, OBJ_FIELD_HEAP))) { + return IMEMO_OBJ_FIELDS(fields_obj)->as.external.ptr; + } + else { + return IMEMO_OBJ_FIELDS(fields_obj)->as.embed.fields; + } +} + +static inline st_table * +rb_imemo_fields_complex_tbl(VALUE fields_obj) +{ + if (!fields_obj) { + return NULL; + } + + RUBY_ASSERT(IMEMO_TYPE_P(fields_obj, imemo_fields) || RB_TYPE_P(fields_obj, T_OBJECT)); + RUBY_ASSERT(FL_TEST_RAW(fields_obj, OBJ_FIELD_HEAP)); + + // Some codepaths unconditionally access the fields_ptr, and assume it can be used as st_table if the + // shape is too_complex. + RUBY_ASSERT((st_table *)rb_imemo_fields_ptr(fields_obj) == IMEMO_OBJ_FIELDS(fields_obj)->as.complex.table); + + return IMEMO_OBJ_FIELDS(fields_obj)->as.complex.table; +} + #endif /* INTERNAL_IMEMO_H */ diff --git a/internal/inits.h b/internal/inits.h index 03e180f77b..dee818285c 100644 --- a/internal/inits.h +++ b/internal/inits.h @@ -9,6 +9,10 @@ * @brief Internal header aggregating init functions. */ +/* box.c */ +void Init_enable_box(void); +void Init_root_box(void); + /* class.c */ void Init_class_hierarchy(void); @@ -19,9 +23,6 @@ void Init_ext(void); /* file.c */ void Init_File(void); -/* gc.c */ -void Init_heap(void); - /* localeinit.c */ int Init_enc_set_filesystem_encoding(void); diff --git a/internal/io.h b/internal/io.h index b5f15499d7..b81774e0a7 100644 --- a/internal/io.h +++ b/internal/io.h @@ -9,7 +9,132 @@ * @brief Internal header for IO. */ #include "ruby/ruby.h" /* for VALUE */ + +#define HAVE_RB_IO_T +struct rb_io; + #include "ruby/io.h" /* for rb_io_t */ +#include "ccan/list/list.h" +#include "serial.h" + +#define IO_WITHOUT_GVL(func, arg) rb_nogvl(func, arg, RUBY_UBF_IO, 0, RB_NOGVL_OFFLOAD_SAFE) +#define IO_WITHOUT_GVL_INT(func, arg) (int)(VALUE)IO_WITHOUT_GVL(func, arg) + +// Represents an in-flight blocking operation: +struct rb_io_blocking_operation { + // The linked list data structure. + struct ccan_list_node list; + + // The execution context of the blocking operation. + struct rb_execution_context_struct *ec; +}; + +/** Ruby's IO, metadata and buffers. */ +struct rb_io { + + /** The IO's Ruby level counterpart. */ + VALUE self; + + /** stdio ptr for read/write, if available. */ + FILE *stdio_file; + + /** file descriptor. */ + int fd; + + /** mode flags: FMODE_XXXs */ + enum rb_io_mode mode; + + /** child's pid (for pipes) */ + rb_pid_t pid; + + /** number of lines read */ + int lineno; + + /** pathname for file */ + VALUE pathv; + + /** finalize proc */ + void (*finalize)(struct rb_io*,int); + + /** Write buffer. */ + rb_io_buffer_t wbuf; + + /** + * (Byte) read buffer. Note also that there is a field called + * ::rb_io_t::cbuf, which also concerns read IO. + */ + rb_io_buffer_t rbuf; + + /** + * Duplex IO object, if set. + * + * @see rb_io_set_write_io() + */ + VALUE tied_io_for_writing; + + struct rb_io_encoding encs; /**< Decomposed encoding flags. */ + + /** Encoding converter used when reading from this IO. */ + rb_econv_t *readconv; + + /** + * rb_io_ungetc() destination. This buffer is read before checking + * ::rb_io_t::rbuf + */ + rb_io_buffer_t cbuf; + + /** Encoding converter used when writing to this IO. */ + rb_econv_t *writeconv; + + /** + * This is, when set, an instance of ::rb_cString which holds the "common" + * encoding. Write conversion can convert strings twice... In case + * conversion from encoding X to encoding Y does not exist, Ruby finds an + * encoding Z that bridges the two, so that X to Z to Y conversion happens. + */ + VALUE writeconv_asciicompat; + + /** Whether ::rb_io_t::writeconv is already set up. */ + int writeconv_initialized; + + /** + * Value of ::rb_io_t::rb_io_enc_t::ecflags stored right before + * initialising ::rb_io_t::writeconv. + */ + int writeconv_pre_ecflags; + + /** + * Value of ::rb_io_t::rb_io_enc_t::ecopts stored right before initialising + * ::rb_io_t::writeconv. + */ + VALUE writeconv_pre_ecopts; + + /** + * This is a Ruby level mutex. It avoids multiple threads to write to an + * IO at once; helps for instance rb_io_puts() to ensure newlines right + * next to its arguments. + * + * This of course doesn't help inter-process IO interleaves, though. + */ + VALUE write_lock; + + /** + * The timeout associated with this IO when performing blocking operations. + */ + VALUE timeout; + + /** + * Threads that are performing a blocking operation without the GVL using + * this IO. On calling IO#close, these threads will be interrupted so that + * the operation can be cancelled. + */ + struct ccan_list_head blocking_operations; + struct rb_execution_context_struct *closing_ec; + VALUE wakeup_mutex; + + // The fork generation of the blocking operations list. + rb_serial_t fork_generation; +}; /* io.c */ void ruby_set_inplace_mode(const char *); @@ -17,22 +142,22 @@ void rb_stdio_set_default_encoding(void); VALUE rb_io_flush_raw(VALUE, int); size_t rb_io_memsize(const rb_io_t *); int rb_stderr_tty_p(void); -void rb_io_fptr_finalize_internal(void *ptr); -#ifdef rb_io_fptr_finalize -# undef rb_io_fptr_finalize -#endif -#define rb_io_fptr_finalize rb_io_fptr_finalize_internal VALUE rb_io_popen(VALUE pname, VALUE pmode, VALUE env, VALUE opt); VALUE rb_io_prep_stdin(void); VALUE rb_io_prep_stdout(void); VALUE rb_io_prep_stderr(void); +int rb_io_notify_close(struct rb_io *fptr); + RUBY_SYMBOL_EXPORT_BEGIN /* io.c (export) */ void rb_maygvl_fd_fix_cloexec(int fd); int rb_gc_for_fd(int err); void rb_write_error_str(VALUE mesg); + +VALUE rb_io_blocking_region_wait(struct rb_io *io, rb_blocking_function_t *function, void *argument, enum rb_io_event events); +VALUE rb_io_blocking_region(struct rb_io *io, rb_blocking_function_t *function, void *argument); RUBY_SYMBOL_EXPORT_END #endif /* INTERNAL_IO_H */ diff --git a/internal/load.h b/internal/load.h index d4c0bb91ba..fb880a43ba 100644 --- a/internal/load.h +++ b/internal/load.h @@ -12,6 +12,8 @@ /* load.c */ VALUE rb_get_expanded_load_path(void); +VALUE rb_load_entrypoint(VALUE args); +VALUE rb_require_relative_entrypoint(VALUE fname); int rb_require_internal(VALUE fname); NORETURN(void rb_load_fail(VALUE, const char*)); diff --git a/internal/missing.h b/internal/missing.h index c0992a151a..6ca508c8f9 100644 --- a/internal/missing.h +++ b/internal/missing.h @@ -13,6 +13,7 @@ /* missing/setproctitle.c */ #ifndef HAVE_SETPROCTITLE extern void ruby_init_setproctitle(int argc, char *argv[]); +extern void ruby_free_proctitle(void); #endif #endif /* INTERNAL_MISSING_H */ diff --git a/internal/numeric.h b/internal/numeric.h index 89bc54b307..d3905f048c 100644 --- a/internal/numeric.h +++ b/internal/numeric.h @@ -85,7 +85,9 @@ VALUE rb_int_cmp(VALUE x, VALUE y); VALUE rb_int_equal(VALUE x, VALUE y); VALUE rb_int_divmod(VALUE x, VALUE y); VALUE rb_int_and(VALUE x, VALUE y); +VALUE rb_int_xor(VALUE x, VALUE y); VALUE rb_int_lshift(VALUE x, VALUE y); +VALUE rb_int_rshift(VALUE x, VALUE y); VALUE rb_int_div(VALUE x, VALUE y); int rb_int_positive_p(VALUE num); int rb_int_negative_p(VALUE num); @@ -111,7 +113,6 @@ RUBY_SYMBOL_EXPORT_BEGIN /* numeric.c (export) */ RUBY_SYMBOL_EXPORT_END -MJIT_SYMBOL_EXPORT_BEGIN VALUE rb_flo_div_flo(VALUE x, VALUE y); double ruby_float_mod(double x, double y); VALUE rb_float_equal(VALUE x, VALUE y); @@ -125,7 +126,54 @@ VALUE rb_int_abs(VALUE num); VALUE rb_int_bit_length(VALUE num); VALUE rb_int_uminus(VALUE num); VALUE rb_int_comp(VALUE num); -MJIT_SYMBOL_EXPORT_END + +// Unified 128-bit integer structures that work with or without native support: +union rb_uint128 { +#ifdef WORDS_BIGENDIAN + struct { + uint64_t high; + uint64_t low; + } parts; +#else + struct { + uint64_t low; + uint64_t high; + } parts; +#endif +#ifdef HAVE_UINT128_T + uint128_t value; +#endif +}; +typedef union rb_uint128 rb_uint128_t; + +union rb_int128 { +#ifdef WORDS_BIGENDIAN + struct { + uint64_t high; + uint64_t low; + } parts; +#else + struct { + uint64_t low; + uint64_t high; + } parts; +#endif +#ifdef HAVE_UINT128_T + int128_t value; +#endif +}; +typedef union rb_int128 rb_int128_t; + +union uint128_int128_conversion { + rb_uint128_t uint128; + rb_int128_t int128; +}; + +// Conversion functions for 128-bit integers: +rb_uint128_t rb_numeric_to_uint128(VALUE x); +rb_int128_t rb_numeric_to_int128(VALUE x); +VALUE rb_uint128_to_numeric(rb_uint128_t n); +VALUE rb_int128_to_numeric(rb_int128_t n); static inline bool INT_POSITIVE_P(VALUE num) @@ -160,7 +208,7 @@ rb_num_compare_with_zero(VALUE num, ID mid) { VALUE zero = INT2FIX(0); VALUE r = rb_check_funcall(num, mid, 1, &zero); - if (r == Qundef) { + if (RB_UNDEF_P(r)) { rb_cmperr(num, zero); } return r; diff --git a/internal/object.h b/internal/object.h index 7b54e13dd2..3bde53c31b 100644 --- a/internal/object.h +++ b/internal/object.h @@ -11,10 +11,14 @@ #include "ruby/ruby.h" /* for VALUE */ /* object.c */ +size_t rb_obj_embedded_size(uint32_t fields_count); +VALUE rb_class_allocate_instance(VALUE klass); VALUE rb_class_search_ancestor(VALUE klass, VALUE super); NORETURN(void rb_undefined_alloc(VALUE klass)); double rb_num_to_dbl(VALUE val); VALUE rb_obj_dig(int argc, VALUE *argv, VALUE self, VALUE notfound); +VALUE rb_obj_clone_setup(VALUE obj, VALUE clone, VALUE kwfreeze); +VALUE rb_obj_dup_setup(VALUE obj, VALUE dup); VALUE rb_immutable_obj_clone(int, VALUE *, VALUE); VALUE rb_check_convert_type_with_id(VALUE,int,const char*,ID); int rb_bool_expected(VALUE, const char *, int raise); @@ -27,7 +31,6 @@ RUBY_SYMBOL_EXPORT_BEGIN int rb_opts_exception_p(VALUE opts, int default_value); RUBY_SYMBOL_EXPORT_END -MJIT_SYMBOL_EXPORT_BEGIN CONSTFUNC(VALUE rb_obj_equal(VALUE obj1, VALUE obj2)); CONSTFUNC(VALUE rb_obj_not(VALUE obj)); VALUE rb_obj_not_equal(VALUE obj1, VALUE obj2); @@ -36,7 +39,6 @@ VALUE rb_false(VALUE obj); VALUE rb_convert_type_with_id(VALUE v, int t, const char* nam, ID mid); VALUE rb_obj_size(VALUE self, VALUE args, VALUE obj); VALUE rb_get_freeze_opt(int argc, VALUE *argv); -MJIT_SYMBOL_EXPORT_END static inline void RBASIC_SET_CLASS_RAW(VALUE obj, VALUE klass) diff --git a/internal/parse.h b/internal/parse.h index f242c384ad..8e04664ae8 100644 --- a/internal/parse.h +++ b/internal/parse.h @@ -8,18 +8,124 @@ * file COPYING are met. Consult the file for details. * @brief Internal header for the parser. */ -#include "ruby/ruby.h" /* for VALUE */ +#include <limits.h> +#include "rubyparser.h" +#include "internal/static_assert.h" + +// The default parser to use for Ruby code. +typedef enum { + RB_DEFAULT_PARSER_PARSE_Y, + RB_DEFAULT_PARSER_PRISM, +} ruby_default_parser_enum; + +ruby_default_parser_enum rb_ruby_default_parser(void); +void rb_ruby_default_parser_set(ruby_default_parser_enum parser); + +#define rb_ruby_prism_p() (rb_ruby_default_parser() == RB_DEFAULT_PARSER_PRISM) + +#ifdef UNIVERSAL_PARSER +#define rb_encoding const void +#endif + struct rb_iseq_struct; /* in vm_core.h */ +/* structs for managing terminator of string literal and heredocment */ +typedef struct rb_strterm_literal_struct { + long nest; + int func; /* STR_FUNC_* (e.g., STR_FUNC_ESCAPE and STR_FUNC_EXPAND) */ + int paren; /* '(' of `%q(...)` */ + int term; /* ')' of `%q(...)` */ +} rb_strterm_literal_t; + +typedef struct rb_strterm_heredoc_struct { + rb_parser_string_t *lastline; /* the string of line that contains `<<"END"` */ + long offset; /* the column of END in `<<"END"` */ + int sourceline; /* lineno of the line that contains `<<"END"` */ + unsigned length; /* the length of END in `<<"END"` */ + uint8_t quote; + uint8_t func; +} rb_strterm_heredoc_t; + +#define HERETERM_LENGTH_MAX UINT_MAX + +typedef struct rb_strterm_struct { + bool heredoc; + union { + rb_strterm_literal_t literal; + rb_strterm_heredoc_t heredoc; + } u; +} rb_strterm_t; + /* parse.y */ -VALUE rb_parser_set_yydebug(VALUE, VALUE); -void *rb_parser_load_file(VALUE parser, VALUE name); -void rb_parser_keep_script_lines(VALUE vparser); -void rb_parser_error_tolerant(VALUE vparser); -void rb_parser_keep_tokens(VALUE vparser); +void rb_ruby_parser_mark(void *ptr); +size_t rb_ruby_parser_memsize(const void *ptr); + +void rb_ruby_parser_set_options(rb_parser_t *p, int print, int loop, int chomp, int split); +rb_parser_t *rb_ruby_parser_set_context(rb_parser_t *p, const struct rb_iseq_struct *base, int main); +void rb_ruby_parser_set_script_lines(rb_parser_t *p); +void rb_ruby_parser_error_tolerant(rb_parser_t *p); +void rb_ruby_parser_keep_tokens(rb_parser_t *p); +typedef rb_parser_string_t*(rb_parser_lex_gets_func)(struct parser_params*, rb_parser_input_data, int); +rb_ast_t *rb_parser_compile(rb_parser_t *p, rb_parser_lex_gets_func *gets, VALUE fname, rb_parser_input_data input, int line); RUBY_SYMBOL_EXPORT_BEGIN -VALUE rb_parser_set_context(VALUE, const struct rb_iseq_struct *, int); + +rb_encoding *rb_ruby_parser_encoding(rb_parser_t *p); +int rb_ruby_parser_end_seen_p(rb_parser_t *p); +int rb_ruby_parser_set_yydebug(rb_parser_t *p, int flag); +rb_parser_string_t *rb_str_to_parser_string(rb_parser_t *p, VALUE str); +void rb_parser_string_free(rb_parser_t *p, rb_parser_string_t *str); + +int rb_parser_dvar_defined_ref(struct parser_params*, ID, ID**); +ID rb_parser_internal_id(struct parser_params*); +typedef void (*rb_parser_reg_fragment_error_func)(struct parser_params *, VALUE); +int rb_parser_reg_fragment_check(struct parser_params*, rb_parser_string_t*, int, rb_parser_reg_fragment_error_func); +int rb_reg_named_capture_assign_iter_impl(struct parser_params *p, const char *s, long len, rb_encoding *enc, NODE **succ_block, const rb_code_location_t *loc, rb_parser_assignable_func assignable); +int rb_parser_local_defined(struct parser_params *p, ID id, const struct rb_iseq_struct *iseq); +NODE *rb_parser_assignable(struct parser_params *p, ID id, NODE *val, const YYLTYPE *loc); + RUBY_SYMBOL_EXPORT_END +#ifndef UNIVERSAL_PARSER +rb_parser_t *rb_ruby_parser_allocate(void); +rb_parser_t *rb_ruby_parser_new(void); +#endif + +#ifdef RIPPER +void ripper_parser_mark(void *ptr); +void ripper_parser_free(void *ptr); +size_t ripper_parser_memsize(const void *ptr); +void ripper_error(struct parser_params *p); +VALUE ripper_value(struct parser_params *p); +int rb_ruby_parser_get_yydebug(rb_parser_t *p); +void rb_ruby_parser_set_value(rb_parser_t *p, VALUE value); +int rb_ruby_parser_error_p(rb_parser_t *p); +VALUE rb_ruby_parser_debug_output(rb_parser_t *p); +void rb_ruby_parser_set_debug_output(rb_parser_t *p, VALUE output); +VALUE rb_ruby_parser_parsing_thread(rb_parser_t *p); +void rb_ruby_parser_set_parsing_thread(rb_parser_t *p, VALUE parsing_thread); +void rb_ruby_parser_ripper_initialize(rb_parser_t *p, rb_parser_lex_gets_func *gets, rb_parser_input_data input, VALUE sourcefile_string, const char *sourcefile, int sourceline); +VALUE rb_ruby_parser_result(rb_parser_t *p); +rb_encoding *rb_ruby_parser_enc(rb_parser_t *p); +VALUE rb_ruby_parser_ruby_sourcefile_string(rb_parser_t *p); +int rb_ruby_parser_ruby_sourceline(rb_parser_t *p); +int rb_ruby_parser_lex_state(rb_parser_t *p); +void rb_ruby_ripper_parse0(rb_parser_t *p); +int rb_ruby_ripper_dedent_string(rb_parser_t *p, rb_parser_string_t *string, int width); +int rb_ruby_ripper_initialized_p(rb_parser_t *p); +void rb_ruby_ripper_parser_initialize(rb_parser_t *p); +long rb_ruby_ripper_column(rb_parser_t *p); +long rb_ruby_ripper_token_len(rb_parser_t *p); +rb_parser_string_t *rb_ruby_ripper_lex_lastline(rb_parser_t *p); +VALUE rb_ruby_ripper_lex_state_name(struct parser_params *p, int state); +#ifdef UNIVERSAL_PARSER +rb_parser_t *rb_ripper_parser_params_allocate(const rb_parser_config_t *config); +#endif +struct parser_params *rb_ruby_ripper_parser_allocate(void); +#endif + +#ifdef UNIVERSAL_PARSER +#undef rb_encoding +#endif + #endif /* INTERNAL_PARSE_H */ diff --git a/internal/proc.h b/internal/proc.h index 2416c31e14..24a077ca6d 100644 --- a/internal/proc.h +++ b/internal/proc.h @@ -22,11 +22,9 @@ int rb_block_min_max_arity(int *max); VALUE rb_block_to_s(VALUE self, const struct rb_block *block, const char *additional_info); VALUE rb_callable_receiver(VALUE); -MJIT_SYMBOL_EXPORT_BEGIN -VALUE rb_func_proc_new(rb_block_call_func_t func, VALUE val); +VALUE rb_func_proc_dup(VALUE src_obj); VALUE rb_func_lambda_new(rb_block_call_func_t func, VALUE val, int min_argc, int max_argc); VALUE rb_iseq_location(const struct rb_iseq_struct *iseq); VALUE rb_sym_to_proc(VALUE sym); -MJIT_SYMBOL_EXPORT_END #endif /* INTERNAL_PROC_H */ diff --git a/internal/process.h b/internal/process.h index ceadfdcbbb..fd4994cb4b 100644 --- a/internal/process.h +++ b/internal/process.h @@ -20,8 +20,8 @@ #endif #include "ruby/ruby.h" /* for VALUE */ +#include "internal/compilers.h" /* for __has_warning */ #include "internal/imemo.h" /* for RB_IMEMO_TMPBUF_PTR */ -#include "internal/warnings.h" /* for COMPILER_WARNING_PUSH */ #define RB_MAX_GROUPS (65536) @@ -121,17 +121,4 @@ ARGVSTR2ARGC(VALUE argv_str) return i - 1; } -#ifdef HAVE_WORKING_FORK -COMPILER_WARNING_PUSH -#if __has_warning("-Wdeprecated-declarations") || RBIMPL_COMPILER_IS(GCC) -COMPILER_WARNING_IGNORED(-Wdeprecated-declarations) -#endif -static inline rb_pid_t -rb_fork(void) -{ - return fork(); -} -COMPILER_WARNING_POP -#endif - #endif /* INTERNAL_PROCESS_H */ diff --git a/internal/ractor.h b/internal/ractor.h index eef5ffdb89..a65907a05a 100644 --- a/internal/ractor.h +++ b/internal/ractor.h @@ -3,4 +3,8 @@ void rb_ractor_ensure_main_ractor(const char *msg); +RUBY_SYMBOL_EXPORT_BEGIN +void rb_ractor_setup_belonging(VALUE obj); +RUBY_SYMBOL_EXPORT_END + #endif /* INTERNAL_RACTOR_H */ diff --git a/internal/random.h b/internal/random.h index 231e2d5d7e..127b908e16 100644 --- a/internal/random.h +++ b/internal/random.h @@ -12,5 +12,6 @@ /* random.c */ int ruby_fill_random_bytes(void *, size_t, int); +void rb_free_default_rand_key(void); #endif /* INTERNAL_RANDOM_H */ diff --git a/internal/range.h b/internal/range.h index 8daba0ecab..80493ce13e 100644 --- a/internal/range.h +++ b/internal/range.h @@ -18,19 +18,19 @@ static inline VALUE RANGE_EXCL(VALUE r); static inline VALUE RANGE_BEG(VALUE r) { - return RSTRUCT(r)->as.ary[0]; + return RSTRUCT_GET_RAW(r, 0); } static inline VALUE RANGE_END(VALUE r) { - return RSTRUCT(r)->as.ary[1]; + return RSTRUCT_GET_RAW(r, 1); } static inline VALUE RANGE_EXCL(VALUE r) { - return RSTRUCT(r)->as.ary[2]; + return RSTRUCT_GET_RAW(r, 2); } VALUE diff --git a/internal/rational.h b/internal/rational.h index 61ddbf089a..f11fab4583 100644 --- a/internal/rational.h +++ b/internal/rational.h @@ -10,7 +10,6 @@ */ #include "ruby/internal/config.h" /* for HAVE_LIBGMP */ #include "ruby/ruby.h" /* for struct RBasic */ -#include "internal/gc.h" /* for RB_OBJ_WRITE */ #include "internal/numeric.h" /* for INT_POSITIVE_P */ #include "ruby_assert.h" /* for assert */ diff --git a/internal/re.h b/internal/re.h index 8b31b3d8a5..593e5c464f 100644 --- a/internal/re.h +++ b/internal/re.h @@ -14,17 +14,20 @@ /* re.c */ VALUE rb_reg_compile(VALUE str, int options, const char *sourcefile, int sourceline); VALUE rb_reg_check_preprocess(VALUE); -long rb_reg_search0(VALUE, VALUE, long, int, int); +long rb_reg_search0(VALUE, VALUE, long, int, int, VALUE *); VALUE rb_reg_match_p(VALUE re, VALUE str, long pos); bool rb_reg_start_with_p(VALUE re, VALUE str); VALUE rb_reg_hash(VALUE re); VALUE rb_reg_equal(VALUE re1, VALUE re2); -void rb_backref_set_string(VALUE string, long pos, long len); +VALUE rb_backref_set_string(VALUE string, long pos, long len); void rb_match_unbusy(VALUE); int rb_match_count(VALUE match); -int rb_match_nth_defined(int nth, VALUE match); -MJIT_SYMBOL_EXPORT_BEGIN VALUE rb_reg_new_ary(VALUE ary, int options); -MJIT_SYMBOL_EXPORT_END +VALUE rb_reg_last_defined(VALUE match); + +#define ARG_REG_OPTION_MASK \ + (ONIG_OPTION_IGNORECASE|ONIG_OPTION_MULTILINE|ONIG_OPTION_EXTEND) +#define ARG_ENCODING_FIXED 16 +#define ARG_ENCODING_NONE 32 #endif /* INTERNAL_RE_H */ diff --git a/internal/ruby_parser.h b/internal/ruby_parser.h new file mode 100644 index 0000000000..8e306d18de --- /dev/null +++ b/internal/ruby_parser.h @@ -0,0 +1,102 @@ +#ifndef INTERNAL_RUBY_PARSE_H +#define INTERNAL_RUBY_PARSE_H + +#include "internal.h" +#include "internal/bignum.h" +#include "internal/compilers.h" +#include "internal/complex.h" +#include "internal/parse.h" +#include "internal/rational.h" +#include "rubyparser.h" +#include "vm.h" + +struct lex_pointer_string { + VALUE str; + long ptr; +}; + +RUBY_SYMBOL_EXPORT_BEGIN +#ifdef UNIVERSAL_PARSER +const rb_parser_config_t *rb_ruby_parser_config(void); +rb_parser_t *rb_parser_params_new(void); +#endif +VALUE rb_parser_set_context(VALUE, const struct rb_iseq_struct *, int); +VALUE rb_parser_new(void); +VALUE rb_parser_compile_string_path(VALUE vparser, VALUE fname, VALUE src, int line); +VALUE rb_str_new_parser_string(rb_parser_string_t *str); +VALUE rb_str_new_mutable_parser_string(rb_parser_string_t *str); +rb_parser_string_t *rb_parser_lex_get_str(struct parser_params *p, struct lex_pointer_string *ptr_str); + +VALUE rb_node_str_string_val(const NODE *); +VALUE rb_node_sym_string_val(const NODE *); +VALUE rb_node_dstr_string_val(const NODE *); +VALUE rb_node_regx_string_val(const NODE *); +VALUE rb_node_dregx_string_val(const NODE *); +VALUE rb_node_line_lineno_val(const NODE *); +VALUE rb_node_file_path_val(const NODE *); +VALUE rb_node_encoding_val(const NODE *); + +VALUE rb_node_integer_literal_val(const NODE *); +VALUE rb_node_float_literal_val(const NODE *); +VALUE rb_node_rational_literal_val(const NODE *); +VALUE rb_node_imaginary_literal_val(const NODE *); +RUBY_SYMBOL_EXPORT_END + +VALUE rb_parser_end_seen_p(VALUE); +VALUE rb_parser_encoding(VALUE); +VALUE rb_parser_set_yydebug(VALUE, VALUE); +VALUE rb_parser_build_script_lines_from(rb_parser_ary_t *script_lines); +void rb_parser_set_options(VALUE, int, int, int, int); +VALUE rb_parser_load_file(VALUE parser, VALUE name); +void rb_parser_set_script_lines(VALUE vparser); +void rb_parser_error_tolerant(VALUE vparser); +void rb_parser_keep_tokens(VALUE vparser); + +VALUE rb_parser_compile_string(VALUE, const char*, VALUE, int); +VALUE rb_parser_compile_file_path(VALUE vparser, VALUE fname, VALUE input, int line); +VALUE rb_parser_compile_generic(VALUE vparser, rb_parser_lex_gets_func *lex_gets, VALUE fname, VALUE input, int line); +VALUE rb_parser_compile_array(VALUE vparser, VALUE fname, VALUE array, int start); + +enum lex_state_bits { + EXPR_BEG_bit, /* ignore newline, +/- is a sign. */ + EXPR_END_bit, /* newline significant, +/- is an operator. */ + EXPR_ENDARG_bit, /* ditto, and unbound braces. */ + EXPR_ENDFN_bit, /* ditto, and unbound braces. */ + EXPR_ARG_bit, /* newline significant, +/- is an operator. */ + EXPR_CMDARG_bit, /* newline significant, +/- is an operator. */ + EXPR_MID_bit, /* newline significant, +/- is an operator. */ + EXPR_FNAME_bit, /* ignore newline, no reserved words. */ + EXPR_DOT_bit, /* right after `.', `&.' or `::', no reserved words. */ + EXPR_CLASS_bit, /* immediate after `class', no here document. */ + EXPR_LABEL_bit, /* flag bit, label is allowed. */ + EXPR_LABELED_bit, /* flag bit, just after a label. */ + EXPR_FITEM_bit, /* symbol literal as FNAME. */ + EXPR_MAX_STATE +}; +/* examine combinations */ +enum lex_state_e { +#define DEF_EXPR(n) EXPR_##n = (1 << EXPR_##n##_bit) + DEF_EXPR(BEG), + DEF_EXPR(END), + DEF_EXPR(ENDARG), + DEF_EXPR(ENDFN), + DEF_EXPR(ARG), + DEF_EXPR(CMDARG), + DEF_EXPR(MID), + DEF_EXPR(FNAME), + DEF_EXPR(DOT), + DEF_EXPR(CLASS), + DEF_EXPR(LABEL), + DEF_EXPR(LABELED), + DEF_EXPR(FITEM), + EXPR_VALUE = EXPR_BEG, + EXPR_BEG_ANY = (EXPR_BEG | EXPR_MID | EXPR_CLASS), + EXPR_ARG_ANY = (EXPR_ARG | EXPR_CMDARG), + EXPR_END_ANY = (EXPR_END | EXPR_ENDARG | EXPR_ENDFN), + EXPR_NONE = 0 +}; + +VALUE rb_ruby_ast_new(const NODE *const root); +rb_ast_t *rb_ruby_ast_data_get(VALUE ast_value); + +#endif /* INTERNAL_RUBY_PARSE_H */ diff --git a/internal/sanitizers.h b/internal/sanitizers.h index 6e2d81137f..feafb4e616 100644 --- a/internal/sanitizers.h +++ b/internal/sanitizers.h @@ -16,25 +16,42 @@ #endif #ifdef HAVE_SANITIZER_ASAN_INTERFACE_H -# include <sanitizer/asan_interface.h> +# if __has_feature(address_sanitizer) || defined(__SANITIZE_ADDRESS__) +# define RUBY_ASAN_ENABLED +# include <sanitizer/asan_interface.h> +# endif #endif #ifdef HAVE_SANITIZER_MSAN_INTERFACE_H # if __has_feature(memory_sanitizer) +# define RUBY_MSAN_ENABLED # include <sanitizer/msan_interface.h> # endif #endif +#ifdef HAVE_SANITIZER_TSAN_INTERFACE_H +# if __has_feature(thread_sanitizer) || defined(__SANITIZE_THREAD__) +# define RUBY_TSAN_ENABLED +# include <sanitizer/tsan_interface.h> +# endif +#endif + #include "ruby/internal/stdbool.h" /* for bool */ #include "ruby/ruby.h" /* for VALUE */ #if 0 -#elif __has_feature(memory_sanitizer) && __has_feature(address_sanitizer) +#elif defined(RUBY_ASAN_ENABLED) && defined(RUBY_MSAN_ENABLED) # define ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS(x) \ __attribute__((__no_sanitize__("memory, address"), __noinline__)) x -#elif __has_feature(address_sanitizer) +#elif defined(RUBY_ASAN_ENABLED) # define ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS(x) \ __attribute__((__no_sanitize__("address"), __noinline__)) x +#elif defined(RUBY_MSAN_ENABLED) + # define ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS(x) \ + __attribute__((__no_sanitize__("memory"), __noinline__)) x +#elif defined(RUBY_TSAN_ENABLED) +# define ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS(x) \ + __attribute__((__no_sanitize__("thread"), __noinline__)) x #elif defined(NO_SANITIZE_ADDRESS) # define ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS(x) \ NO_SANITIZE_ADDRESS(NOINLINE(x)) @@ -50,23 +67,26 @@ # include "internal/warnings.h" # undef NO_SANITIZE # define NO_SANITIZE(x, y) \ - COMPILER_WARNING_PUSH; \ - COMPILER_WARNING_IGNORED(-Wattributes); \ + COMPILER_WARNING_PUSH \ + COMPILER_WARNING_IGNORED(-Wattributes) \ __attribute__((__no_sanitize__(x))) y; \ - COMPILER_WARNING_POP + COMPILER_WARNING_POP \ + y #endif #ifndef NO_SANITIZE # define NO_SANITIZE(x, y) y #endif -#if !__has_feature(address_sanitizer) +#ifndef RUBY_ASAN_ENABLED # define __asan_poison_memory_region(x, y) # define __asan_unpoison_memory_region(x, y) # define __asan_region_is_poisoned(x, y) 0 +# define __asan_get_current_fake_stack() NULL +# define __asan_addr_is_in_fake_stack(fake_stack, slot, start, end) NULL #endif -#if !__has_feature(memory_sanitizer) +#ifndef RUBY_MSAN_ENABLED # define __msan_allocated_memory(x, y) ((void)(x), (void)(y)) # define __msan_poison(x, y) ((void)(x), (void)(y)) # define __msan_unpoison(x, y) ((void)(x), (void)(y)) @@ -89,9 +109,7 @@ # define VALGRIND_MAKE_MEM_UNDEFINED(p, n) 0 #endif -#ifndef MJIT_HEADER - -/*! +/** * This function asserts that a (continuous) memory region from ptr to size * being "poisoned". Both read / write access to such memory region are * prohibited until properly unpoisoned. The region must be previously @@ -101,8 +119,8 @@ * region to reuse later: poison when you keep it unused, and unpoison when you * reuse. * - * \param[in] ptr pointer to the beginning of the memory region to poison. - * \param[in] size the length of the memory region to poison. + * @param[in] ptr pointer to the beginning of the memory region to poison. + * @param[in] size the length of the memory region to poison. */ static inline void asan_poison_memory_region(const volatile void *ptr, size_t size) @@ -111,41 +129,48 @@ asan_poison_memory_region(const volatile void *ptr, size_t size) __asan_poison_memory_region(ptr, size); } -/*! - * This is a variant of asan_poison_memory_region that takes a VALUE. - * - * \param[in] obj target object. - */ -static inline void -asan_poison_object(VALUE obj) -{ - MAYBE_UNUSED(struct RVALUE *) ptr = (void *)obj; - asan_poison_memory_region(ptr, SIZEOF_VALUE); -} - -#if !__has_feature(address_sanitizer) -#define asan_poison_object_if(ptr, obj) ((void)(ptr), (void)(obj)) -#else +#ifdef RUBY_ASAN_ENABLED #define asan_poison_object_if(ptr, obj) do { \ - if (ptr) asan_poison_object(obj); \ + if (ptr) rb_asan_poison_object(obj); \ } while (0) +#else +#define asan_poison_object_if(ptr, obj) ((void)(ptr), (void)(obj)) #endif -/*! +#ifdef RUBY_ASAN_ENABLED +RUBY_SYMBOL_EXPORT_BEGIN +/** + * This is a variant of asan_poison_memory_region that takes a VALUE. + * + * @param[in] obj target object. + */ +void rb_asan_poison_object(VALUE obj); + +/** * This function predicates if the given object is fully addressable or not. * - * \param[in] obj target object. - * \retval 0 the given object is fully addressable. - * \retval otherwise pointer to first such byte who is poisoned. + * @param[in] obj target object. + * @retval 0 the given object is fully addressable. + * @retval otherwise pointer to first such byte who is poisoned. */ -static inline void * -asan_poisoned_object_p(VALUE obj) -{ - MAYBE_UNUSED(struct RVALUE *) ptr = (void *)obj; - return __asan_region_is_poisoned(ptr, SIZEOF_VALUE); -} +void *rb_asan_poisoned_object_p(VALUE obj); -/*! +/** + * This is a variant of asan_unpoison_memory_region that takes a VALUE. + * + * @param[in] obj target object. + * @param[in] malloc_p if the memory region is like a malloc's return value or not. + */ +void rb_asan_unpoison_object(VALUE obj, bool newobj_p); + +RUBY_SYMBOL_EXPORT_END +#else +# define rb_asan_poison_object(obj) ((void)obj) +# define rb_asan_poisoned_object_p(obj) ((void)obj, NULL) +# define rb_asan_unpoison_object(obj, newobj_p) ((void)obj, (void)newobj_p) +#endif + +/** * This function asserts that a (formally poisoned) memory region from ptr to * size is now addressable. Write access to such memory region gets allowed. * However read access might or might not be possible depending on situations, @@ -156,9 +181,9 @@ asan_poisoned_object_p(VALUE obj) * the other hand, that memory region is fully defined and can be read * immediately. * - * \param[in] ptr pointer to the beginning of the memory region to unpoison. - * \param[in] size the length of the memory region. - * \param[in] malloc_p if the memory region is like a malloc's return value or not. + * @param[in] ptr pointer to the beginning of the memory region to unpoison. + * @param[in] size the length of the memory region. + * @param[in] malloc_p if the memory region is like a malloc's return value or not. */ static inline void asan_unpoison_memory_region(const volatile void *ptr, size_t size, bool malloc_p) @@ -172,19 +197,150 @@ asan_unpoison_memory_region(const volatile void *ptr, size_t size, bool malloc_p } } -/*! - * This is a variant of asan_unpoison_memory_region that takes a VALUE. +static inline void * +asan_unpoison_object_temporary(VALUE obj) +{ + void *ptr = rb_asan_poisoned_object_p(obj); + rb_asan_unpoison_object(obj, false); + return ptr; +} + +static inline void * +asan_poison_object_restore(VALUE obj, void *ptr) +{ + if (ptr) { + rb_asan_poison_object(obj); + } + return NULL; +} + +#define asan_unpoisoning_object(obj) \ + for (void *poisoned = asan_unpoison_object_temporary(obj), \ + *unpoisoning = &poisoned; /* flag to loop just once */ \ + unpoisoning; \ + unpoisoning = asan_poison_object_restore(obj, poisoned)) + + +static inline void * +asan_unpoison_memory_region_temporary(void *ptr, size_t len) +{ + void *poisoned_ptr = __asan_region_is_poisoned(ptr, len); + asan_unpoison_memory_region(ptr, len, false); + return poisoned_ptr; +} + +static inline void * +asan_poison_memory_region_restore(void *ptr, size_t len, void *poisoned_ptr) +{ + if (poisoned_ptr) { + asan_poison_memory_region(ptr, len); + } + return NULL; +} + +#define asan_unpoisoning_memory_region(ptr, len) \ + for (void *poisoned = asan_unpoison_memory_region_temporary(ptr, len), \ + *unpoisoning = &poisoned; /* flag to loop just once */ \ + unpoisoning; \ + unpoisoning = asan_poison_memory_region_restore(ptr, len, poisoned)) + +/** + * Checks if the given pointer is on an ASAN fake stack. If so, it returns the + * address this variable has on the real frame; if not, it returns the origin + * address unmodified. * - * \param[in] obj target object. - * \param[in] malloc_p if the memory region is like a malloc's return value or not. + * n.b. - _dereferencing_ the returned address is meaningless and should not + * be done; even though ASAN reserves space for the variable in both the real and + * fake stacks, the _value_ of that variable is only in the fake stack. + * + * n.b. - this only works for addresses passed in from local variables on the same + * thread, because the ASAN fake stacks are threadlocal. + * + * @param[in] slot the address of some local variable + * @retval a pointer to something from that frame on the _real_ machine stack */ -static inline void -asan_unpoison_object(VALUE obj, bool newobj_p) +static inline void * +asan_get_real_stack_addr(void* slot) +{ + VALUE *addr; + addr = __asan_addr_is_in_fake_stack(__asan_get_current_fake_stack(), slot, NULL, NULL); + return addr ? addr : slot; +} + +/** + * Gets the current thread's fake stack handle, which can be passed into get_fake_stack_extents + * + * @retval An opaque value which can be passed to asan_get_fake_stack_extents + */ +static inline void * +asan_get_thread_fake_stack_handle(void) { - MAYBE_UNUSED(struct RVALUE *) ptr = (void *)obj; - asan_unpoison_memory_region(ptr, SIZEOF_VALUE, newobj_p); + return __asan_get_current_fake_stack(); } -#endif /* MJIT_HEADER */ +/** + * Checks if the given VALUE _actually_ represents a pointer to an ASAN fake stack. + * + * If the given slot _is_ actually a reference to an ASAN fake stack, and that fake stack + * contains the real values for the passed-in range of machine stack addresses, returns true + * and the range of the fake stack through the outparams. + * + * Otherwise, returns false, and sets the outparams to NULL. + * + * Note that this function expects "start" to be > "end" on downward-growing stack architectures; + * + * @param[in] thread_fake_stack_handle The asan fake stack reference for the thread we're scanning + * @param[in] slot The value on the machine stack we want to inspect + * @param[in] machine_stack_start The extents of the real machine stack on which slot lives + * @param[in] machine_stack_end The extents of the real machine stack on which slot lives + * @param[out] fake_stack_start_out The extents of the fake stack which contains real VALUEs + * @param[out] fake_stack_end_out The extents of the fake stack which contains real VALUEs + * @return Whether slot is a pointer to a fake stack for the given machine stack range +*/ + +static inline bool +asan_get_fake_stack_extents(void *thread_fake_stack_handle, VALUE slot, + void *machine_stack_start, void *machine_stack_end, + void **fake_stack_start_out, void **fake_stack_end_out) +{ + /* the ifdef is needed here to suppress a warning about fake_frame_{start/end} being + uninitialized if __asan_addr_is_in_fake_stack is an empty macro */ +#ifdef RUBY_ASAN_ENABLED + void *fake_frame_start; + void *fake_frame_end; + void *real_stack_frame = __asan_addr_is_in_fake_stack( + thread_fake_stack_handle, (void *)slot, &fake_frame_start, &fake_frame_end + ); + if (real_stack_frame) { + bool in_range; +#if STACK_GROW_DIRECTION < 0 + in_range = machine_stack_start >= real_stack_frame && real_stack_frame >= machine_stack_end; +#else + in_range = machine_stack_start <= real_stack_frame && real_stack_frame <= machine_stack_end; +#endif + if (in_range) { + *fake_stack_start_out = fake_frame_start; + *fake_stack_end_out = fake_frame_end; + return true; + } + } +#endif + *fake_stack_start_out = 0; + *fake_stack_end_out = 0; + return false; +} + +extern const char ruby_asan_default_options[]; + +#ifdef RUBY_ASAN_ENABLED +/* Compile in the ASAN options Ruby needs, rather than relying on environment variables, so + * that even tests which fork ruby with a clean environment will run ASAN with the right + * settings */ +# undef RUBY__ASAN_DEFAULT_OPTIONS +# define RUBY__ASAN_DEFAULT_OPTIONS \ + RBIMPL_SYMBOL_EXPORT_BEGIN() \ + const char * __asan_default_options(void) {return ruby_asan_default_options;} \ + RBIMPL_SYMBOL_EXPORT_END() +#endif #endif /* INTERNAL_SANITIZERS_H */ diff --git a/internal/set_table.h b/internal/set_table.h new file mode 100644 index 0000000000..3c29abb4f5 --- /dev/null +++ b/internal/set_table.h @@ -0,0 +1,70 @@ +#ifndef INTERNAL_SET_TABLE_H +#define INTERNAL_SET_TABLE_H + +#include "ruby/st.h" + +struct set_table_entry; + +typedef struct set_table_entry set_table_entry; + +struct set_table { + /* Cached features of the table -- see st.c for more details. */ + unsigned char entry_power, bin_power, size_ind; + /* How many times the table was rebuilt. */ + unsigned int rebuilds_num; + const struct st_hash_type *type; + /* Number of entries currently in the table. */ + st_index_t num_entries; + + /* Start and bound index of entries in array entries. + entries_starts and entries_bound are in interval + [0,allocated_entries]. */ + st_index_t entries_start, entries_bound; + + /** + * Array of size 2^entry_power. + * Followed by st_index_t *bins, Array of bins used for access by keys. + */ + set_table_entry *entries; +}; + +typedef struct set_table set_table; + +typedef int set_foreach_callback_func(st_data_t, st_data_t); +typedef int set_foreach_check_callback_func(st_data_t, st_data_t, int); +typedef int set_update_callback_func(st_data_t *key, st_data_t arg, int existing); + +#define set_table_size rb_set_table_size +size_t rb_set_table_size(const struct set_table *tbl); +#define set_init_table_with_size rb_set_init_table_with_size +set_table *rb_set_init_table_with_size(set_table *tab, const struct st_hash_type *, st_index_t); +#define set_init_numtable rb_set_init_numtable +set_table *rb_set_init_numtable(void); +#define set_init_numtable_with_size rb_set_init_numtable_with_size +set_table *rb_set_init_numtable_with_size(st_index_t size); +#define set_table_delete rb_set_table_delete +int rb_set_table_delete(set_table *, st_data_t *); /* returns 0:notfound 1:deleted */ +#define set_insert rb_set_insert +int rb_set_insert(set_table *, st_data_t); +#define set_table_lookup rb_set_table_lookup +int rb_set_table_lookup(set_table *, st_data_t); +#define set_foreach_with_replace rb_set_foreach_with_replace +int rb_set_foreach_with_replace(set_table *tab, set_foreach_check_callback_func *func, set_update_callback_func *replace, st_data_t arg); +#define set_table_foreach rb_set_table_foreach +int rb_set_table_foreach(set_table *, set_foreach_callback_func *, st_data_t); +#define set_foreach_check rb_set_foreach_check +int rb_set_foreach_check(set_table *, set_foreach_check_callback_func *, st_data_t, st_data_t); +#define set_keys rb_set_keys +st_index_t rb_set_keys(set_table *table, st_data_t *keys, st_index_t size); +#define set_free_table rb_set_free_table +void rb_set_free_table(set_table *); +#define set_table_clear rb_set_table_clear +void rb_set_table_clear(set_table *); +#define set_copy rb_set_copy +set_table *rb_set_copy(set_table *new_table, set_table *old_table); +#define set_memsize rb_set_memsize +PUREFUNC(size_t rb_set_memsize(const set_table *)); +#define set_compact_table rb_set_compact_table +void set_compact_table(set_table *tab); + +#endif diff --git a/internal/signal.h b/internal/signal.h index 86fb54e949..904747e226 100644 --- a/internal/signal.h +++ b/internal/signal.h @@ -13,9 +13,13 @@ extern int ruby_enable_coredump; int rb_get_next_signal(void); +#ifdef POSIX_SIGNAL +void (*ruby_posix_signal(int, void (*)(int)))(int); +#endif + RUBY_SYMBOL_EXPORT_BEGIN /* signal.c (export) */ -int rb_grantpt(int fd); +void rb_signal_atfork(void); RUBY_SYMBOL_EXPORT_END #endif /* INTERNAL_SIGNAL_H */ diff --git a/internal/st.h b/internal/st.h new file mode 100644 index 0000000000..c220edd9f0 --- /dev/null +++ b/internal/st.h @@ -0,0 +1,11 @@ +#ifndef INTERNAL_ST_H +#define INTERNAL_ST_H + +#include "ruby/st.h" + +st_table *rb_st_replace(st_table *new_tab, st_table *old_tab); +#define st_replace rb_st_replace +st_table *rb_st_init_existing_table_with_size(st_table *tab, const struct st_hash_type *type, st_index_t size); +#define st_init_existing_table_with_size rb_st_init_existing_table_with_size + +#endif diff --git a/internal/string.h b/internal/string.h index 12edbff2b1..9212ce8986 100644 --- a/internal/string.h +++ b/internal/string.h @@ -14,15 +14,57 @@ #include "ruby/internal/stdbool.h" /* for bool */ #include "ruby/encoding.h" /* for rb_encoding */ #include "ruby/ruby.h" /* for VALUE */ +#include "encindex.h" -#define STR_NOEMBED FL_USER1 -#define STR_SHARED FL_USER2 /* = ELTS_SHARED */ +#define STR_SHARED FL_USER0 /* = ELTS_SHARED */ +#define STR_NOEMBED FL_USER1 +#define STR_CHILLED (FL_USER2 | FL_USER3) +#define STR_CHILLED_LITERAL FL_USER2 +#define STR_CHILLED_SYMBOL_TO_S FL_USER3 + +enum ruby_rstring_private_flags { + RSTRING_CHILLED = STR_CHILLED, +}; #ifdef rb_fstring_cstr # undef rb_fstring_cstr #endif +static inline bool +rb_str_encindex_fastpath(int encindex) +{ + // The overwhelming majority of strings are in one of these 3 encodings, + // which are all either ASCII or perfect ASCII supersets. + // Hence you can use fast, single byte algorithms on them, such as `memchr` etc, + // without all the overhead of fetching the rb_encoding and using functions such as + // rb_enc_mbminlen etc. + // Many other encodings could qualify, but they are expected to be rare occurences, + // so it's better to keep that list small. + switch (encindex) { + case ENCINDEX_ASCII_8BIT: + case ENCINDEX_UTF_8: + case ENCINDEX_US_ASCII: + return true; + default: + return false; + } +} + +static inline bool +rb_str_enc_fastpath(VALUE str) +{ + return rb_str_encindex_fastpath(ENCODING_GET_INLINED(str)); +} + +static inline rb_encoding * +rb_str_enc_get(VALUE str) +{ + RUBY_ASSERT(RB_TYPE_P(str, T_STRING)); + return rb_enc_from_index(ENCODING_GET(str)); +} + /* string.c */ +VALUE rb_str_dup_m(VALUE str); VALUE rb_fstring(VALUE); VALUE rb_fstring_cstr(const char *str); VALUE rb_fstring_enc_new(const char *ptr, long len, rb_encoding *enc); @@ -45,6 +87,21 @@ void rb_str_make_independent(VALUE str); int rb_enc_str_coderange_scan(VALUE str, rb_encoding *enc); int rb_ascii8bit_appendable_encoding_index(rb_encoding *enc, unsigned int code); VALUE rb_str_include(VALUE str, VALUE arg); +VALUE rb_str_byte_substr(VALUE str, VALUE beg, VALUE len); +VALUE rb_str_substr_two_fixnums(VALUE str, VALUE beg, VALUE len, int empty); +VALUE rb_str_tmp_frozen_no_embed_acquire(VALUE str); +void rb_str_make_embedded(VALUE); +VALUE rb_str_upto_each(VALUE, VALUE, int, int (*each)(VALUE, VALUE), VALUE); +size_t rb_str_size_as_embedded(VALUE); +bool rb_str_reembeddable_p(VALUE); +VALUE rb_str_upto_endless_each(VALUE, int (*each)(VALUE, VALUE), VALUE); +VALUE rb_str_with_debug_created_info(VALUE, VALUE, int); +VALUE rb_str_frozen_bare_string(VALUE); +const char *rb_str_null_check(VALUE); + +/* error.c */ +void rb_warn_unchilled_literal(VALUE str); +void rb_warn_unchilled_symbol_to_s(VALUE str); static inline bool STR_EMBED_P(VALUE str); static inline bool STR_SHARED_P(VALUE str); @@ -59,26 +116,22 @@ RUBY_SYMBOL_EXPORT_BEGIN VALUE rb_str_tmp_frozen_acquire(VALUE str); void rb_str_tmp_frozen_release(VALUE str, VALUE tmp); VALUE rb_setup_fake_str(struct RString *fake_str, const char *name, long len, rb_encoding *enc); -VALUE rb_str_upto_each(VALUE, VALUE, int, int (*each)(VALUE, VALUE), VALUE); -VALUE rb_str_upto_endless_each(VALUE, int (*each)(VALUE, VALUE), VALUE); -void rb_str_make_embedded(VALUE); -size_t rb_str_size_as_embedded(VALUE); -bool rb_str_reembeddable_p(VALUE); -void rb_str_update_shared_ary(VALUE str, VALUE old_root, VALUE new_root); RUBY_SYMBOL_EXPORT_END -MJIT_SYMBOL_EXPORT_BEGIN VALUE rb_fstring_new(const char *ptr, long len); +void rb_gc_free_fstring(VALUE obj); +bool rb_obj_is_fstring_table(VALUE obj); +void Init_fstring_table(); VALUE rb_obj_as_string_result(VALUE str, VALUE obj); VALUE rb_str_opt_plus(VALUE x, VALUE y); VALUE rb_str_concat_literals(size_t num, const VALUE *strary); VALUE rb_str_eql(VALUE str1, VALUE str2); VALUE rb_id_quote_unprintable(ID); VALUE rb_sym_proc_call(ID mid, int argc, const VALUE *argv, int kw_splat, VALUE passed_proc); +VALUE rb_enc_literal_str(const char *ptr, long len, rb_encoding *enc); struct rb_execution_context_struct; -VALUE rb_ec_str_resurrect(struct rb_execution_context_struct *ec, VALUE str); -MJIT_SYMBOL_EXPORT_END +VALUE rb_ec_str_resurrect(struct rb_execution_context_struct *ec, VALUE str, bool chilled); #define rb_fstring_lit(str) rb_fstring_new((str), rb_strlen_lit(str)) #define rb_fstring_literal(str) rb_fstring_lit(str) @@ -110,6 +163,29 @@ STR_SHARED_P(VALUE str) } static inline bool +CHILLED_STRING_P(VALUE obj) +{ + return RB_TYPE_P(obj, T_STRING) && FL_TEST_RAW(obj, STR_CHILLED); +} + +static inline void +CHILLED_STRING_MUTATED(VALUE str) +{ + VALUE chilled_reason = RB_FL_TEST_RAW(str, STR_CHILLED); + FL_UNSET_RAW(str, STR_CHILLED); + switch (chilled_reason) { + case STR_CHILLED_SYMBOL_TO_S: + rb_warn_unchilled_symbol_to_s(str); + break; + case STR_CHILLED_LITERAL: + rb_warn_unchilled_literal(str); + break; + default: + rb_bug("RString was chilled for multiple reasons"); + } +} + +static inline bool is_ascii_string(VALUE str) { return rb_enc_str_coderange(str) == ENC_CODERANGE_7BIT; @@ -121,6 +197,21 @@ is_broken_string(VALUE str) return rb_enc_str_coderange(str) == ENC_CODERANGE_BROKEN; } +static inline bool +at_char_boundary(const char *s, const char *p, const char *e, rb_encoding *enc) +{ + return rb_enc_left_char_head(s, p, e, enc) == p; +} + +static inline bool +at_char_right_boundary(const char *s, const char *p, const char *e, rb_encoding *enc) +{ + RUBY_ASSERT(s <= p); + RUBY_ASSERT(p <= e); + + return rb_enc_right_char_head(s, p, e, enc) == p; +} + /* expect tail call optimization */ // YJIT needs this function to never allocate and never raise static inline VALUE diff --git a/internal/struct.h b/internal/struct.h index 8acc00ec3c..d3c8157393 100644 --- a/internal/struct.h +++ b/internal/struct.h @@ -9,14 +9,25 @@ * @brief Internal header for Struct. */ #include "ruby/internal/stdbool.h" /* for bool */ -#include "internal/gc.h" /* for RB_OBJ_WRITE */ #include "ruby/ruby.h" /* for struct RBasic */ +/* Flags of RStruct + * + * 1-7: RSTRUCT_EMBED_LEN + * If non-zero, the struct is embedded (its contents follow the + * header, rather than being on a separately allocated buffer) and + * these bits are the length of the Struct. + * 8: RSTRUCT_GEN_FIELDS + * The struct is embedded and has no space left to store the + * IMEMO/fields reference. Any ivar this struct may have will be in + * the generic_fields_tbl. This flag doesn't imply the struct has + * ivars. + */ enum { - RSTRUCT_EMBED_LEN_MAX = RVALUE_EMBED_LEN_MAX, - RSTRUCT_EMBED_LEN_MASK = (RUBY_FL_USER2|RUBY_FL_USER1), + RSTRUCT_EMBED_LEN_MASK = RUBY_FL_USER7 | RUBY_FL_USER6 | RUBY_FL_USER5 | RUBY_FL_USER4 | + RUBY_FL_USER3 | RUBY_FL_USER2 | RUBY_FL_USER1, RSTRUCT_EMBED_LEN_SHIFT = (RUBY_FL_USHIFT+1), - RSTRUCT_TRANSIENT_FLAG = FL_USER3, + RSTRUCT_GEN_FIELDS = RUBY_FL_USER8, }; struct RStruct { @@ -25,73 +36,29 @@ struct RStruct { struct { long len; const VALUE *ptr; + VALUE fields_obj; } heap; - const VALUE ary[RSTRUCT_EMBED_LEN_MAX]; + /* This is a length 1 array because: + * 1. GCC has a bug that does not optimize C flexible array members + * (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=102452) + * 2. Zero length arrays are not supported by all compilers + */ + const VALUE ary[1]; } as; }; #define RSTRUCT(obj) ((struct RStruct *)(obj)) -#ifdef RSTRUCT_LEN -# undef RSTRUCT_LEN -#endif - -#ifdef RSTRUCT_PTR -# undef RSTRUCT_PTR -#endif - -#ifdef RSTRUCT_SET -# undef RSTRUCT_SET -#endif - -#ifdef RSTRUCT_GET -# undef RSTRUCT_GET -#endif - -#define RSTRUCT_LEN internal_RSTRUCT_LEN -#define RSTRUCT_SET internal_RSTRUCT_SET -#define RSTRUCT_GET internal_RSTRUCT_GET - /* struct.c */ VALUE rb_struct_init_copy(VALUE copy, VALUE s); VALUE rb_struct_lookup(VALUE s, VALUE idx); VALUE rb_struct_s_keyword_init(VALUE klass); -static inline const VALUE *rb_struct_const_heap_ptr(VALUE st); -static inline bool RSTRUCT_TRANSIENT_P(VALUE st); -static inline void RSTRUCT_TRANSIENT_SET(VALUE st); -static inline void RSTRUCT_TRANSIENT_UNSET(VALUE st); static inline long RSTRUCT_EMBED_LEN(VALUE st); -static inline long RSTRUCT_LEN(VALUE st); +static inline long RSTRUCT_LEN_RAW(VALUE st); static inline int RSTRUCT_LENINT(VALUE st); static inline const VALUE *RSTRUCT_CONST_PTR(VALUE st); -static inline void RSTRUCT_SET(VALUE st, long k, VALUE v); -static inline VALUE RSTRUCT_GET(VALUE st, long k); - -static inline bool -RSTRUCT_TRANSIENT_P(VALUE st) -{ -#if USE_TRANSIENT_HEAP - return FL_TEST_RAW(st, RSTRUCT_TRANSIENT_FLAG); -#else - return false; -#endif -} - -static inline void -RSTRUCT_TRANSIENT_SET(VALUE st) -{ -#if USE_TRANSIENT_HEAP - FL_SET_RAW(st, RSTRUCT_TRANSIENT_FLAG); -#endif -} - -static inline void -RSTRUCT_TRANSIENT_UNSET(VALUE st) -{ -#if USE_TRANSIENT_HEAP - FL_UNSET_RAW(st, RSTRUCT_TRANSIENT_FLAG); -#endif -} +static inline void RSTRUCT_SET_RAW(VALUE st, long k, VALUE v); +static inline VALUE RSTRUCT_GET_RAW(VALUE st, long k); static inline long RSTRUCT_EMBED_LEN(VALUE st) @@ -102,7 +69,7 @@ RSTRUCT_EMBED_LEN(VALUE st) } static inline long -RSTRUCT_LEN(VALUE st) +RSTRUCT_LEN_RAW(VALUE st) { if (FL_TEST_RAW(st, RSTRUCT_EMBED_LEN_MASK)) { return RSTRUCT_EMBED_LEN(st); @@ -115,7 +82,7 @@ RSTRUCT_LEN(VALUE st) static inline int RSTRUCT_LENINT(VALUE st) { - return rb_long2int(RSTRUCT_LEN(st)); + return rb_long2int(RSTRUCT_LEN_RAW(st)); } static inline const VALUE * @@ -132,22 +99,42 @@ RSTRUCT_CONST_PTR(VALUE st) } static inline void -RSTRUCT_SET(VALUE st, long k, VALUE v) +RSTRUCT_SET_RAW(VALUE st, long k, VALUE v) { RB_OBJ_WRITE(st, &RSTRUCT_CONST_PTR(st)[k], v); } static inline VALUE -RSTRUCT_GET(VALUE st, long k) +RSTRUCT_GET_RAW(VALUE st, long k) { return RSTRUCT_CONST_PTR(st)[k]; } -static inline const VALUE * -rb_struct_const_heap_ptr(VALUE st) +static inline VALUE +RSTRUCT_FIELDS_OBJ(VALUE st) { - /* TODO: check embed on debug mode */ - return RSTRUCT(st)->as.heap.ptr; + const long embed_len = RSTRUCT_EMBED_LEN(st); + VALUE fields_obj; + if (embed_len) { + RUBY_ASSERT(!FL_TEST_RAW(st, RSTRUCT_GEN_FIELDS)); + fields_obj = RSTRUCT_GET_RAW(st, embed_len); + } + else { + fields_obj = RSTRUCT(st)->as.heap.fields_obj; + } + return fields_obj; } +static inline void +RSTRUCT_SET_FIELDS_OBJ(VALUE st, VALUE fields_obj) +{ + const long embed_len = RSTRUCT_EMBED_LEN(st); + if (embed_len) { + RUBY_ASSERT(!FL_TEST_RAW(st, RSTRUCT_GEN_FIELDS)); + RSTRUCT_SET_RAW(st, embed_len, fields_obj); + } + else { + RB_OBJ_WRITE(st, &RSTRUCT(st)->as.heap.fields_obj, fields_obj); + } +} #endif /* INTERNAL_STRUCT_H */ diff --git a/internal/symbol.h b/internal/symbol.h index 30c81ea004..b9109b1347 100644 --- a/internal/symbol.h +++ b/internal/symbol.h @@ -17,6 +17,7 @@ #endif /* symbol.c */ +void rb_sym_global_symbols_mark_and_move(void); VALUE rb_to_symbol_type(VALUE obj); VALUE rb_sym_intern(const char *ptr, long len, rb_encoding *enc); VALUE rb_sym_intern_ascii(const char *ptr, long len); @@ -29,8 +30,11 @@ PUREFUNC(int rb_is_const_sym(VALUE sym)); PUREFUNC(int rb_is_attrset_sym(VALUE sym)); ID rb_make_internal_id(void); ID rb_make_temporary_id(size_t n); +bool rb_obj_is_symbol_table(VALUE obj); +void rb_sym_global_symbol_table_foreach_weak_reference(int (*callback)(VALUE *key, void *data), void *data); void rb_gc_free_dsymbol(VALUE); int rb_static_id_valid_p(ID id); +void rb_free_global_symbol_table(void); #if __has_builtin(__builtin_constant_p) #define rb_sym_intern_ascii_cstr(ptr) \ diff --git a/internal/thread.h b/internal/thread.h index c3e54de683..ea891b4372 100644 --- a/internal/thread.h +++ b/internal/thread.h @@ -10,8 +10,17 @@ */ #include "ruby/ruby.h" /* for VALUE */ #include "ruby/intern.h" /* for rb_blocking_function_t */ +#include "ccan/list/list.h" /* for list in rb_io_close_wait_list */ struct rb_thread_struct; /* in vm_core.h */ +struct rb_io; + +#define RB_VM_SAVE_MACHINE_CONTEXT(th) \ + do { \ + FLUSH_REGISTER_WINDOWS; \ + setjmp((th)->ec->machine.regs); \ + SET_MACHINE_STACK_END(&(th)->ec->machine.stack_end); \ + } while (0) /* thread.c */ #define COVERAGE_INDEX_LINES 0 @@ -22,6 +31,10 @@ struct rb_thread_struct; /* in vm_core.h */ #define COVERAGE_TARGET_ONESHOT_LINES 8 #define COVERAGE_TARGET_EVAL 16 +#define RUBY_FATAL_THREAD_KILLED INT2FIX(0) +#define RUBY_FATAL_THREAD_TERMINATED INT2FIX(1) +#define RUBY_FATAL_FIBER_KILLED RB_INT2FIX(2) + VALUE rb_obj_is_mutex(VALUE obj); VALUE rb_suppress_tracing(VALUE (*func)(VALUE), VALUE arg); void rb_thread_execute_interrupts(VALUE th); @@ -34,23 +47,66 @@ VALUE rb_thread_shield_wait(VALUE self); VALUE rb_thread_shield_release(VALUE self); VALUE rb_thread_shield_destroy(VALUE self); int rb_thread_to_be_killed(VALUE thread); +void rb_thread_acquire_fork_lock(void); +void rb_thread_release_fork_lock(void); +void rb_thread_reset_fork_lock(void); void rb_mutex_allow_trap(VALUE self, int val); VALUE rb_uninterruptible(VALUE (*b_proc)(VALUE), VALUE data); VALUE rb_mutex_owned_p(VALUE self); VALUE rb_exec_recursive_outer_mid(VALUE (*f)(VALUE g, VALUE h, int r), VALUE g, VALUE h, ID mid); +void ruby_mn_threads_params(void); + +int rb_thread_io_wait(struct rb_thread_struct *th, struct rb_io *io, int events, struct timeval * timeout); +int rb_thread_wait_for_single_fd(struct rb_thread_struct *th, int fd, int events, struct timeval * timeout); + +size_t rb_thread_io_close_interrupt(struct rb_io *); +void rb_thread_io_close_wait(struct rb_io *); + +void rb_ec_check_ints(struct rb_execution_context_struct *ec); -int rb_thread_wait_for_single_fd(int fd, int events, struct timeval * timeout); +void rb_thread_free_native_thread(void *th_ptr); RUBY_SYMBOL_EXPORT_BEGIN + +void *rb_thread_prevent_fork(void *(*func)(void *), void *data); /* for ext/socket/raddrinfo.c */ + /* Temporary. This API will be removed (renamed). */ -VALUE rb_thread_io_blocking_region(rb_blocking_function_t *func, void *data1, int fd); +VALUE rb_thread_io_blocking_region(struct rb_io *io, rb_blocking_function_t *func, void *data1); +VALUE rb_thread_io_blocking_call(struct rb_io *io, rb_blocking_function_t *func, void *data1, int events); + +// Invoke the given function, with the specified argument, in a way that `IO#close` from another execution context can interrupt it. +VALUE rb_thread_io_blocking_operation(VALUE self, VALUE(*function)(VALUE), VALUE argument); /* thread.c (export) */ int ruby_thread_has_gvl_p(void); /* for ext/fiddle/closure.c */ + RUBY_SYMBOL_EXPORT_END -MJIT_SYMBOL_EXPORT_BEGIN int rb_threadptr_execute_interrupts(struct rb_thread_struct *th, int blocking_timing); -MJIT_SYMBOL_EXPORT_END +bool rb_thread_mn_schedulable(VALUE thread); + +bool rb_thread_resolve_unblock_function(rb_unblock_function_t **unblock_function, void **data2, struct rb_thread_struct *thread); + +// interrupt exec + +typedef VALUE (rb_interrupt_exec_func_t)(void *data); + +enum rb_interrupt_exec_flag { + rb_interrupt_exec_flag_none = 0x00, + rb_interrupt_exec_flag_value_data = 0x01, + rb_interrupt_exec_flag_new_thread = 0x02, +}; + +// interrupt the target_th and run func. +struct rb_ractor_struct; + +void rb_threadptr_interrupt_exec(struct rb_thread_struct *target_th, + rb_interrupt_exec_func_t *func, void *data, enum rb_interrupt_exec_flag flags); + +// create a thread in the target_r and run func on the created thread. +void rb_ractor_interrupt_exec(struct rb_ractor_struct *target_r, + rb_interrupt_exec_func_t *func, void *data, enum rb_interrupt_exec_flag flags); + +void rb_threadptr_interrupt_exec_task_mark(struct rb_thread_struct *th); #endif /* INTERNAL_THREAD_H */ diff --git a/internal/time.h b/internal/time.h index a3bf0587ec..1f3505f5bc 100644 --- a/internal/time.h +++ b/internal/time.h @@ -27,8 +27,8 @@ struct timeval rb_time_timeval(VALUE); RUBY_SYMBOL_EXPORT_BEGIN /* time.c (export) */ -void ruby_reset_leap_second_info(void); -void ruby_reset_timezone(void); RUBY_SYMBOL_EXPORT_END +void ruby_reset_timezone(const char *); + #endif /* INTERNAL_TIME_H */ diff --git a/internal/transcode.h b/internal/transcode.h index 9922332ea9..ce4f2341be 100644 --- a/internal/transcode.h +++ b/internal/transcode.h @@ -17,4 +17,7 @@ extern VALUE rb_cEncodingConverter; size_t rb_econv_memsize(rb_econv_t *); +/* vm.c */ +void rb_free_transcoder_table(void); + #endif /* INTERNAL_TRANSCODE_H */ diff --git a/internal/variable.h b/internal/variable.h index 6dec6a6759..ca5e189c90 100644 --- a/internal/variable.h +++ b/internal/variable.h @@ -13,78 +13,62 @@ #include "constant.h" /* for rb_const_entry_t */ #include "ruby/internal/stdbool.h" /* for bool */ #include "ruby/ruby.h" /* for VALUE */ -#include "shape.h" /* for rb_shape_t */ - -/* global variable */ - -#define ROBJECT_TRANSIENT_FLAG FL_USER2 +#include "shape.h" /* for shape_id_t */ /* variable.c */ void rb_gc_mark_global_tbl(void); void rb_gc_update_global_tbl(void); -size_t rb_generic_ivar_memsize(VALUE); VALUE rb_search_class_path(VALUE); VALUE rb_attr_delete(VALUE, ID); void rb_autoload_str(VALUE mod, ID id, VALUE file); VALUE rb_autoload_at_p(VALUE, ID, int); +void rb_autoload_copy_table_for_box(st_table *, const rb_box_t *); NORETURN(VALUE rb_mod_const_missing(VALUE,VALUE)); rb_gvar_getter_t *rb_gvar_getter_function_of(ID); rb_gvar_setter_t *rb_gvar_setter_function_of(ID); void rb_gvar_readonly_setter(VALUE v, ID id, VALUE *_); void rb_gvar_ractor_local(const char *name); -static inline bool ROBJ_TRANSIENT_P(VALUE obj); -static inline void ROBJ_TRANSIENT_SET(VALUE obj); -static inline void ROBJ_TRANSIENT_UNSET(VALUE obj); +void rb_gvar_box_ready(const char *name); + +/** + * Sets the name of a module. + * + * Non-permanently named classes can have a temporary name assigned (or + * cleared). In that case the name will be used for `#inspect` and `#to_s`, and + * nested classes/modules will be named with the temporary name as a prefix. + * + * After the module is assigned to a constant, the temporary name will be + * discarded, and the name will be computed based on the nesting. + * + * @param[in] mod An instance of ::rb_cModule. + * @param[in] name An instance of ::rb_cString. + * @retval mod + */ +VALUE rb_mod_set_temporary_name(VALUE, VALUE); -struct gen_ivtbl; -int rb_gen_ivtbl_get(VALUE obj, ID id, struct gen_ivtbl **ivtbl); -int rb_obj_evacuate_ivs_to_hash_table(ID key, VALUE val, st_data_t arg); +void rb_obj_copy_ivs_to_hash_table(VALUE obj, st_table *table); +void rb_obj_init_too_complex(VALUE obj, st_table *table); +void rb_evict_ivars_to_hash(VALUE obj); +VALUE rb_obj_field_get(VALUE obj, shape_id_t target_shape_id); +void rb_ivar_set_internal(VALUE obj, ID id, VALUE val); +attr_index_t rb_ivar_set_index(VALUE obj, ID id, VALUE val); +attr_index_t rb_obj_field_set(VALUE obj, shape_id_t target_shape_id, ID field_name, VALUE val); +VALUE rb_ivar_get_at(VALUE obj, attr_index_t index, ID id); +VALUE rb_ivar_get_at_no_ractor_check(VALUE obj, attr_index_t index); RUBY_SYMBOL_EXPORT_BEGIN /* variable.c (export) */ -void rb_mark_generic_ivar(VALUE); -void rb_mv_generic_ivar(VALUE src, VALUE dst); +void rb_mark_generic_ivar(VALUE obj); VALUE rb_const_missing(VALUE klass, VALUE name); -int rb_class_ivar_set(VALUE klass, ID vid, VALUE value); -void rb_iv_tbl_copy(VALUE dst, VALUE src); +bool rb_class_ivar_set(VALUE klass, ID vid, VALUE value); +void rb_fields_tbl_copy(VALUE dst, VALUE src); RUBY_SYMBOL_EXPORT_END -MJIT_SYMBOL_EXPORT_BEGIN VALUE rb_ivar_lookup(VALUE obj, ID id, VALUE undef); VALUE rb_gvar_get(ID); VALUE rb_gvar_set(ID, VALUE); VALUE rb_gvar_defined(ID); void rb_const_warn_if_deprecated(const rb_const_entry_t *, VALUE, ID); -rb_shape_t * rb_grow_iv_list(VALUE obj); -void rb_ensure_iv_list_size(VALUE obj, uint32_t len, uint32_t newsize); -struct gen_ivtbl *rb_ensure_generic_iv_list_size(VALUE obj, rb_shape_t *shape, uint32_t newsize); -attr_index_t rb_obj_ivar_set(VALUE obj, ID id, VALUE val); -MJIT_SYMBOL_EXPORT_END - -static inline bool -ROBJ_TRANSIENT_P(VALUE obj) -{ -#if USE_TRANSIENT_HEAP - return FL_TEST_RAW(obj, ROBJECT_TRANSIENT_FLAG); -#else - return false; -#endif -} - -static inline void -ROBJ_TRANSIENT_SET(VALUE obj) -{ -#if USE_TRANSIENT_HEAP - FL_SET_RAW(obj, ROBJECT_TRANSIENT_FLAG); -#endif -} - -static inline void -ROBJ_TRANSIENT_UNSET(VALUE obj) -{ -#if USE_TRANSIENT_HEAP - FL_UNSET_RAW(obj, ROBJECT_TRANSIENT_FLAG); -#endif -} +void rb_ensure_iv_list_size(VALUE obj, uint32_t current_len, uint32_t newsize); #endif /* INTERNAL_VARIABLE_H */ diff --git a/internal/vm.h b/internal/vm.h index cf245c6579..029b19d555 100644 --- a/internal/vm.h +++ b/internal/vm.h @@ -45,60 +45,64 @@ VALUE rb_vm_push_frame_fname(struct rb_execution_context_struct *ec, VALUE fname /* vm.c */ VALUE rb_obj_is_thread(VALUE obj); void rb_vm_mark(void *ptr); +void rb_vm_register_global_object(VALUE obj); void rb_vm_each_stack_value(void *ptr, void (*cb)(VALUE, void*), void *ctx); PUREFUNC(VALUE rb_vm_top_self(void)); const void **rb_vm_get_insns_address_table(void); VALUE rb_source_location(int *pline); const char *rb_source_location_cstr(int *pline); -MJIT_STATIC void rb_vm_pop_cfunc_frame(void); -int rb_vm_add_root_module(VALUE module); +void rb_vm_pop_cfunc_frame(void); void rb_vm_check_redefinition_by_prepend(VALUE klass); int rb_vm_check_optimizable_mid(VALUE mid); VALUE rb_yield_refine_block(VALUE refinement, VALUE refinements); -MJIT_STATIC VALUE ruby_vm_special_exception_copy(VALUE); -PUREFUNC(st_table *rb_vm_fstring_table(void)); +VALUE ruby_vm_special_exception_copy(VALUE); -MJIT_SYMBOL_EXPORT_BEGIN -VALUE vm_exec(struct rb_execution_context_struct *, bool); /* used in JIT-ed code */ -MJIT_SYMBOL_EXPORT_END +void rb_lastline_set_up(VALUE val, unsigned int up); /* vm_eval.c */ VALUE rb_current_realfilepath(void); VALUE rb_check_block_call(VALUE, ID, int, const VALUE *, rb_block_call_func_t, VALUE); typedef void rb_check_funcall_hook(int, VALUE, ID, int, const VALUE *, VALUE); -VALUE rb_check_funcall_with_hook(VALUE recv, ID mid, int argc, const VALUE *argv, - rb_check_funcall_hook *hook, VALUE arg); VALUE rb_check_funcall_with_hook_kw(VALUE recv, ID mid, int argc, const VALUE *argv, rb_check_funcall_hook *hook, VALUE arg, int kw_splat); const char *rb_type_str(enum ruby_value_type type); VALUE rb_check_funcall_default(VALUE, ID, int, const VALUE *, VALUE); VALUE rb_check_funcall_basic_kw(VALUE, ID, VALUE, int, const VALUE*, int); VALUE rb_yield_1(VALUE val); +VALUE rb_ec_yield(struct rb_execution_context_struct *ec, VALUE val); VALUE rb_yield_force_blockarg(VALUE values); VALUE rb_lambda_call(VALUE obj, ID mid, int argc, const VALUE *argv, rb_block_call_func_t bl_proc, int min_argc, int max_argc, VALUE data2); void rb_check_stack_overflow(void); +#define RB_BLOCK_NO_USE_PACKED_ARGS 2 +VALUE rb_block_call2(VALUE obj, ID mid, int argc, const VALUE *argv, rb_block_call_func_t bl_proc, VALUE data2, long flags); +struct vm_ifunc *rb_current_ifunc(void); +VALUE rb_gccct_clear_table(VALUE); +VALUE rb_eval_cmd_call_kw(VALUE cmd, int argc, const VALUE *argv, int kw_splat); + +#if USE_YJIT || USE_ZJIT +/* vm_exec.c */ +extern uint64_t rb_vm_insn_count; +#endif + +extern bool rb_free_at_exit; + +/* miniinit.c and builtin.c */ +void rb_free_loaded_builtin_table(void); /* vm_insnhelper.c */ VALUE rb_equal_opt(VALUE obj1, VALUE obj2); VALUE rb_eql_opt(VALUE obj1, VALUE obj2); struct rb_iseq_struct; -MJIT_SYMBOL_EXPORT_BEGIN const struct rb_callcache *rb_vm_search_method_slowpath(const struct rb_callinfo *ci, VALUE klass); -MJIT_SYMBOL_EXPORT_END /* vm_method.c */ -struct rb_execution_context_struct; -MJIT_SYMBOL_EXPORT_BEGIN int rb_ec_obj_respond_to(struct rb_execution_context_struct *ec, VALUE obj, ID id, int priv); -MJIT_SYMBOL_EXPORT_END - -void rb_clear_constant_cache(void); /* vm_dump.c */ -void rb_print_backtrace(void); +void rb_print_backtrace(FILE *); /* vm_backtrace.c */ VALUE rb_vm_thread_backtrace(int argc, const VALUE *argv, VALUE thval); @@ -106,19 +110,17 @@ VALUE rb_vm_thread_backtrace_locations(int argc, const VALUE *argv, VALUE thval) VALUE rb_vm_backtrace(int argc, const VALUE * argv, struct rb_execution_context_struct * ec); VALUE rb_vm_backtrace_locations(int argc, const VALUE * argv, struct rb_execution_context_struct * ec); VALUE rb_make_backtrace(void); -void rb_backtrace_print_as_bugreport(void); +void rb_backtrace_print_as_bugreport(FILE*); int rb_backtrace_p(VALUE obj); VALUE rb_backtrace_to_str_ary(VALUE obj); VALUE rb_backtrace_to_location_ary(VALUE obj); +VALUE rb_location_ary_to_backtrace(VALUE ary); void rb_backtrace_each(VALUE (*iter)(VALUE recv, VALUE str), VALUE output); int rb_frame_info_p(VALUE obj); int rb_get_node_id_from_frame_info(VALUE obj); const struct rb_iseq_struct *rb_get_iseq_from_frame_info(VALUE obj); -MJIT_SYMBOL_EXPORT_BEGIN VALUE rb_ec_backtrace_object(const struct rb_execution_context_struct *ec); -void rb_backtrace_use_iseq_first_lineno_for_last_location(VALUE self); -MJIT_SYMBOL_EXPORT_END #define RUBY_DTRACE_CREATE_HOOK(name, arg) \ RUBY_DTRACE_HOOK(name##_CREATE, arg) |