diff options
Diffstat (limited to 'string.c')
| -rw-r--r-- | string.c | 14615 |
1 files changed, 11461 insertions, 3154 deletions
@@ -3,218 +3,1716 @@ string.c - $Author$ - $Date$ created at: Mon Aug 9 17:12:58 JST 1993 - Copyright (C) 1993-2003 Yukihiro Matsumoto + Copyright (C) 1993-2007 Yukihiro Matsumoto Copyright (C) 2000 Network Applied Communication Laboratory, Inc. Copyright (C) 2000 Information-technology Promotion Agency, Japan **********************************************************************/ -#include "ruby.h" -#include "re.h" -#include "version.h" +#include "ruby/internal/config.h" -#define BEG(no) regs->beg[no] -#define END(no) regs->end[no] - -#include <math.h> #include <ctype.h> +#include <errno.h> +#include <math.h> #ifdef HAVE_UNISTD_H -#include <unistd.h> +# include <unistd.h> +#endif + +#include "debug_counter.h" +#include "encindex.h" +#include "id.h" +#include "internal.h" +#include "internal/array.h" +#include "internal/compar.h" +#include "internal/compilers.h" +#include "internal/concurrent_set.h" +#include "internal/encoding.h" +#include "internal/error.h" +#include "internal/gc.h" +#include "internal/hash.h" +#include "internal/numeric.h" +#include "internal/object.h" +#include "internal/proc.h" +#include "internal/re.h" +#include "internal/sanitizers.h" +#include "internal/string.h" +#include "internal/transcode.h" +#include "probes.h" +#include "ruby/encoding.h" +#include "ruby/re.h" +#include "ruby/thread.h" +#include "ruby/util.h" +#include "ruby/ractor.h" +#include "ruby_assert.h" +#include "shape.h" +#include "vm_sync.h" +#include "ruby/internal/attr/nonstring.h" + +#if defined HAVE_CRYPT_R +# if defined HAVE_CRYPT_H +# include <crypt.h> +# endif +#elif !defined HAVE_CRYPT +# include "missing/crypt.h" +# define HAVE_CRYPT_R 1 #endif +#define BEG(no) (regs->beg[(no)]) +#define END(no) (regs->end[(no)]) + +#undef rb_str_new +#undef rb_usascii_str_new +#undef rb_utf8_str_new +#undef rb_enc_str_new +#undef rb_str_new_cstr +#undef rb_usascii_str_new_cstr +#undef rb_utf8_str_new_cstr +#undef rb_enc_str_new_cstr +#undef rb_external_str_new_cstr +#undef rb_locale_str_new_cstr +#undef rb_str_dup_frozen +#undef rb_str_buf_new_cstr +#undef rb_str_buf_cat +#undef rb_str_buf_cat2 +#undef rb_str_cat2 +#undef rb_str_cat_cstr +#undef rb_fstring_cstr + VALUE rb_cString; +VALUE rb_cSymbol; + +/* Flags of RString + * + * 0: STR_SHARED (equal to ELTS_SHARED) + * The string is shared. The buffer this string points to is owned by + * another string (the shared root). + * 1: RSTRING_NOEMBED + * The string is not embedded. When a string is embedded, the contents + * follow the header. When a string is not embedded, the contents is + * on a separately allocated buffer. + * 2: STR_CHILLED_LITERAL (will be frozen in a future version) + * The string was allocated as a literal in a file without an explicit `frozen_string_literal` comment. + * It emits a deprecation warning when mutated for the first time. + * 3: STR_CHILLED_SYMBOL_TO_S (will be frozen in a future version) + * The string was allocated by the `Symbol#to_s` method. + * It emits a deprecation warning when mutated for the first time. + * 4: STR_PRECOMPUTED_HASH + * The string is embedded and has its precomputed hashcode stored + * after the terminator. + * 5: STR_SHARED_ROOT + * Other strings may point to the contents of this string. When this + * flag is set, STR_SHARED must not be set. + * 6: STR_BORROWED + * When RSTRING_NOEMBED is set and klass is 0, this string is unsafe + * to be unshared by rb_str_tmp_frozen_release. + * 7: STR_TMPLOCK + * The pointer to the buffer is passed to a system call such as + * read(2). Any modification and realloc is prohibited. + * 8-9: ENC_CODERANGE + * Stores the coderange of the string. + * 10-16: ENCODING + * Stores the encoding of the string. + * 17: RSTRING_FSTR + * The string is a fstring. The string is deduplicated in the fstring + * table. + * 18: STR_NOFREE + * Do not free this string's buffer when the string is reclaimed + * by the garbage collector. Used for when the string buffer is a C + * string literal. + * 19: STR_FAKESTR + * The string is not allocated or managed by the garbage collector. + * Typically, the string object header (struct RString) is temporarily + * allocated on C stack. + */ + +#define RUBY_MAX_CHAR_LEN 16 +#define STR_PRECOMPUTED_HASH FL_USER4 +#define STR_SHARED_ROOT FL_USER5 +#define STR_BORROWED FL_USER6 +#define STR_TMPLOCK FL_USER7 +#define STR_NOFREE FL_USER18 +#define STR_FAKESTR FL_USER19 + +#define STR_SET_NOEMBED(str) do {\ + FL_SET((str), STR_NOEMBED);\ + FL_UNSET((str), STR_SHARED | STR_SHARED_ROOT | STR_BORROWED);\ +} while (0) +#define STR_SET_EMBED(str) FL_UNSET((str), STR_NOEMBED | STR_SHARED | STR_NOFREE) + +#define STR_SET_LEN(str, n) do { \ + RSTRING(str)->len = (n); \ +} while (0) -#define STR_ASSOC FL_USER3 +#define TERM_LEN(str) (rb_str_enc_fastpath(str) ? 1 : rb_enc_mbminlen(rb_enc_from_index(ENCODING_GET(str)))) +#define TERM_FILL(ptr, termlen) do {\ + char *const term_fill_ptr = (ptr);\ + const int term_fill_len = (termlen);\ + *term_fill_ptr = '\0';\ + if (UNLIKELY(term_fill_len > 1))\ + memset(term_fill_ptr, 0, term_fill_len);\ +} while (0) #define RESIZE_CAPA(str,capacity) do {\ - REALLOC_N(RSTRING(str)->ptr, char, (capacity)+1);\ - RSTRING(str)->aux.capa = (capacity);\ + const int termlen = TERM_LEN(str);\ + RESIZE_CAPA_TERM(str,capacity,termlen);\ +} while (0) +#define RESIZE_CAPA_TERM(str,capacity,termlen) do {\ + if (STR_EMBED_P(str)) {\ + if (str_embed_capa(str) < capacity + termlen) {\ + char *const tmp = ALLOC_N(char, (size_t)(capacity) + (termlen));\ + const long tlen = RSTRING_LEN(str);\ + memcpy(tmp, RSTRING_PTR(str), tlen);\ + RSTRING(str)->as.heap.ptr = tmp;\ + RSTRING(str)->len = tlen;\ + STR_SET_NOEMBED(str);\ + RSTRING(str)->as.heap.aux.capa = (capacity);\ + }\ + }\ + else {\ + RUBY_ASSERT(!FL_TEST((str), STR_SHARED)); \ + SIZED_REALLOC_N(RSTRING(str)->as.heap.ptr, char, \ + (size_t)(capacity) + (termlen), STR_HEAP_SIZE(str)); \ + RSTRING(str)->as.heap.aux.capa = (capacity);\ + }\ } while (0) -VALUE rb_fs; +#define STR_SET_SHARED(str, shared_str) do { \ + if (!FL_TEST(str, STR_FAKESTR)) { \ + RUBY_ASSERT(RSTRING_PTR(shared_str) <= RSTRING_PTR(str)); \ + RUBY_ASSERT(RSTRING_PTR(str) <= RSTRING_PTR(shared_str) + RSTRING_LEN(shared_str)); \ + RB_OBJ_WRITE((str), &RSTRING(str)->as.heap.aux.shared, (shared_str)); \ + FL_SET((str), STR_SHARED); \ + rb_gc_register_pinning_obj(str); \ + FL_SET((shared_str), STR_SHARED_ROOT); \ + if (RBASIC_CLASS((shared_str)) == 0) /* for CoW-friendliness */ \ + FL_SET_RAW((shared_str), STR_BORROWED); \ + } \ +} while (0) + +#define STR_HEAP_PTR(str) (RSTRING(str)->as.heap.ptr) +#define STR_HEAP_SIZE(str) ((size_t)RSTRING(str)->as.heap.aux.capa + TERM_LEN(str)) +/* TODO: include the terminator size in capa. */ + +#define STR_ENC_GET(str) get_encoding(str) + +#if !defined SHARABLE_MIDDLE_SUBSTRING +# define SHARABLE_MIDDLE_SUBSTRING 0 +#endif +#if !SHARABLE_MIDDLE_SUBSTRING +#define SHARABLE_SUBSTRING_P(beg, len, end) ((beg) + (len) == (end)) +#else +#define SHARABLE_SUBSTRING_P(beg, len, end) 1 +#endif + + +static inline long +str_embed_capa(VALUE str) +{ + return rb_gc_obj_slot_size(str) - offsetof(struct RString, as.embed.ary); +} + +bool +rb_str_reembeddable_p(VALUE str) +{ + return !FL_TEST(str, STR_NOFREE|STR_SHARED_ROOT|STR_SHARED); +} + +static inline size_t +rb_str_embed_size(long capa, long termlen) +{ + size_t size = offsetof(struct RString, as.embed.ary) + capa + termlen; + if (size < sizeof(struct RString)) size = sizeof(struct RString); + return size; +} + +size_t +rb_str_size_as_embedded(VALUE str) +{ + size_t real_size; + if (STR_EMBED_P(str)) { + size_t capa = RSTRING(str)->len; + if (FL_TEST_RAW(str, STR_PRECOMPUTED_HASH)) capa += sizeof(st_index_t); + + real_size = rb_str_embed_size(capa, TERM_LEN(str)); + } + /* if the string is not currently embedded, but it can be embedded, how + * much space would it require */ + else if (rb_str_reembeddable_p(str)) { + size_t capa = RSTRING(str)->as.heap.aux.capa; + if (FL_TEST_RAW(str, STR_PRECOMPUTED_HASH)) capa += sizeof(st_index_t); + + real_size = rb_str_embed_size(capa, TERM_LEN(str)); + } + else { + real_size = sizeof(struct RString); + } + + return real_size; +} + +static inline bool +STR_EMBEDDABLE_P(long len, long termlen) +{ + return rb_gc_size_allocatable_p(rb_str_embed_size(len, termlen)); +} + +static VALUE str_replace_shared_without_enc(VALUE str2, VALUE str); +static VALUE str_new_frozen(VALUE klass, VALUE orig); +static VALUE str_new_frozen_buffer(VALUE klass, VALUE orig, int copy_encoding); +static VALUE str_new_static(VALUE klass, const char *ptr, long len, int encindex); +static VALUE str_new(VALUE klass, const char *ptr, long len); +static void str_make_independent_expand(VALUE str, long len, long expand, const int termlen); +static inline void str_modifiable(VALUE str); +static VALUE rb_str_downcase(int argc, VALUE *argv, VALUE str); +static inline VALUE str_alloc_embed(VALUE klass, size_t capa); + +static inline void +str_make_independent(VALUE str) +{ + long len = RSTRING_LEN(str); + int termlen = TERM_LEN(str); + str_make_independent_expand((str), len, 0L, termlen); +} + +static inline int str_dependent_p(VALUE str); + +void +rb_str_make_independent(VALUE str) +{ + if (str_dependent_p(str)) { + str_make_independent(str); + } +} + +void +rb_str_make_embedded(VALUE str) +{ + RUBY_ASSERT(rb_str_reembeddable_p(str)); + RUBY_ASSERT(!STR_EMBED_P(str)); + + char *buf = RSTRING(str)->as.heap.ptr; + long len = RSTRING(str)->len; + + STR_SET_EMBED(str); + STR_SET_LEN(str, len); + + if (len > 0) { + memcpy(RSTRING_PTR(str), buf, len); + ruby_xfree(buf); + } + + TERM_FILL(RSTRING(str)->as.embed.ary + len, TERM_LEN(str)); +} + +void +rb_debug_rstring_null_ptr(const char *func) +{ + fprintf(stderr, "%s is returning NULL!! " + "SIGSEGV is highly expected to follow immediately.\n" + "If you could reproduce, attach your debugger here, " + "and look at the passed string.\n", + func); +} + +/* symbols for [up|down|swap]case/capitalize options */ +static VALUE sym_ascii, sym_turkic, sym_lithuanian, sym_fold; + +static rb_encoding * +get_encoding(VALUE str) +{ + return rb_enc_from_index(ENCODING_GET(str)); +} + +static void +mustnot_broken(VALUE str) +{ + if (is_broken_string(str)) { + rb_raise(rb_eArgError, "invalid byte sequence in %s", rb_enc_name(STR_ENC_GET(str))); + } +} + +static void +mustnot_wchar(VALUE str) +{ + rb_encoding *enc = STR_ENC_GET(str); + if (rb_enc_mbminlen(enc) > 1) { + rb_raise(rb_eArgError, "wide char encoding: %s", rb_enc_name(enc)); + } +} + +static VALUE register_fstring(VALUE str, bool copy, bool force_precompute_hash); + +#if SIZEOF_LONG == SIZEOF_VOIDP +#define PRECOMPUTED_FAKESTR_HASH 1 +#else +#endif + +static inline bool +BARE_STRING_P(VALUE str) +{ + return RBASIC_CLASS(str) == rb_cString && !rb_shape_obj_has_ivars(str); +} + +static inline st_index_t +str_do_hash(VALUE str) +{ + st_index_t h = rb_memhash((const void *)RSTRING_PTR(str), RSTRING_LEN(str)); + int e = RSTRING_LEN(str) ? ENCODING_GET(str) : 0; + if (e && !is_ascii_string(str)) { + h = rb_hash_end(rb_hash_uint32(h, (uint32_t)e)); + } + return h; +} -static VALUE str_alloc _((VALUE)); static VALUE -str_alloc(klass) - VALUE klass; +str_store_precomputed_hash(VALUE str, st_index_t hash) +{ + RUBY_ASSERT(!FL_TEST_RAW(str, STR_PRECOMPUTED_HASH)); + RUBY_ASSERT(STR_EMBED_P(str)); + +#if RUBY_DEBUG + size_t used_bytes = (RSTRING_LEN(str) + TERM_LEN(str)); + size_t free_bytes = str_embed_capa(str) - used_bytes; + RUBY_ASSERT(free_bytes >= sizeof(st_index_t)); +#endif + + memcpy(RSTRING_END(str) + TERM_LEN(str), &hash, sizeof(hash)); + + FL_SET(str, STR_PRECOMPUTED_HASH); + + return str; +} + +VALUE +rb_fstring(VALUE str) +{ + VALUE fstr; + int bare; + + Check_Type(str, T_STRING); + + if (FL_TEST(str, RSTRING_FSTR)) + return str; + + bare = BARE_STRING_P(str); + if (!bare) { + if (STR_EMBED_P(str)) { + OBJ_FREEZE(str); + return str; + } + + if (FL_TEST_RAW(str, STR_SHARED_ROOT | STR_SHARED) == STR_SHARED_ROOT) { + RUBY_ASSERT(OBJ_FROZEN(str)); + return str; + } + } + + if (!FL_TEST_RAW(str, FL_FREEZE | STR_NOFREE | STR_CHILLED)) + rb_str_resize(str, RSTRING_LEN(str)); + + fstr = register_fstring(str, false, false); + + if (!bare) { + str_replace_shared_without_enc(str, fstr); + OBJ_FREEZE(str); + return str; + } + return fstr; +} + +static VALUE fstring_table_obj; + +static VALUE +fstring_concurrent_set_hash(VALUE str) +{ +#ifdef PRECOMPUTED_FAKESTR_HASH + st_index_t h; + if (FL_TEST_RAW(str, STR_FAKESTR)) { + // register_fstring precomputes the hash and stores it in capa for fake strings + h = (st_index_t)RSTRING(str)->as.heap.aux.capa; + } + else { + h = rb_str_hash(str); + } + // rb_str_hash doesn't include the encoding for ascii only strings, so + // we add it to avoid common collisions between `:sym.name` (ASCII) and `"sym"` (UTF-8) + return (VALUE)rb_hash_end(rb_hash_uint32(h, (uint32_t)ENCODING_GET_INLINED(str))); +#else + return (VALUE)rb_str_hash(str); +#endif +} + +static bool +fstring_concurrent_set_cmp(VALUE a, VALUE b) +{ + long alen, blen; + const char *aptr, *bptr; + + RUBY_ASSERT(RB_TYPE_P(a, T_STRING)); + RUBY_ASSERT(RB_TYPE_P(b, T_STRING)); + + RSTRING_GETMEM(a, aptr, alen); + RSTRING_GETMEM(b, bptr, blen); + return (alen == blen && + ENCODING_GET(a) == ENCODING_GET(b) && + memcmp(aptr, bptr, alen) == 0); +} + +struct fstr_create_arg { + bool copy; + bool force_precompute_hash; +}; + +static VALUE +fstring_concurrent_set_create(VALUE str, void *data) +{ + struct fstr_create_arg *arg = data; + + // Unless the string is empty or binary, its coderange has been precomputed. + int coderange = ENC_CODERANGE(str); + + if (FL_TEST_RAW(str, STR_FAKESTR)) { + if (arg->copy) { + VALUE new_str; + long len = RSTRING_LEN(str); + long capa = len + sizeof(st_index_t); + int term_len = TERM_LEN(str); + + if (arg->force_precompute_hash && STR_EMBEDDABLE_P(capa, term_len)) { + new_str = str_alloc_embed(rb_cString, capa + term_len); + memcpy(RSTRING_PTR(new_str), RSTRING_PTR(str), len); + STR_SET_LEN(new_str, RSTRING_LEN(str)); + TERM_FILL(RSTRING_END(new_str), TERM_LEN(str)); + rb_enc_copy(new_str, str); + str_store_precomputed_hash(new_str, str_do_hash(str)); + } + else { + new_str = str_new(rb_cString, RSTRING(str)->as.heap.ptr, RSTRING(str)->len); + rb_enc_copy(new_str, str); +#ifdef PRECOMPUTED_FAKESTR_HASH + if (rb_str_capacity(new_str) >= RSTRING_LEN(str) + term_len + sizeof(st_index_t)) { + str_store_precomputed_hash(new_str, (st_index_t)RSTRING(str)->as.heap.aux.capa); + } +#endif + } + str = new_str; + } + else { + str = str_new_static(rb_cString, RSTRING(str)->as.heap.ptr, + RSTRING(str)->len, + ENCODING_GET(str)); + } + OBJ_FREEZE(str); + } + else { + if (!OBJ_FROZEN(str) || CHILLED_STRING_P(str)) { + str = str_new_frozen(rb_cString, str); + } + if (STR_SHARED_P(str)) { /* str should not be shared */ + /* shared substring */ + str_make_independent(str); + RUBY_ASSERT(OBJ_FROZEN(str)); + } + if (!BARE_STRING_P(str)) { + str = str_new_frozen(rb_cString, str); + } + } + + ENC_CODERANGE_SET(str, coderange); + RBASIC(str)->flags |= RSTRING_FSTR; + if (!RB_OBJ_SHAREABLE_P(str)) { + RB_OBJ_SET_SHAREABLE(str); + } + RUBY_ASSERT((rb_gc_verify_shareable(str), 1)); + RUBY_ASSERT(RB_TYPE_P(str, T_STRING)); + RUBY_ASSERT(OBJ_FROZEN(str)); + RUBY_ASSERT(!FL_TEST_RAW(str, STR_FAKESTR)); + RUBY_ASSERT(!rb_shape_obj_has_ivars(str)); + RUBY_ASSERT(RBASIC_CLASS(str) == rb_cString); + RUBY_ASSERT(!rb_objspace_garbage_object_p(str)); + + return str; +} + +static const struct rb_concurrent_set_funcs fstring_concurrent_set_funcs = { + .hash = fstring_concurrent_set_hash, + .cmp = fstring_concurrent_set_cmp, + .create = fstring_concurrent_set_create, + .free = NULL, +}; + +void +Init_fstring_table(void) +{ + fstring_table_obj = rb_concurrent_set_new(&fstring_concurrent_set_funcs, 8192); + rb_gc_register_address(&fstring_table_obj); +} + +static VALUE +register_fstring(VALUE str, bool copy, bool force_precompute_hash) +{ + struct fstr_create_arg args = { + .copy = copy, + .force_precompute_hash = force_precompute_hash + }; + +#if SIZEOF_VOIDP == SIZEOF_LONG + if (FL_TEST_RAW(str, STR_FAKESTR)) { + // if the string hasn't been interned, we'll need the hash twice, so we + // compute it once and store it in capa + RSTRING(str)->as.heap.aux.capa = (long)str_do_hash(str); + } +#endif + + VALUE result = rb_concurrent_set_find_or_insert(&fstring_table_obj, str, &args); + + RUBY_ASSERT(!rb_objspace_garbage_object_p(result)); + RUBY_ASSERT(RB_TYPE_P(result, T_STRING)); + RUBY_ASSERT(OBJ_FROZEN(result)); + RUBY_ASSERT(RB_OBJ_SHAREABLE_P(result)); + RUBY_ASSERT((rb_gc_verify_shareable(result), 1)); + RUBY_ASSERT(!FL_TEST_RAW(result, STR_FAKESTR)); + RUBY_ASSERT(RBASIC_CLASS(result) == rb_cString); + + return result; +} + +bool +rb_obj_is_fstring_table(VALUE obj) +{ + ASSERT_vm_locking(); + + return obj == fstring_table_obj; +} + +void +rb_gc_free_fstring(VALUE obj) +{ + ASSERT_vm_locking_with_barrier(); + + RUBY_ASSERT(FL_TEST(obj, RSTRING_FSTR)); + RUBY_ASSERT(OBJ_FROZEN(obj)); + RUBY_ASSERT(!FL_TEST(obj, STR_SHARED)); + + rb_concurrent_set_delete_by_identity(fstring_table_obj, obj); + + RB_DEBUG_COUNTER_INC(obj_str_fstr); + + FL_UNSET(obj, RSTRING_FSTR); +} + +void +rb_fstring_foreach_with_replace(int (*callback)(VALUE *str, void *data), void *data) +{ + if (fstring_table_obj) { + rb_concurrent_set_foreach_with_replace(fstring_table_obj, callback, data); + } +} + +static VALUE +setup_fake_str(struct RString *fake_str, const char *name, long len, int encidx) +{ + fake_str->basic.flags = T_STRING|RSTRING_NOEMBED|STR_NOFREE|STR_FAKESTR; + RBASIC_SET_SHAPE_ID((VALUE)fake_str, ROOT_SHAPE_ID); + + if (!name) { + RUBY_ASSERT_ALWAYS(len == 0); + name = ""; + } + + ENCODING_SET_INLINED((VALUE)fake_str, encidx); + + RBASIC_SET_CLASS_RAW((VALUE)fake_str, rb_cString); + fake_str->len = len; + fake_str->as.heap.ptr = (char *)name; + fake_str->as.heap.aux.capa = len; + return (VALUE)fake_str; +} + +/* + * set up a fake string which refers a static string literal. + */ +VALUE +rb_setup_fake_str(struct RString *fake_str, const char *name, long len, rb_encoding *enc) +{ + return setup_fake_str(fake_str, name, len, rb_enc_to_index(enc)); +} + +/* + * rb_fstring_new and rb_fstring_cstr family create or lookup a frozen + * shared string which refers a static string literal. `ptr` must + * point a constant string. + */ +VALUE +rb_fstring_new(const char *ptr, long len) +{ + struct RString fake_str = {RBASIC_INIT}; + return register_fstring(setup_fake_str(&fake_str, ptr, len, ENCINDEX_US_ASCII), false, false); +} + +VALUE +rb_fstring_enc_new(const char *ptr, long len, rb_encoding *enc) +{ + struct RString fake_str = {RBASIC_INIT}; + return register_fstring(rb_setup_fake_str(&fake_str, ptr, len, enc), false, false); +} + +VALUE +rb_fstring_cstr(const char *ptr) +{ + return rb_fstring_new(ptr, strlen(ptr)); +} + +static inline bool +single_byte_optimizable(VALUE str) +{ + int encindex = ENCODING_GET(str); + switch (encindex) { + case ENCINDEX_ASCII_8BIT: + case ENCINDEX_US_ASCII: + return true; + case ENCINDEX_UTF_8: + // For UTF-8 it's worth scanning the string coderange when unknown. + return rb_enc_str_coderange(str) == ENC_CODERANGE_7BIT; + } + /* Conservative. It may be ENC_CODERANGE_UNKNOWN. */ + if (ENC_CODERANGE(str) == ENC_CODERANGE_7BIT) { + return true; + } + + if (rb_enc_mbmaxlen(rb_enc_from_index(encindex)) == 1) { + return true; + } + + /* Conservative. Possibly single byte. + * "\xa1" in Shift_JIS for example. */ + return false; +} + +VALUE rb_fs; + +static inline const char * +search_nonascii(const char *p, const char *e) +{ + const char *s, *t; + +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) +# if SIZEOF_UINTPTR_T == 8 +# define NONASCII_MASK UINT64_C(0x8080808080808080) +# elif SIZEOF_UINTPTR_T == 4 +# define NONASCII_MASK UINT32_C(0x80808080) +# else +# error "don't know what to do." +# endif +#else +# if SIZEOF_UINTPTR_T == 8 +# define NONASCII_MASK ((uintptr_t)0x80808080UL << 32 | (uintptr_t)0x80808080UL) +# elif SIZEOF_UINTPTR_T == 4 +# define NONASCII_MASK 0x80808080UL /* or...? */ +# else +# error "don't know what to do." +# endif +#endif + + if (UNALIGNED_WORD_ACCESS || e - p >= SIZEOF_VOIDP) { +#if !UNALIGNED_WORD_ACCESS + if ((uintptr_t)p % SIZEOF_VOIDP) { + int l = SIZEOF_VOIDP - (uintptr_t)p % SIZEOF_VOIDP; + p += l; + switch (l) { + default: UNREACHABLE; +#if SIZEOF_VOIDP > 4 + case 7: if (p[-7]&0x80) return p-7; + case 6: if (p[-6]&0x80) return p-6; + case 5: if (p[-5]&0x80) return p-5; + case 4: if (p[-4]&0x80) return p-4; +#endif + case 3: if (p[-3]&0x80) return p-3; + case 2: if (p[-2]&0x80) return p-2; + case 1: if (p[-1]&0x80) return p-1; + case 0: break; + } + } +#endif +#if defined(HAVE_BUILTIN___BUILTIN_ASSUME_ALIGNED) &&! UNALIGNED_WORD_ACCESS +#define aligned_ptr(value) \ + __builtin_assume_aligned((value), sizeof(uintptr_t)) +#else +#define aligned_ptr(value) (value) +#endif + s = aligned_ptr(p); + t = (e - (SIZEOF_VOIDP-1)); +#undef aligned_ptr + for (;s < t; s += sizeof(uintptr_t)) { + uintptr_t word; + memcpy(&word, s, sizeof(word)); + if (word & NONASCII_MASK) { +#ifdef WORDS_BIGENDIAN + return (const char *)s + (nlz_intptr(word&NONASCII_MASK)>>3); +#else + return (const char *)s + (ntz_intptr(word&NONASCII_MASK)>>3); +#endif + } + } + p = (const char *)s; + } + + switch (e - p) { + default: UNREACHABLE; +#if SIZEOF_VOIDP > 4 + case 7: if (e[-7]&0x80) return e-7; + case 6: if (e[-6]&0x80) return e-6; + case 5: if (e[-5]&0x80) return e-5; + case 4: if (e[-4]&0x80) return e-4; +#endif + case 3: if (e[-3]&0x80) return e-3; + case 2: if (e[-2]&0x80) return e-2; + case 1: if (e[-1]&0x80) return e-1; + case 0: return NULL; + } +} + +static int +coderange_scan(const char *p, long len, rb_encoding *enc) +{ + const char *e = p + len; + + if (rb_enc_to_index(enc) == rb_ascii8bit_encindex()) { + /* enc is ASCII-8BIT. ASCII-8BIT string never be broken. */ + p = search_nonascii(p, e); + return p ? ENC_CODERANGE_VALID : ENC_CODERANGE_7BIT; + } + + if (rb_enc_asciicompat(enc)) { + p = search_nonascii(p, e); + if (!p) return ENC_CODERANGE_7BIT; + for (;;) { + int ret = rb_enc_precise_mbclen(p, e, enc); + if (!MBCLEN_CHARFOUND_P(ret)) return ENC_CODERANGE_BROKEN; + p += MBCLEN_CHARFOUND_LEN(ret); + if (p == e) break; + p = search_nonascii(p, e); + if (!p) break; + } + } + else { + while (p < e) { + int ret = rb_enc_precise_mbclen(p, e, enc); + if (!MBCLEN_CHARFOUND_P(ret)) return ENC_CODERANGE_BROKEN; + p += MBCLEN_CHARFOUND_LEN(ret); + } + } + return ENC_CODERANGE_VALID; +} + +long +rb_str_coderange_scan_restartable(const char *s, const char *e, rb_encoding *enc, int *cr) +{ + const char *p = s; + + if (*cr == ENC_CODERANGE_BROKEN) + return e - s; + + if (rb_enc_to_index(enc) == rb_ascii8bit_encindex()) { + /* enc is ASCII-8BIT. ASCII-8BIT string never be broken. */ + if (*cr == ENC_CODERANGE_VALID) return e - s; + p = search_nonascii(p, e); + *cr = p ? ENC_CODERANGE_VALID : ENC_CODERANGE_7BIT; + return e - s; + } + else if (rb_enc_asciicompat(enc)) { + p = search_nonascii(p, e); + if (!p) { + if (*cr != ENC_CODERANGE_VALID) *cr = ENC_CODERANGE_7BIT; + return e - s; + } + for (;;) { + int ret = rb_enc_precise_mbclen(p, e, enc); + if (!MBCLEN_CHARFOUND_P(ret)) { + *cr = MBCLEN_INVALID_P(ret) ? ENC_CODERANGE_BROKEN: ENC_CODERANGE_UNKNOWN; + return p - s; + } + p += MBCLEN_CHARFOUND_LEN(ret); + if (p == e) break; + p = search_nonascii(p, e); + if (!p) break; + } + } + else { + while (p < e) { + int ret = rb_enc_precise_mbclen(p, e, enc); + if (!MBCLEN_CHARFOUND_P(ret)) { + *cr = MBCLEN_INVALID_P(ret) ? ENC_CODERANGE_BROKEN: ENC_CODERANGE_UNKNOWN; + return p - s; + } + p += MBCLEN_CHARFOUND_LEN(ret); + } + } + *cr = ENC_CODERANGE_VALID; + return e - s; +} + +static inline void +str_enc_copy(VALUE str1, VALUE str2) +{ + rb_enc_set_index(str1, ENCODING_GET(str2)); +} + +/* Like str_enc_copy, but does not check frozen status of str1. + * You should use this only if you're certain that str1 is not frozen. */ +static inline void +str_enc_copy_direct(VALUE str1, VALUE str2) { - NEWOBJ(str, struct RString); - OBJSETUP(str, klass, T_STRING); + int inlined_encoding = RB_ENCODING_GET_INLINED(str2); + if (inlined_encoding == ENCODING_INLINE_MAX) { + rb_enc_set_index(str1, rb_enc_get_index(str2)); + } + else { + ENCODING_SET_INLINED(str1, inlined_encoding); + } +} + +static void +rb_enc_cr_str_copy_for_substr(VALUE dest, VALUE src) +{ + /* this function is designed for copying encoding and coderange + * from src to new string "dest" which is made from the part of src. + */ + str_enc_copy(dest, src); + if (RSTRING_LEN(dest) == 0) { + if (!rb_enc_asciicompat(STR_ENC_GET(src))) + ENC_CODERANGE_SET(dest, ENC_CODERANGE_VALID); + else + ENC_CODERANGE_SET(dest, ENC_CODERANGE_7BIT); + return; + } + switch (ENC_CODERANGE(src)) { + case ENC_CODERANGE_7BIT: + ENC_CODERANGE_SET(dest, ENC_CODERANGE_7BIT); + break; + case ENC_CODERANGE_VALID: + if (!rb_enc_asciicompat(STR_ENC_GET(src)) || + search_nonascii(RSTRING_PTR(dest), RSTRING_END(dest))) + ENC_CODERANGE_SET(dest, ENC_CODERANGE_VALID); + else + ENC_CODERANGE_SET(dest, ENC_CODERANGE_7BIT); + break; + default: + break; + } +} + +static void +rb_enc_cr_str_exact_copy(VALUE dest, VALUE src) +{ + str_enc_copy(dest, src); + ENC_CODERANGE_SET(dest, ENC_CODERANGE(src)); +} + +static int +enc_coderange_scan(VALUE str, rb_encoding *enc) +{ + return coderange_scan(RSTRING_PTR(str), RSTRING_LEN(str), enc); +} + +int +rb_enc_str_coderange_scan(VALUE str, rb_encoding *enc) +{ + return enc_coderange_scan(str, enc); +} + +int +rb_enc_str_coderange(VALUE str) +{ + int cr = ENC_CODERANGE(str); + + if (cr == ENC_CODERANGE_UNKNOWN) { + cr = enc_coderange_scan(str, get_encoding(str)); + ENC_CODERANGE_SET(str, cr); + } + return cr; +} + +static inline bool +rb_enc_str_asciicompat(VALUE str) +{ + int encindex = ENCODING_GET_INLINED(str); + return rb_str_encindex_fastpath(encindex) || rb_enc_asciicompat(rb_enc_get_from_index(encindex)); +} + +int +rb_enc_str_asciionly_p(VALUE str) +{ + switch(ENC_CODERANGE(str)) { + case ENC_CODERANGE_UNKNOWN: + return rb_enc_str_asciicompat(str) && is_ascii_string(str); + case ENC_CODERANGE_7BIT: + return true; + default: + return false; + } +} + +static inline void +str_mod_check(VALUE s, const char *p, long len) +{ + if (RSTRING_PTR(s) != p || RSTRING_LEN(s) != len){ + rb_raise(rb_eRuntimeError, "string modified"); + } +} + +static size_t +str_capacity(VALUE str, const int termlen) +{ + if (STR_EMBED_P(str)) { + return str_embed_capa(str) - termlen; + } + else if (FL_ANY_RAW(str, STR_SHARED|STR_NOFREE)) { + return RSTRING(str)->len; + } + else { + return RSTRING(str)->as.heap.aux.capa; + } +} + +size_t +rb_str_capacity(VALUE str) +{ + return str_capacity(str, TERM_LEN(str)); +} + +static inline void +must_not_null(const char *ptr) +{ + if (!ptr) { + rb_raise(rb_eArgError, "NULL pointer given"); + } +} + +static inline VALUE +str_alloc_embed(VALUE klass, size_t capa) +{ + size_t size = rb_str_embed_size(capa, 0); + RUBY_ASSERT(size > 0); + RUBY_ASSERT(rb_gc_size_allocatable_p(size)); + + NEWOBJ_OF(str, struct RString, klass, + T_STRING | (RGENGC_WB_PROTECTED_STRING ? FL_WB_PROTECTED : 0), size, 0); - str->ptr = 0; str->len = 0; - str->aux.capa = 0; + str->as.embed.ary[0] = 0; return (VALUE)str; } +static inline VALUE +str_alloc_heap(VALUE klass) +{ + NEWOBJ_OF(str, struct RString, klass, + T_STRING | STR_NOEMBED | (RGENGC_WB_PROTECTED_STRING ? FL_WB_PROTECTED : 0), sizeof(struct RString), 0); + + str->len = 0; + str->as.heap.aux.capa = 0; + str->as.heap.ptr = NULL; + + return (VALUE)str; +} + +static inline VALUE +empty_str_alloc(VALUE klass) +{ + RUBY_DTRACE_CREATE_HOOK(STRING, 0); + VALUE str = str_alloc_embed(klass, 0); + memset(RSTRING(str)->as.embed.ary, 0, str_embed_capa(str)); + ENC_CODERANGE_SET(str, ENC_CODERANGE_7BIT); + return str; +} + static VALUE -str_new(klass, ptr, len) - VALUE klass; - const char *ptr; - long len; +str_enc_new(VALUE klass, const char *ptr, long len, rb_encoding *enc) { VALUE str; if (len < 0) { - rb_raise(rb_eArgError, "negative string size (or size too big)"); + rb_raise(rb_eArgError, "negative string size (or size too big)"); + } + + if (enc == NULL) { + enc = rb_ascii8bit_encoding(); + } + + RUBY_DTRACE_CREATE_HOOK(STRING, len); + + int termlen = rb_enc_mbminlen(enc); + + if (STR_EMBEDDABLE_P(len, termlen)) { + str = str_alloc_embed(klass, len + termlen); + if (len == 0) { + ENC_CODERANGE_SET(str, rb_enc_asciicompat(enc) ? ENC_CODERANGE_7BIT : ENC_CODERANGE_VALID); + } + } + else { + str = str_alloc_heap(klass); + RSTRING(str)->as.heap.aux.capa = len; + /* :FIXME: @shyouhei guesses `len + termlen` is guaranteed to never + * integer overflow. If we can STATIC_ASSERT that, the following + * mul_add_mul can be reverted to a simple ALLOC_N. */ + RSTRING(str)->as.heap.ptr = + rb_xmalloc_mul_add_mul(sizeof(char), len, sizeof(char), termlen); } - str = str_alloc(klass); - RSTRING(str)->len = len; - RSTRING(str)->aux.capa = len; - RSTRING(str)->ptr = ALLOC_N(char,len+1); + rb_enc_raw_set(str, enc); + if (ptr) { - memcpy(RSTRING(str)->ptr, ptr, len); + memcpy(RSTRING_PTR(str), ptr, len); + } + else { + memset(RSTRING_PTR(str), 0, len); } - RSTRING(str)->ptr[len] = '\0'; + + STR_SET_LEN(str, len); + TERM_FILL(RSTRING_PTR(str) + len, termlen); return str; } +static VALUE +str_new(VALUE klass, const char *ptr, long len) +{ + return str_enc_new(klass, ptr, len, rb_ascii8bit_encoding()); +} + VALUE -rb_str_new(ptr, len) - const char *ptr; - long len; +rb_str_new(const char *ptr, long len) { return str_new(rb_cString, ptr, len); } VALUE -rb_str_new2(ptr) - const char *ptr; +rb_usascii_str_new(const char *ptr, long len) { - if (!ptr) { - rb_raise(rb_eArgError, "NULL pointer given"); - } + return str_enc_new(rb_cString, ptr, len, rb_usascii_encoding()); +} + +VALUE +rb_utf8_str_new(const char *ptr, long len) +{ + return str_enc_new(rb_cString, ptr, len, rb_utf8_encoding()); +} + +VALUE +rb_enc_str_new(const char *ptr, long len, rb_encoding *enc) +{ + return str_enc_new(rb_cString, ptr, len, enc); +} + +VALUE +rb_str_new_cstr(const char *ptr) +{ + must_not_null(ptr); + /* rb_str_new_cstr() can take pointer from non-malloc-generated + * memory regions, and that cannot be detected by the MSAN. Just + * trust the programmer that the argument passed here is a sane C + * string. */ + __msan_unpoison_string(ptr); return rb_str_new(ptr, strlen(ptr)); } VALUE -rb_tainted_str_new(ptr, len) - const char *ptr; - long len; +rb_usascii_str_new_cstr(const char *ptr) +{ + return rb_enc_str_new_cstr(ptr, rb_usascii_encoding()); +} + +VALUE +rb_utf8_str_new_cstr(const char *ptr) +{ + return rb_enc_str_new_cstr(ptr, rb_utf8_encoding()); +} + +VALUE +rb_enc_str_new_cstr(const char *ptr, rb_encoding *enc) { - VALUE str = rb_str_new(ptr, len); + must_not_null(ptr); + if (rb_enc_mbminlen(enc) != 1) { + rb_raise(rb_eArgError, "wchar encoding given"); + } + return rb_enc_str_new(ptr, strlen(ptr), enc); +} - OBJ_TAINT(str); +static VALUE +str_new_static(VALUE klass, const char *ptr, long len, int encindex) +{ + VALUE str; + + if (len < 0) { + rb_raise(rb_eArgError, "negative string size (or size too big)"); + } + + if (!ptr) { + str = str_enc_new(klass, ptr, len, rb_enc_from_index(encindex)); + } + else { + RUBY_DTRACE_CREATE_HOOK(STRING, len); + str = str_alloc_heap(klass); + RSTRING(str)->len = len; + RSTRING(str)->as.heap.ptr = (char *)ptr; + RSTRING(str)->as.heap.aux.capa = len; + RBASIC(str)->flags |= STR_NOFREE; + rb_enc_associate_index(str, encindex); + } return str; } VALUE -rb_tainted_str_new2(ptr) +rb_str_new_static(const char *ptr, long len) +{ + return str_new_static(rb_cString, ptr, len, 0); +} + +VALUE +rb_usascii_str_new_static(const char *ptr, long len) +{ + return str_new_static(rb_cString, ptr, len, ENCINDEX_US_ASCII); +} + +VALUE +rb_utf8_str_new_static(const char *ptr, long len) +{ + return str_new_static(rb_cString, ptr, len, ENCINDEX_UTF_8); +} + +VALUE +rb_enc_str_new_static(const char *ptr, long len, rb_encoding *enc) +{ + return str_new_static(rb_cString, ptr, len, rb_enc_to_index(enc)); +} + +static VALUE str_cat_conv_enc_opts(VALUE newstr, long ofs, const char *ptr, long len, + rb_encoding *from, rb_encoding *to, + int ecflags, VALUE ecopts); + +static inline bool +is_enc_ascii_string(VALUE str, rb_encoding *enc) +{ + int encidx = rb_enc_to_index(enc); + if (rb_enc_get_index(str) == encidx) + return is_ascii_string(str); + return enc_coderange_scan(str, enc) == ENC_CODERANGE_7BIT; +} + +VALUE +rb_str_conv_enc_opts(VALUE str, rb_encoding *from, rb_encoding *to, int ecflags, VALUE ecopts) +{ + long len; const char *ptr; + VALUE newstr; + + if (!to) return str; + if (!from) from = rb_enc_get(str); + if (from == to) return str; + if ((rb_enc_asciicompat(to) && is_enc_ascii_string(str, from)) || + rb_is_ascii8bit_enc(to)) { + if (STR_ENC_GET(str) != to) { + str = rb_str_dup(str); + rb_enc_associate(str, to); + } + return str; + } + + RSTRING_GETMEM(str, ptr, len); + newstr = str_cat_conv_enc_opts(rb_str_buf_new(len), 0, ptr, len, + from, to, ecflags, ecopts); + if (NIL_P(newstr)) { + /* some error, return original */ + return str; + } + return newstr; +} + +VALUE +rb_str_cat_conv_enc_opts(VALUE newstr, long ofs, const char *ptr, long len, + rb_encoding *from, int ecflags, VALUE ecopts) { - VALUE str = rb_str_new2(ptr); + long olen; - OBJ_TAINT(str); + olen = RSTRING_LEN(newstr); + if (ofs < -olen || olen < ofs) + rb_raise(rb_eIndexError, "index %ld out of string", ofs); + if (ofs < 0) ofs += olen; + if (!from) { + STR_SET_LEN(newstr, ofs); + return rb_str_cat(newstr, ptr, len); + } + + rb_str_modify(newstr); + return str_cat_conv_enc_opts(newstr, ofs, ptr, len, from, + rb_enc_get(newstr), + ecflags, ecopts); +} + +VALUE +rb_str_initialize(VALUE str, const char *ptr, long len, rb_encoding *enc) +{ + STR_SET_LEN(str, 0); + rb_enc_associate(str, enc); + rb_str_cat(str, ptr, len); return str; } static VALUE -str_new3(klass, str) - VALUE klass, str; -{ - VALUE str2 = str_alloc(klass); +str_cat_conv_enc_opts(VALUE newstr, long ofs, const char *ptr, long len, + rb_encoding *from, rb_encoding *to, + int ecflags, VALUE ecopts) +{ + rb_econv_t *ec; + rb_econv_result_t ret; + long olen; + VALUE econv_wrapper; + const unsigned char *start, *sp; + unsigned char *dest, *dp; + size_t converted_output = (size_t)ofs; + + olen = rb_str_capacity(newstr); + + econv_wrapper = rb_obj_alloc(rb_cEncodingConverter); + RBASIC_CLEAR_CLASS(econv_wrapper); + ec = rb_econv_open_opts(from->name, to->name, ecflags, ecopts); + if (!ec) return Qnil; + DATA_PTR(econv_wrapper) = ec; + + sp = (unsigned char*)ptr; + start = sp; + while ((dest = (unsigned char*)RSTRING_PTR(newstr)), + (dp = dest + converted_output), + (ret = rb_econv_convert(ec, &sp, start + len, &dp, dest + olen, 0)), + ret == econv_destination_buffer_full) { + /* destination buffer short */ + size_t converted_input = sp - start; + size_t rest = len - converted_input; + converted_output = dp - dest; + rb_str_set_len(newstr, converted_output); + if (converted_input && converted_output && + rest < (LONG_MAX / converted_output)) { + rest = (rest * converted_output) / converted_input; + } + else { + rest = olen; + } + olen += rest < 2 ? 2 : rest; + rb_str_resize(newstr, olen); + } + DATA_PTR(econv_wrapper) = 0; + RB_GC_GUARD(econv_wrapper); + rb_econv_close(ec); + switch (ret) { + case econv_finished: + len = dp - (unsigned char*)RSTRING_PTR(newstr); + rb_str_set_len(newstr, len); + rb_enc_associate(newstr, to); + return newstr; - RSTRING(str2)->len = RSTRING(str)->len; - RSTRING(str2)->ptr = RSTRING(str)->ptr; - RSTRING(str2)->aux.shared = str; - FL_SET(str2, ELTS_SHARED); - OBJ_INFECT(str2, str); + default: + return Qnil; + } +} - return str2; +VALUE +rb_str_conv_enc(VALUE str, rb_encoding *from, rb_encoding *to) +{ + return rb_str_conv_enc_opts(str, from, to, 0, Qnil); } VALUE -rb_str_new3(str) +rb_external_str_new_with_enc(const char *ptr, long len, rb_encoding *eenc) +{ + rb_encoding *ienc; VALUE str; + const int eidx = rb_enc_to_index(eenc); + + if (!ptr) { + return rb_enc_str_new(ptr, len, eenc); + } + + /* ASCII-8BIT case, no conversion */ + if ((eidx == rb_ascii8bit_encindex()) || + (eidx == rb_usascii_encindex() && search_nonascii(ptr, ptr + len))) { + return rb_str_new(ptr, len); + } + /* no default_internal or same encoding, no conversion */ + ienc = rb_default_internal_encoding(); + if (!ienc || eenc == ienc) { + return rb_enc_str_new(ptr, len, eenc); + } + /* ASCII compatible, and ASCII only string, no conversion in + * default_internal */ + if ((eidx == rb_ascii8bit_encindex()) || + (eidx == rb_usascii_encindex()) || + (rb_enc_asciicompat(eenc) && !search_nonascii(ptr, ptr + len))) { + return rb_enc_str_new(ptr, len, ienc); + } + /* convert from the given encoding to default_internal */ + str = rb_enc_str_new(NULL, 0, ienc); + /* when the conversion failed for some reason, just ignore the + * default_internal and result in the given encoding as-is. */ + if (NIL_P(rb_str_cat_conv_enc_opts(str, 0, ptr, len, eenc, 0, Qnil))) { + rb_str_initialize(str, ptr, len, eenc); + } + return str; +} + +VALUE +rb_external_str_with_enc(VALUE str, rb_encoding *eenc) +{ + int eidx = rb_enc_to_index(eenc); + if (eidx == rb_usascii_encindex() && + !is_ascii_string(str)) { + rb_enc_associate_index(str, rb_ascii8bit_encindex()); + return str; + } + rb_enc_associate_index(str, eidx); + return rb_str_conv_enc(str, eenc, rb_default_internal_encoding()); +} + +VALUE +rb_external_str_new(const char *ptr, long len) +{ + return rb_external_str_new_with_enc(ptr, len, rb_default_external_encoding()); +} + +VALUE +rb_external_str_new_cstr(const char *ptr) { - return str_new3(rb_obj_class(str), str); + return rb_external_str_new_with_enc(ptr, strlen(ptr), rb_default_external_encoding()); +} + +VALUE +rb_locale_str_new(const char *ptr, long len) +{ + return rb_external_str_new_with_enc(ptr, len, rb_locale_encoding()); +} + +VALUE +rb_locale_str_new_cstr(const char *ptr) +{ + return rb_external_str_new_with_enc(ptr, strlen(ptr), rb_locale_encoding()); +} + +VALUE +rb_filesystem_str_new(const char *ptr, long len) +{ + return rb_external_str_new_with_enc(ptr, len, rb_filesystem_encoding()); +} + +VALUE +rb_filesystem_str_new_cstr(const char *ptr) +{ + return rb_external_str_new_with_enc(ptr, strlen(ptr), rb_filesystem_encoding()); +} + +VALUE +rb_str_export(VALUE str) +{ + return rb_str_export_to_enc(str, rb_default_external_encoding()); +} + +VALUE +rb_str_export_locale(VALUE str) +{ + return rb_str_export_to_enc(str, rb_locale_encoding()); +} + +VALUE +rb_str_export_to_enc(VALUE str, rb_encoding *enc) +{ + return rb_str_conv_enc(str, STR_ENC_GET(str), enc); } static VALUE -str_new4(klass, str) - VALUE klass, str; +str_replace_shared_without_enc(VALUE str2, VALUE str) { - VALUE str2 = str_alloc(klass); + const int termlen = TERM_LEN(str); + char *ptr; + long len; - RSTRING(str2)->len = RSTRING(str)->len; - RSTRING(str2)->ptr = RSTRING(str)->ptr; - if (FL_TEST(str, ELTS_SHARED)) { - FL_SET(str2, ELTS_SHARED); - RSTRING(str2)->aux.shared = RSTRING(str)->aux.shared; + RSTRING_GETMEM(str, ptr, len); + if (str_embed_capa(str2) >= len + termlen) { + char *ptr2 = RSTRING(str2)->as.embed.ary; + STR_SET_EMBED(str2); + memcpy(ptr2, RSTRING_PTR(str), len); + TERM_FILL(ptr2+len, termlen); } else { - FL_SET(str, ELTS_SHARED); - RSTRING(str)->aux.shared = str2; + VALUE root; + if (STR_SHARED_P(str)) { + root = RSTRING(str)->as.heap.aux.shared; + RSTRING_GETMEM(str, ptr, len); + } + else { + root = rb_str_new_frozen(str); + RSTRING_GETMEM(root, ptr, len); + } + RUBY_ASSERT(OBJ_FROZEN(root)); + + if (!STR_EMBED_P(str2) && !FL_TEST_RAW(str2, STR_SHARED|STR_NOFREE)) { + if (FL_TEST_RAW(str2, STR_SHARED_ROOT)) { + rb_fatal("about to free a possible shared root"); + } + char *ptr2 = STR_HEAP_PTR(str2); + if (ptr2 != ptr) { + ruby_sized_xfree(ptr2, STR_HEAP_SIZE(str2)); + } + } + FL_SET(str2, STR_NOEMBED); + RSTRING(str2)->as.heap.ptr = ptr; + STR_SET_SHARED(str2, root); } + STR_SET_LEN(str2, len); + + return str2; +} + +static VALUE +str_replace_shared(VALUE str2, VALUE str) +{ + str_replace_shared_without_enc(str2, str); + rb_enc_cr_str_exact_copy(str2, str); return str2; } +static VALUE +str_new_shared(VALUE klass, VALUE str) +{ + return str_replace_shared(str_alloc_heap(klass), str); +} + VALUE -rb_str_new4(orig) - VALUE orig; +rb_str_new_shared(VALUE str) { - VALUE klass, str; + return str_new_shared(rb_obj_class(str), str); +} + +VALUE +rb_str_new_frozen(VALUE orig) +{ + if (RB_FL_TEST_RAW(orig, FL_FREEZE | STR_CHILLED) == FL_FREEZE) return orig; + return str_new_frozen(rb_obj_class(orig), orig); +} + +static VALUE +rb_str_new_frozen_String(VALUE orig) +{ + if (OBJ_FROZEN(orig) && rb_obj_class(orig) == rb_cString) return orig; + return str_new_frozen(rb_cString, orig); +} + + +VALUE +rb_str_frozen_bare_string(VALUE orig) +{ + if (RB_LIKELY(BARE_STRING_P(orig) && OBJ_FROZEN_RAW(orig))) return orig; + return str_new_frozen(rb_cString, orig); +} - if (OBJ_FROZEN(orig)) return orig; - klass = rb_obj_class(orig); - if (FL_TEST(orig, ELTS_SHARED) && (str = RSTRING(orig)->aux.shared) && klass == RBASIC(str)->klass) { - long ofs; - ofs = RSTRING(str)->len - RSTRING(orig)->len; - if (ofs > 0) { - str = str_new3(klass, str); - RSTRING(str)->ptr += ofs; - RSTRING(str)->len -= ofs; - } +VALUE +rb_str_tmp_frozen_acquire(VALUE orig) +{ + if (OBJ_FROZEN_RAW(orig)) return orig; + return str_new_frozen_buffer(0, orig, FALSE); +} + +VALUE +rb_str_tmp_frozen_no_embed_acquire(VALUE orig) +{ + if (OBJ_FROZEN_RAW(orig) && !STR_EMBED_P(orig) && !rb_str_reembeddable_p(orig)) return orig; + if (STR_SHARED_P(orig) && !STR_EMBED_P(RSTRING(orig)->as.heap.aux.shared)) return rb_str_tmp_frozen_acquire(orig); + + VALUE str = str_alloc_heap(0); + OBJ_FREEZE(str); + /* Always set the STR_SHARED_ROOT to ensure it does not get re-embedded. */ + FL_SET(str, STR_SHARED_ROOT); + + size_t capa = str_capacity(orig, TERM_LEN(orig)); + + /* If the string is embedded then we want to create a copy that is heap + * allocated. If the string is shared then the shared root must be + * embedded, so we want to create a copy. If the string is a shared root + * then it must be embedded, so we want to create a copy. */ + if (STR_EMBED_P(orig) || FL_TEST_RAW(orig, STR_SHARED | STR_SHARED_ROOT | RSTRING_FSTR)) { + RSTRING(str)->as.heap.ptr = rb_xmalloc_mul_add_mul(sizeof(char), capa, sizeof(char), TERM_LEN(orig)); + memcpy(RSTRING(str)->as.heap.ptr, RSTRING_PTR(orig), capa); } - else if (FL_TEST(orig, STR_ASSOC)) { - str = str_new(klass, RSTRING(orig)->ptr, RSTRING(orig)->len); + else { + /* orig must be heap allocated and not shared, so we can safely transfer + * the pointer to str. */ + RSTRING(str)->as.heap.ptr = RSTRING(orig)->as.heap.ptr; + RBASIC(str)->flags |= RBASIC(orig)->flags & STR_NOFREE; + RBASIC(orig)->flags &= ~STR_NOFREE; + STR_SET_SHARED(orig, str); + if (RB_OBJ_SHAREABLE_P(orig)) { + RB_OBJ_SET_SHAREABLE(str); + RUBY_ASSERT((rb_gc_verify_shareable(str), 1)); + } + } + + RSTRING(str)->len = RSTRING(orig)->len; + RSTRING(str)->as.heap.aux.capa = capa; + + return str; +} + +void +rb_str_tmp_frozen_release(VALUE orig, VALUE tmp) +{ + if (RBASIC_CLASS(tmp) != 0) + return; + + if (STR_EMBED_P(tmp)) { + RUBY_ASSERT(OBJ_FROZEN_RAW(tmp)); + } + else if (FL_TEST_RAW(orig, STR_SHARED | STR_TMPLOCK) == STR_TMPLOCK && + !OBJ_FROZEN_RAW(orig)) { + VALUE shared = RSTRING(orig)->as.heap.aux.shared; + + if (shared == tmp && !FL_TEST_RAW(tmp, STR_BORROWED)) { + RUBY_ASSERT(RSTRING(orig)->as.heap.ptr == RSTRING(tmp)->as.heap.ptr); + RUBY_ASSERT(RSTRING_LEN(orig) == RSTRING_LEN(tmp)); + + /* Unshare orig since the root (tmp) only has this one child. */ + FL_UNSET_RAW(orig, STR_SHARED); + RSTRING(orig)->as.heap.aux.capa = RSTRING(tmp)->as.heap.aux.capa; + RBASIC(orig)->flags |= RBASIC(tmp)->flags & STR_NOFREE; + RUBY_ASSERT(OBJ_FROZEN_RAW(tmp)); + + /* Make tmp embedded and empty so it is safe for sweeping. */ + STR_SET_EMBED(tmp); + STR_SET_LEN(tmp, 0); + } + } +} + +static VALUE +str_new_frozen(VALUE klass, VALUE orig) +{ + return str_new_frozen_buffer(klass, orig, TRUE); +} + +static VALUE +heap_str_make_shared(VALUE klass, VALUE orig) +{ + RUBY_ASSERT(!STR_EMBED_P(orig)); + RUBY_ASSERT(!STR_SHARED_P(orig)); + RUBY_ASSERT(!RB_OBJ_SHAREABLE_P(orig)); + + VALUE str = str_alloc_heap(klass); + STR_SET_LEN(str, RSTRING_LEN(orig)); + RSTRING(str)->as.heap.ptr = RSTRING_PTR(orig); + RSTRING(str)->as.heap.aux.capa = RSTRING(orig)->as.heap.aux.capa; + RBASIC(str)->flags |= RBASIC(orig)->flags & STR_NOFREE; + RBASIC(orig)->flags &= ~STR_NOFREE; + STR_SET_SHARED(orig, str); + if (klass == 0) + FL_UNSET_RAW(str, STR_BORROWED); + return str; +} + +static VALUE +str_new_frozen_buffer(VALUE klass, VALUE orig, int copy_encoding) +{ + VALUE str; + + long len = RSTRING_LEN(orig); + rb_encoding *enc = copy_encoding ? STR_ENC_GET(orig) : rb_ascii8bit_encoding(); + int termlen = copy_encoding ? TERM_LEN(orig) : 1; + + if (STR_EMBED_P(orig) || STR_EMBEDDABLE_P(len, termlen)) { + str = str_enc_new(klass, RSTRING_PTR(orig), len, enc); + RUBY_ASSERT(STR_EMBED_P(str)); } else { - str = str_new4(klass, orig); + if (FL_TEST_RAW(orig, STR_SHARED)) { + VALUE shared = RSTRING(orig)->as.heap.aux.shared; + long ofs = RSTRING(orig)->as.heap.ptr - RSTRING_PTR(shared); + long rest = RSTRING_LEN(shared) - ofs - RSTRING_LEN(orig); + RUBY_ASSERT(ofs >= 0); + RUBY_ASSERT(rest >= 0); + RUBY_ASSERT(ofs + rest <= RSTRING_LEN(shared)); + RUBY_ASSERT(OBJ_FROZEN(shared)); + + if ((ofs > 0) || (rest > 0) || + (klass != RBASIC(shared)->klass) || + ENCODING_GET(shared) != ENCODING_GET(orig)) { + str = str_new_shared(klass, shared); + RUBY_ASSERT(!STR_EMBED_P(str)); + RSTRING(str)->as.heap.ptr += ofs; + STR_SET_LEN(str, RSTRING_LEN(str) - (ofs + rest)); + } + else { + if (RBASIC_CLASS(shared) == 0) + FL_SET_RAW(shared, STR_BORROWED); + return shared; + } + } + else if (STR_EMBEDDABLE_P(RSTRING_LEN(orig), TERM_LEN(orig))) { + str = str_alloc_embed(klass, RSTRING_LEN(orig) + TERM_LEN(orig)); + STR_SET_EMBED(str); + memcpy(RSTRING_PTR(str), RSTRING_PTR(orig), RSTRING_LEN(orig)); + STR_SET_LEN(str, RSTRING_LEN(orig)); + ENC_CODERANGE_SET(str, ENC_CODERANGE(orig)); + TERM_FILL(RSTRING_END(str), TERM_LEN(orig)); + } + else { + if (RB_OBJ_SHAREABLE_P(orig)) { + str = str_new(klass, RSTRING_PTR(orig), RSTRING_LEN(orig)); + } + else { + str = heap_str_make_shared(klass, orig); + } + } } + + if (copy_encoding) rb_enc_cr_str_exact_copy(str, orig); OBJ_FREEZE(str); return str; } VALUE -rb_str_new5(obj, ptr, len) - VALUE obj; - const char *ptr; - long len; +rb_str_new_with_class(VALUE obj, const char *ptr, long len) { - return str_new(rb_obj_class(obj), ptr, len); + return str_enc_new(rb_obj_class(obj), ptr, len, STR_ENC_GET(obj)); } -#define STR_BUF_MIN_SIZE 128 +static VALUE +str_new_empty_String(VALUE str) +{ + VALUE v = rb_str_new(0, 0); + rb_enc_copy(v, str); + return v; +} + +#define STR_BUF_MIN_SIZE 63 VALUE -rb_str_buf_new(capa) - long capa; +rb_str_buf_new(long capa) { - VALUE str = str_alloc(rb_cString); - - if (capa < STR_BUF_MIN_SIZE) { - capa = STR_BUF_MIN_SIZE; + if (STR_EMBEDDABLE_P(capa, 1)) { + return str_alloc_embed(rb_cString, capa + 1); } - RSTRING(str)->ptr = 0; - RSTRING(str)->len = 0; - RSTRING(str)->aux.capa = capa; - RSTRING(str)->ptr = ALLOC_N(char, capa+1); - RSTRING(str)->ptr[0] = '\0'; + + VALUE str = str_alloc_heap(rb_cString); + + RSTRING(str)->as.heap.aux.capa = capa; + RSTRING(str)->as.heap.ptr = ALLOC_N(char, (size_t)capa + 1); + RSTRING(str)->as.heap.ptr[0] = '\0'; return str; } VALUE -rb_str_buf_new2(ptr) - const char *ptr; +rb_str_buf_new_cstr(const char *ptr) { VALUE str; long len = strlen(ptr); @@ -226,1741 +1724,4591 @@ rb_str_buf_new2(ptr) } VALUE -rb_str_to_str(str) - VALUE str; +rb_str_tmp_new(long len) { - return rb_convert_type(str, T_STRING, "String", "to_str"); + return str_new(0, 0, len); } -static void -rb_str_shared_replace(str, str2) - VALUE str, str2; +void +rb_str_free(VALUE str) { - if (str == str2) return; - rb_str_modify(str); - if (!FL_TEST(str, ELTS_SHARED)) free(RSTRING(str)->ptr); - if (NIL_P(str2)) { - RSTRING(str)->ptr = 0; - RSTRING(str)->len = 0; - RSTRING(str)->aux.capa = 0; - FL_UNSET(str, ELTS_SHARED|STR_ASSOC); - return; - } - RSTRING(str)->ptr = RSTRING(str2)->ptr; - RSTRING(str)->len = RSTRING(str2)->len; - FL_UNSET(str, ELTS_SHARED|STR_ASSOC); - if (FL_TEST(str2, ELTS_SHARED|STR_ASSOC)) { - FL_SET(str, RBASIC(str2)->flags & (ELTS_SHARED|STR_ASSOC)); - RSTRING(str)->aux.shared = RSTRING(str2)->aux.shared; + if (STR_EMBED_P(str)) { + RB_DEBUG_COUNTER_INC(obj_str_embed); + } + else if (FL_TEST(str, STR_SHARED | STR_NOFREE)) { + (void)RB_DEBUG_COUNTER_INC_IF(obj_str_shared, FL_TEST(str, STR_SHARED)); + (void)RB_DEBUG_COUNTER_INC_IF(obj_str_shared, FL_TEST(str, STR_NOFREE)); } else { - RSTRING(str)->aux.capa = RSTRING(str2)->aux.capa; + RB_DEBUG_COUNTER_INC(obj_str_ptr); + ruby_sized_xfree(STR_HEAP_PTR(str), STR_HEAP_SIZE(str)); } - RSTRING(str2)->ptr = 0; /* abandon str2 */ - RSTRING(str2)->len = 0; - RSTRING(str2)->aux.capa = 0; - FL_UNSET(str2, ELTS_SHARED|STR_ASSOC); - if (OBJ_TAINTED(str2)) OBJ_TAINT(str); } -static ID id_to_s; +size_t +rb_str_memsize(VALUE str) +{ + if (FL_TEST(str, STR_NOEMBED|STR_SHARED|STR_NOFREE) == STR_NOEMBED) { + return STR_HEAP_SIZE(str); + } + else { + return 0; + } +} VALUE -rb_obj_as_string(obj) - VALUE obj; +rb_str_to_str(VALUE str) +{ + return rb_convert_type_with_id(str, T_STRING, "String", idTo_str); +} + +static inline void str_discard(VALUE str); +static void str_shared_replace(VALUE str, VALUE str2); + +void +rb_str_shared_replace(VALUE str, VALUE str2) +{ + if (str != str2) str_shared_replace(str, str2); +} + +static void +str_shared_replace(VALUE str, VALUE str2) +{ + rb_encoding *enc; + int cr; + int termlen; + + RUBY_ASSERT(str2 != str); + enc = STR_ENC_GET(str2); + cr = ENC_CODERANGE(str2); + str_discard(str); + termlen = rb_enc_mbminlen(enc); + + STR_SET_LEN(str, RSTRING_LEN(str2)); + + if (str_embed_capa(str) >= RSTRING_LEN(str2) + termlen) { + STR_SET_EMBED(str); + memcpy(RSTRING_PTR(str), RSTRING_PTR(str2), (size_t)RSTRING_LEN(str2) + termlen); + rb_enc_associate(str, enc); + ENC_CODERANGE_SET(str, cr); + } + else { + if (STR_EMBED_P(str2)) { + RUBY_ASSERT(!FL_TEST(str2, STR_SHARED)); + long len = RSTRING_LEN(str2); + RUBY_ASSERT(len + termlen <= str_embed_capa(str2)); + + char *new_ptr = ALLOC_N(char, len + termlen); + memcpy(new_ptr, RSTRING(str2)->as.embed.ary, len + termlen); + RSTRING(str2)->as.heap.ptr = new_ptr; + STR_SET_LEN(str2, len); + RSTRING(str2)->as.heap.aux.capa = len; + STR_SET_NOEMBED(str2); + } + + STR_SET_NOEMBED(str); + FL_UNSET(str, STR_SHARED); + RSTRING(str)->as.heap.ptr = RSTRING_PTR(str2); + + if (FL_TEST(str2, STR_SHARED)) { + VALUE shared = RSTRING(str2)->as.heap.aux.shared; + STR_SET_SHARED(str, shared); + } + else { + RSTRING(str)->as.heap.aux.capa = RSTRING(str2)->as.heap.aux.capa; + } + + /* abandon str2 */ + STR_SET_EMBED(str2); + RSTRING_PTR(str2)[0] = 0; + STR_SET_LEN(str2, 0); + rb_enc_associate(str, enc); + ENC_CODERANGE_SET(str, cr); + } +} + +VALUE +rb_obj_as_string(VALUE obj) { VALUE str; - if (TYPE(obj) == T_STRING) { - return obj; + if (RB_TYPE_P(obj, T_STRING)) { + return obj; } - str = rb_funcall(obj, id_to_s, 0); - if (TYPE(str) != T_STRING) - return rb_any_to_s(obj); - if (OBJ_TAINTED(obj)) OBJ_TAINT(str); + str = rb_funcall(obj, idTo_s, 0); + return rb_obj_as_string_result(str, obj); +} + +VALUE +rb_obj_as_string_result(VALUE str, VALUE obj) +{ + if (!RB_TYPE_P(str, T_STRING)) + return rb_any_to_s(obj); return str; } -static VALUE rb_str_replace _((VALUE, VALUE)); +static VALUE +str_replace(VALUE str, VALUE str2) +{ + long len; -VALUE -rb_str_dup(str) - VALUE str; + len = RSTRING_LEN(str2); + if (STR_SHARED_P(str2)) { + VALUE shared = RSTRING(str2)->as.heap.aux.shared; + RUBY_ASSERT(OBJ_FROZEN(shared)); + STR_SET_NOEMBED(str); + STR_SET_LEN(str, len); + RSTRING(str)->as.heap.ptr = RSTRING_PTR(str2); + STR_SET_SHARED(str, shared); + rb_enc_cr_str_exact_copy(str, str2); + } + else { + str_replace_shared(str, str2); + } + + return str; +} + +static inline VALUE +ec_str_alloc_embed(struct rb_execution_context_struct *ec, VALUE klass, size_t capa) +{ + size_t size = rb_str_embed_size(capa, 0); + RUBY_ASSERT(size > 0); + RUBY_ASSERT(rb_gc_size_allocatable_p(size)); + + NEWOBJ_OF(str, struct RString, klass, + T_STRING | (RGENGC_WB_PROTECTED_STRING ? FL_WB_PROTECTED : 0), size, ec); + + str->len = 0; + + return (VALUE)str; +} + +static inline VALUE +ec_str_alloc_heap(struct rb_execution_context_struct *ec, VALUE klass) +{ + NEWOBJ_OF(str, struct RString, klass, + T_STRING | STR_NOEMBED | (RGENGC_WB_PROTECTED_STRING ? FL_WB_PROTECTED : 0), sizeof(struct RString), ec); + + str->as.heap.aux.capa = 0; + str->as.heap.ptr = NULL; + + return (VALUE)str; +} + +static inline VALUE +str_duplicate_setup_encoding(VALUE str, VALUE dup, VALUE flags) { - VALUE dup = str_alloc(rb_obj_class(str)); - rb_str_replace(dup, str); + int encidx = 0; + if ((flags & ENCODING_MASK) == (ENCODING_INLINE_MAX<<ENCODING_SHIFT)) { + encidx = rb_enc_get_index(str); + flags &= ~ENCODING_MASK; + } + FL_SET_RAW(dup, flags & ~FL_FREEZE); + if (encidx) rb_enc_associate_index(dup, encidx); return dup; } +static const VALUE flag_mask = ENC_CODERANGE_MASK | ENCODING_MASK | FL_FREEZE; + +static inline VALUE +str_duplicate_setup_embed(VALUE klass, VALUE str, VALUE dup) +{ + VALUE flags = FL_TEST_RAW(str, flag_mask); + long len = RSTRING_LEN(str); + + RUBY_ASSERT(STR_EMBED_P(dup)); + RUBY_ASSERT(str_embed_capa(dup) >= len + TERM_LEN(str)); + MEMCPY(RSTRING(dup)->as.embed.ary, RSTRING(str)->as.embed.ary, char, len + TERM_LEN(str)); + STR_SET_LEN(dup, RSTRING_LEN(str)); + return str_duplicate_setup_encoding(str, dup, flags); +} + +static inline VALUE +str_duplicate_setup_heap(VALUE klass, VALUE str, VALUE dup) +{ + VALUE flags = FL_TEST_RAW(str, flag_mask); + VALUE root = str; + if (FL_TEST_RAW(str, STR_SHARED)) { + root = RSTRING(str)->as.heap.aux.shared; + } + else if (UNLIKELY(!OBJ_FROZEN_RAW(str))) { + root = str = str_new_frozen(klass, str); + flags = FL_TEST_RAW(str, flag_mask); + } + RUBY_ASSERT(!STR_SHARED_P(root)); + RUBY_ASSERT(RB_OBJ_FROZEN_RAW(root)); + + RSTRING(dup)->as.heap.ptr = RSTRING_PTR(str); + FL_SET_RAW(dup, RSTRING_NOEMBED); + STR_SET_SHARED(dup, root); + flags |= RSTRING_NOEMBED | STR_SHARED; + + STR_SET_LEN(dup, RSTRING_LEN(str)); + return str_duplicate_setup_encoding(str, dup, flags); +} + +static inline VALUE +str_duplicate_setup(VALUE klass, VALUE str, VALUE dup) +{ + if (STR_EMBED_P(str)) { + return str_duplicate_setup_embed(klass, str, dup); + } + else { + return str_duplicate_setup_heap(klass, str, dup); + } +} + +static inline VALUE +str_duplicate(VALUE klass, VALUE str) +{ + VALUE dup; + if (STR_EMBED_P(str)) { + dup = str_alloc_embed(klass, RSTRING_LEN(str) + TERM_LEN(str)); + } + else { + dup = str_alloc_heap(klass); + } + + return str_duplicate_setup(klass, str, dup); +} + +VALUE +rb_str_dup(VALUE str) +{ + return str_duplicate(rb_obj_class(str), str); +} + +/* :nodoc: */ +VALUE +rb_str_dup_m(VALUE str) +{ + if (LIKELY(BARE_STRING_P(str))) { + return str_duplicate(rb_cString, str); + } + else { + return rb_obj_dup(str); + } +} + +VALUE +rb_str_resurrect(VALUE str) +{ + RUBY_DTRACE_CREATE_HOOK(STRING, RSTRING_LEN(str)); + return str_duplicate(rb_cString, str); +} + +VALUE +rb_ec_str_resurrect(struct rb_execution_context_struct *ec, VALUE str, bool chilled) +{ + RUBY_DTRACE_CREATE_HOOK(STRING, RSTRING_LEN(str)); + VALUE new_str, klass = rb_cString; + + if (!(chilled && RTEST(rb_ivar_defined(str, id_debug_created_info))) && STR_EMBED_P(str)) { + new_str = ec_str_alloc_embed(ec, klass, RSTRING_LEN(str) + TERM_LEN(str)); + str_duplicate_setup_embed(klass, str, new_str); + } + else { + new_str = ec_str_alloc_heap(ec, klass); + str_duplicate_setup_heap(klass, str, new_str); + } + if (chilled) { + FL_SET_RAW(new_str, STR_CHILLED_LITERAL); + } + return new_str; +} + +VALUE +rb_str_with_debug_created_info(VALUE str, VALUE path, int line) +{ + VALUE debug_info = rb_ary_new_from_args(2, path, INT2FIX(line)); + if (OBJ_FROZEN_RAW(str)) str = rb_str_dup(str); + rb_ivar_set(str, id_debug_created_info, rb_ary_freeze(debug_info)); + FL_SET_RAW(str, STR_CHILLED_LITERAL); + return rb_str_freeze(str); +} /* + * The documentation block below uses an include (instead of inline text) + * because the included text has non-ASCII characters (which are not allowed in a C file). + */ + +/* + * * call-seq: - * String.new(str="") => new_str - * - * Returns a new string object containing a copy of <i>str</i>. + * String.new(string = ''.encode(Encoding::ASCII_8BIT) , **options) -> new_string + * + * :include: doc/string/new.rdoc + * */ static VALUE -rb_str_init(argc, argv, str) - int argc; - VALUE *argv; - VALUE str; +rb_str_init(int argc, VALUE *argv, VALUE str) { - VALUE orig; + static ID keyword_ids[2]; + VALUE orig, opt, venc, vcapa; + VALUE kwargs[2]; + rb_encoding *enc = 0; + int n; + + if (!keyword_ids[0]) { + keyword_ids[0] = rb_id_encoding(); + CONST_ID(keyword_ids[1], "capacity"); + } + + n = rb_scan_args(argc, argv, "01:", &orig, &opt); + if (!NIL_P(opt)) { + rb_get_kwargs(opt, keyword_ids, 0, 2, kwargs); + venc = kwargs[0]; + vcapa = kwargs[1]; + if (!UNDEF_P(venc) && !NIL_P(venc)) { + enc = rb_to_encoding(venc); + } + if (!UNDEF_P(vcapa) && !NIL_P(vcapa)) { + long capa = NUM2LONG(vcapa); + long len = 0; + int termlen = enc ? rb_enc_mbminlen(enc) : 1; + + if (capa < STR_BUF_MIN_SIZE) { + capa = STR_BUF_MIN_SIZE; + } + if (n == 1) { + StringValue(orig); + len = RSTRING_LEN(orig); + if (capa < len) { + capa = len; + } + if (orig == str) n = 0; + } + str_modifiable(str); + if (STR_EMBED_P(str) || FL_TEST(str, STR_SHARED|STR_NOFREE)) { + /* make noembed always */ + const size_t size = (size_t)capa + termlen; + const char *const old_ptr = RSTRING_PTR(str); + const size_t osize = RSTRING_LEN(str) + TERM_LEN(str); + char *new_ptr = ALLOC_N(char, size); + if (STR_EMBED_P(str)) RUBY_ASSERT((long)osize <= str_embed_capa(str)); + memcpy(new_ptr, old_ptr, osize < size ? osize : size); + FL_UNSET_RAW(str, STR_SHARED|STR_NOFREE); + RSTRING(str)->as.heap.ptr = new_ptr; + } + else if (STR_HEAP_SIZE(str) != (size_t)capa + termlen) { + SIZED_REALLOC_N(RSTRING(str)->as.heap.ptr, char, + (size_t)capa + termlen, STR_HEAP_SIZE(str)); + } + STR_SET_LEN(str, len); + TERM_FILL(&RSTRING(str)->as.heap.ptr[len], termlen); + if (n == 1) { + memcpy(RSTRING(str)->as.heap.ptr, RSTRING_PTR(orig), len); + rb_enc_cr_str_exact_copy(str, orig); + } + FL_SET(str, STR_NOEMBED); + RSTRING(str)->as.heap.aux.capa = capa; + } + else if (n == 1) { + rb_str_replace(str, orig); + } + if (enc) { + rb_enc_associate(str, enc); + ENC_CODERANGE_CLEAR(str); + } + } + else if (n == 1) { + rb_str_replace(str, orig); + } + return str; +} + +/* :nodoc: */ +static VALUE +rb_str_s_new(int argc, VALUE *argv, VALUE klass) +{ + if (klass != rb_cString) { + return rb_class_new_instance_pass_kw(argc, argv, klass); + } + + static ID keyword_ids[2]; + VALUE orig, opt, encoding = Qnil, capacity = Qnil; + VALUE kwargs[2]; + rb_encoding *enc = NULL; + + int n = rb_scan_args(argc, argv, "01:", &orig, &opt); + if (NIL_P(opt)) { + return rb_class_new_instance_pass_kw(argc, argv, klass); + } + + keyword_ids[0] = rb_id_encoding(); + CONST_ID(keyword_ids[1], "capacity"); + rb_get_kwargs(opt, keyword_ids, 0, 2, kwargs); + encoding = kwargs[0]; + capacity = kwargs[1]; + + if (n == 1) { + orig = StringValue(orig); + } + else { + orig = Qnil; + } + + if (UNDEF_P(encoding)) { + if (!NIL_P(orig)) { + encoding = rb_obj_encoding(orig); + } + } + + if (!UNDEF_P(encoding)) { + enc = rb_to_encoding(encoding); + } + + // If capacity is nil, we're basically just duping `orig`. + if (UNDEF_P(capacity)) { + if (NIL_P(orig)) { + VALUE empty_str = str_new(klass, "", 0); + if (enc) { + rb_enc_associate(empty_str, enc); + } + return empty_str; + } + VALUE copy = str_duplicate(klass, orig); + rb_enc_associate(copy, enc); + ENC_CODERANGE_CLEAR(copy); + return copy; + } + + long capa = 0; + capa = NUM2LONG(capacity); + if (capa < 0) { + capa = 0; + } + + if (!NIL_P(orig)) { + long orig_capa = rb_str_capacity(orig); + if (orig_capa > capa) { + capa = orig_capa; + } + } + + VALUE str = str_enc_new(klass, NULL, capa, enc); + STR_SET_LEN(str, 0); + TERM_FILL(RSTRING_PTR(str), enc ? rb_enc_mbmaxlen(enc) : 1); + + if (!NIL_P(orig)) { + rb_str_buf_append(str, orig); + } - if (rb_scan_args(argc, argv, "01", &orig) == 1) - rb_str_replace(str, orig); return str; } +#ifdef NONASCII_MASK +#define is_utf8_lead_byte(c) (((c)&0xC0) != 0x80) + +/* + * UTF-8 leading bytes have either 0xxxxxxx or 11xxxxxx + * bit representation. (see https://en.wikipedia.org/wiki/UTF-8) + * Therefore, the following pseudocode can detect UTF-8 leading bytes. + * + * if (!(byte & 0x80)) + * byte |= 0x40; // turn on bit6 + * return ((byte>>6) & 1); // bit6 represent whether this byte is leading or not. + * + * This function calculates whether a byte is leading or not for all bytes + * in the argument word by concurrently using the above logic, and then + * adds up the number of leading bytes in the word. + */ +static inline uintptr_t +count_utf8_lead_bytes_with_word(const uintptr_t *s) +{ + uintptr_t d = *s; + + /* Transform so that bit0 indicates whether we have a UTF-8 leading byte or not. */ + d = (d>>6) | (~d>>7); + d &= NONASCII_MASK >> 7; + + /* Gather all bytes. */ +#if defined(HAVE_BUILTIN___BUILTIN_POPCOUNT) && defined(__POPCNT__) + /* use only if it can use POPCNT */ + return rb_popcount_intptr(d); +#else + d += (d>>8); + d += (d>>16); +# if SIZEOF_VOIDP == 8 + d += (d>>32); +# endif + return (d&0xF); +#endif +} +#endif + +static inline long +enc_strlen(const char *p, const char *e, rb_encoding *enc, int cr) +{ + long c; + const char *q; + + if (rb_enc_mbmaxlen(enc) == rb_enc_mbminlen(enc)) { + long diff = (long)(e - p); + return diff / rb_enc_mbminlen(enc) + !!(diff % rb_enc_mbminlen(enc)); + } +#ifdef NONASCII_MASK + else if (cr == ENC_CODERANGE_VALID && enc == rb_utf8_encoding()) { + uintptr_t len = 0; + if ((int)sizeof(uintptr_t) * 2 < e - p) { + const uintptr_t *s, *t; + const uintptr_t lowbits = sizeof(uintptr_t) - 1; + s = (const uintptr_t*)(~lowbits & ((uintptr_t)p + lowbits)); + t = (const uintptr_t*)(~lowbits & (uintptr_t)e); + while (p < (const char *)s) { + if (is_utf8_lead_byte(*p)) len++; + p++; + } + while (s < t) { + len += count_utf8_lead_bytes_with_word(s); + s++; + } + p = (const char *)s; + } + while (p < e) { + if (is_utf8_lead_byte(*p)) len++; + p++; + } + return (long)len; + } +#endif + else if (rb_enc_asciicompat(enc)) { + c = 0; + if (ENC_CODERANGE_CLEAN_P(cr)) { + while (p < e) { + if (ISASCII(*p)) { + q = search_nonascii(p, e); + if (!q) + return c + (e - p); + c += q - p; + p = q; + } + p += rb_enc_fast_mbclen(p, e, enc); + c++; + } + } + else { + while (p < e) { + if (ISASCII(*p)) { + q = search_nonascii(p, e); + if (!q) + return c + (e - p); + c += q - p; + p = q; + } + p += rb_enc_mbclen(p, e, enc); + c++; + } + } + return c; + } + + for (c=0; p<e; c++) { + p += rb_enc_mbclen(p, e, enc); + } + return c; +} + +long +rb_enc_strlen(const char *p, const char *e, rb_encoding *enc) +{ + return enc_strlen(p, e, enc, ENC_CODERANGE_UNKNOWN); +} + +/* To get strlen with cr + * Note that given cr is not used. + */ +long +rb_enc_strlen_cr(const char *p, const char *e, rb_encoding *enc, int *cr) +{ + long c; + const char *q; + int ret; + + *cr = 0; + if (rb_enc_mbmaxlen(enc) == rb_enc_mbminlen(enc)) { + long diff = (long)(e - p); + return diff / rb_enc_mbminlen(enc) + !!(diff % rb_enc_mbminlen(enc)); + } + else if (rb_enc_asciicompat(enc)) { + c = 0; + while (p < e) { + if (ISASCII(*p)) { + q = search_nonascii(p, e); + if (!q) { + if (!*cr) *cr = ENC_CODERANGE_7BIT; + return c + (e - p); + } + c += q - p; + p = q; + } + ret = rb_enc_precise_mbclen(p, e, enc); + if (MBCLEN_CHARFOUND_P(ret)) { + *cr |= ENC_CODERANGE_VALID; + p += MBCLEN_CHARFOUND_LEN(ret); + } + else { + *cr = ENC_CODERANGE_BROKEN; + p++; + } + c++; + } + if (!*cr) *cr = ENC_CODERANGE_7BIT; + return c; + } + + for (c=0; p<e; c++) { + ret = rb_enc_precise_mbclen(p, e, enc); + if (MBCLEN_CHARFOUND_P(ret)) { + *cr |= ENC_CODERANGE_VALID; + p += MBCLEN_CHARFOUND_LEN(ret); + } + else { + *cr = ENC_CODERANGE_BROKEN; + if (p + rb_enc_mbminlen(enc) <= e) + p += rb_enc_mbminlen(enc); + else + p = e; + } + } + if (!*cr) *cr = ENC_CODERANGE_7BIT; + return c; +} + +/* enc must be str's enc or rb_enc_check(str, str2) */ +static long +str_strlen(VALUE str, rb_encoding *enc) +{ + const char *p, *e; + int cr; + + if (single_byte_optimizable(str)) return RSTRING_LEN(str); + if (!enc) enc = STR_ENC_GET(str); + p = RSTRING_PTR(str); + e = RSTRING_END(str); + cr = ENC_CODERANGE(str); + + if (cr == ENC_CODERANGE_UNKNOWN) { + long n = rb_enc_strlen_cr(p, e, enc, &cr); + if (cr) ENC_CODERANGE_SET(str, cr); + return n; + } + else { + return enc_strlen(p, e, enc, cr); + } +} + +long +rb_str_strlen(VALUE str) +{ + return str_strlen(str, NULL); +} + /* * call-seq: - * str.length => integer - * - * Returns the length of <i>str</i>. + * length -> integer + * + * :include: doc/string/length.rdoc + * */ -static VALUE -rb_str_length(str) - VALUE str; +VALUE +rb_str_length(VALUE str) +{ + return LONG2NUM(str_strlen(str, NULL)); +} + +/* + * call-seq: + * bytesize -> integer + * + * :include: doc/string/bytesize.rdoc + * + */ + +VALUE +rb_str_bytesize(VALUE str) { - return LONG2NUM(RSTRING(str)->len); + return LONG2NUM(RSTRING_LEN(str)); } /* * call-seq: - * str.empty? => true or false - * - * Returns <code>true</code> if <i>str</i> has a length of zero. - * - * "hello".empty? #=> false - * "".empty? #=> true + * empty? -> true or false + * + * Returns whether the length of +self+ is zero: + * + * 'hello'.empty? # => false + * ' '.empty? # => false + * ''.empty? # => true + * + * Related: see {Querying}[rdoc-ref:String@Querying]. */ static VALUE -rb_str_empty(str) - VALUE str; +rb_str_empty(VALUE str) { - if (RSTRING(str)->len == 0) - return Qtrue; - return Qfalse; + return RBOOL(RSTRING_LEN(str) == 0); } /* * call-seq: - * str + other_str => new_str - * - * Concatenation---Returns a new <code>String</code> containing - * <i>other_str</i> concatenated to <i>str</i>. - * - * "Hello from " + self.to_s #=> "Hello from main" + * self + other_string -> new_string + * + * Returns a new string containing +other_string+ concatenated to +self+: + * + * 'Hello from ' + self.to_s # => "Hello from main" + * + * Related: see {Converting to New String}[rdoc-ref:String@Converting+to+New+String]. */ VALUE -rb_str_plus(str1, str2) - VALUE str1, str2; +rb_str_plus(VALUE str1, VALUE str2) { VALUE str3; + rb_encoding *enc; + char *ptr1, *ptr2, *ptr3; + long len1, len2; + int termlen; StringValue(str2); - str3 = rb_str_new(0, RSTRING(str1)->len+RSTRING(str2)->len); - memcpy(RSTRING(str3)->ptr, RSTRING(str1)->ptr, RSTRING(str1)->len); - memcpy(RSTRING(str3)->ptr + RSTRING(str1)->len, - RSTRING(str2)->ptr, RSTRING(str2)->len); - RSTRING(str3)->ptr[RSTRING(str3)->len] = '\0'; - - if (OBJ_TAINTED(str1) || OBJ_TAINTED(str2)) - OBJ_TAINT(str3); + enc = rb_enc_check_str(str1, str2); + RSTRING_GETMEM(str1, ptr1, len1); + RSTRING_GETMEM(str2, ptr2, len2); + termlen = rb_enc_mbminlen(enc); + if (len1 > LONG_MAX - len2) { + rb_raise(rb_eArgError, "string size too big"); + } + str3 = str_enc_new(rb_cString, 0, len1+len2, enc); + ptr3 = RSTRING_PTR(str3); + memcpy(ptr3, ptr1, len1); + memcpy(ptr3+len1, ptr2, len2); + TERM_FILL(&ptr3[len1+len2], termlen); + + ENCODING_CODERANGE_SET(str3, rb_enc_to_index(enc), + ENC_CODERANGE_AND(ENC_CODERANGE(str1), ENC_CODERANGE(str2))); + RB_GC_GUARD(str1); + RB_GC_GUARD(str2); return str3; } +/* A variant of rb_str_plus that does not raise but return Qundef instead. */ +VALUE +rb_str_opt_plus(VALUE str1, VALUE str2) +{ + RUBY_ASSERT(RBASIC_CLASS(str1) == rb_cString); + RUBY_ASSERT(RBASIC_CLASS(str2) == rb_cString); + long len1, len2; + MAYBE_UNUSED(char) *ptr1, *ptr2; + RSTRING_GETMEM(str1, ptr1, len1); + RSTRING_GETMEM(str2, ptr2, len2); + int enc1 = rb_enc_get_index(str1); + int enc2 = rb_enc_get_index(str2); + + if (enc1 < 0) { + return Qundef; + } + else if (enc2 < 0) { + return Qundef; + } + else if (enc1 != enc2) { + return Qundef; + } + else if (len1 > LONG_MAX - len2) { + return Qundef; + } + else { + return rb_str_plus(str1, str2); + } + +} + /* * call-seq: - * str * integer => new_str - * - * Copy---Returns a new <code>String</code> containing <i>integer</i> copies of - * the receiver. - * - * "Ho! " * 3 #=> "Ho! Ho! Ho! " + * self * n -> new_string + * + * Returns a new string containing +n+ copies of +self+: + * + * 'Ho!' * 3 # => "Ho!Ho!Ho!" + * 'No!' * 0 # => "" + * + * Related: see {Converting to New String}[rdoc-ref:String@Converting+to+New+String]. */ VALUE -rb_str_times(str, times) - VALUE str; - VALUE times; +rb_str_times(VALUE str, VALUE times) { VALUE str2; - long i, len; + long n, len; + char *ptr2; + int termlen; + if (times == INT2FIX(1)) { + return str_duplicate(rb_cString, str); + } + if (times == INT2FIX(0)) { + str2 = str_alloc_embed(rb_cString, 0); + rb_enc_copy(str2, str); + return str2; + } len = NUM2LONG(times); - if (len == 0) return rb_str_new5(str,0,0); if (len < 0) { - rb_raise(rb_eArgError, "negative argument"); + rb_raise(rb_eArgError, "negative argument"); } - if (LONG_MAX/len < RSTRING(str)->len) { - rb_raise(rb_eArgError, "argument too big"); + if (RSTRING_LEN(str) == 1 && RSTRING_PTR(str)[0] == 0) { + if (STR_EMBEDDABLE_P(len, 1)) { + str2 = str_alloc_embed(rb_cString, len + 1); + memset(RSTRING_PTR(str2), 0, len + 1); + } + else { + str2 = str_alloc_heap(rb_cString); + RSTRING(str2)->as.heap.aux.capa = len; + RSTRING(str2)->as.heap.ptr = ZALLOC_N(char, (size_t)len + 1); + } + STR_SET_LEN(str2, len); + rb_enc_copy(str2, str); + return str2; } - - str2 = rb_str_new5(str,0, RSTRING(str)->len*len); - for (i=0; i<len; i++) { - memcpy(RSTRING(str2)->ptr+(i*RSTRING(str)->len), - RSTRING(str)->ptr, RSTRING(str)->len); + if (len && LONG_MAX/len < RSTRING_LEN(str)) { + rb_raise(rb_eArgError, "argument too big"); } - RSTRING(str2)->ptr[RSTRING(str2)->len] = '\0'; - OBJ_INFECT(str2, str); + len *= RSTRING_LEN(str); + termlen = TERM_LEN(str); + str2 = str_enc_new(rb_cString, 0, len, STR_ENC_GET(str)); + ptr2 = RSTRING_PTR(str2); + if (len) { + n = RSTRING_LEN(str); + memcpy(ptr2, RSTRING_PTR(str), n); + while (n <= len/2) { + memcpy(ptr2 + n, ptr2, n); + n *= 2; + } + memcpy(ptr2 + n, ptr2, len-n); + } + STR_SET_LEN(str2, len); + TERM_FILL(&ptr2[len], termlen); + rb_enc_cr_str_copy_for_substr(str2, str); return str2; } /* * call-seq: - * str % arg => new_str - * - * Format---Uses <i>str</i> as a format specification, and returns the result - * of applying it to <i>arg</i>. If the format specification contains more than - * one substitution, then <i>arg</i> must be an <code>Array</code> containing - * the values to be substituted. See <code>Kernel::sprintf</code> for details - * of the format string. - * - * "%05d" % 123 #=> "00123" - * "%-5s: %08x" % [ "ID", self.id ] #=> "ID : 200e14d6" + * self % object -> new_string + * + * Returns the result of formatting +object+ into the format specifications + * contained in +self+ + * (see {Format Specifications}[rdoc-ref:language/format_specifications.rdoc]): + * + * '%05d' % 123 # => "00123" + * + * If +self+ contains multiple format specifications, + * +object+ must be an array or hash containing the objects to be formatted: + * + * '%-5s: %016x' % [ 'ID', self.object_id ] # => "ID : 00002b054ec93168" + * 'foo = %{foo}' % {foo: 'bar'} # => "foo = bar" + * 'foo = %{foo}, baz = %{baz}' % {foo: 'bar', baz: 'bat'} # => "foo = bar, baz = bat" + * + * Related: see {Converting to New String}[rdoc-ref:String@Converting+to+New+String]. */ static VALUE -rb_str_format(str, arg) - VALUE str, arg; +rb_str_format_m(VALUE str, VALUE arg) { - VALUE *argv; + VALUE tmp = rb_check_array_type(arg); - if (TYPE(arg) == T_ARRAY) { - argv = ALLOCA_N(VALUE, RARRAY(arg)->len + 1); - argv[0] = str; - MEMCPY(argv+1, RARRAY(arg)->ptr, VALUE, RARRAY(arg)->len); - return rb_f_sprintf(RARRAY(arg)->len+1, argv); + if (!NIL_P(tmp)) { + return rb_str_format(RARRAY_LENINT(tmp), RARRAY_CONST_PTR(tmp), str); } - - argv = ALLOCA_N(VALUE, 2); - argv[0] = str; - argv[1] = arg; - return rb_f_sprintf(2, argv); + return rb_str_format(1, &arg, str); } -static int -str_independent(str) - VALUE str; +static inline void +rb_check_lockedtmp(VALUE str) { - if (OBJ_FROZEN(str)) rb_error_frozen("string"); - if (!OBJ_TAINTED(str) && rb_safe_level() >= 4) - rb_raise(rb_eSecurityError, "Insecure: can't modify string"); - if (!FL_TEST(str, ELTS_SHARED)) return 1; - return 0; + if (FL_TEST(str, STR_TMPLOCK)) { + rb_raise(rb_eRuntimeError, "can't modify string; temporarily locked"); + } +} + +// If none of these flags are set, we know we have an modifiable string. +// If any is set, we need to do more detailed checks. +#define STR_UNMODIFIABLE_MASK (FL_FREEZE | STR_TMPLOCK | STR_CHILLED) +static inline void +str_modifiable(VALUE str) +{ + RUBY_ASSERT(ruby_thread_has_gvl_p()); + + if (RB_UNLIKELY(FL_ANY_RAW(str, STR_UNMODIFIABLE_MASK))) { + if (CHILLED_STRING_P(str)) { + CHILLED_STRING_MUTATED(str); + } + rb_check_lockedtmp(str); + rb_check_frozen(str); + } +} + +static inline int +str_dependent_p(VALUE str) +{ + if (STR_EMBED_P(str) || !FL_TEST(str, STR_SHARED|STR_NOFREE)) { + return FALSE; + } + else { + return TRUE; + } +} + +// If none of these flags are set, we know we have an independent string. +// If any is set, we need to do more detailed checks. +#define STR_DEPENDANT_MASK (STR_UNMODIFIABLE_MASK | STR_SHARED | STR_NOFREE) +static inline int +str_independent(VALUE str) +{ + RUBY_ASSERT(ruby_thread_has_gvl_p()); + + if (RB_UNLIKELY(FL_ANY_RAW(str, STR_DEPENDANT_MASK))) { + str_modifiable(str); + return !str_dependent_p(str); + } + return TRUE; } static void -str_make_independent(str) - VALUE str; +str_make_independent_expand(VALUE str, long len, long expand, const int termlen) { + RUBY_ASSERT(ruby_thread_has_gvl_p()); + char *ptr; + char *oldptr; + long capa = len + expand; - ptr = ALLOC_N(char, RSTRING(str)->len+1); - if (RSTRING(str)->ptr) { - memcpy(ptr, RSTRING(str)->ptr, RSTRING(str)->len); + if (len > capa) len = capa; + + if (!STR_EMBED_P(str) && str_embed_capa(str) >= capa + termlen) { + ptr = RSTRING(str)->as.heap.ptr; + STR_SET_EMBED(str); + memcpy(RSTRING(str)->as.embed.ary, ptr, len); + TERM_FILL(RSTRING(str)->as.embed.ary + len, termlen); + STR_SET_LEN(str, len); + return; + } + + ptr = ALLOC_N(char, (size_t)capa + termlen); + oldptr = RSTRING_PTR(str); + if (oldptr) { + memcpy(ptr, oldptr, len); + } + if (FL_TEST_RAW(str, STR_NOEMBED|STR_NOFREE|STR_SHARED) == STR_NOEMBED) { + xfree(oldptr); } - ptr[RSTRING(str)->len] = 0; - RSTRING(str)->ptr = ptr; - RSTRING(str)->aux.capa = RSTRING(str)->len; - FL_UNSET(str, ELTS_SHARED|STR_ASSOC); + STR_SET_NOEMBED(str); + FL_UNSET(str, STR_SHARED|STR_NOFREE); + TERM_FILL(ptr + len, termlen); + RSTRING(str)->as.heap.ptr = ptr; + STR_SET_LEN(str, len); + RSTRING(str)->as.heap.aux.capa = capa; } void -rb_str_modify(str) - VALUE str; +rb_str_modify(VALUE str) { if (!str_independent(str)) - str_make_independent(str); + str_make_independent(str); + ENC_CODERANGE_CLEAR(str); } void -rb_str_associate(str, add) - VALUE str, add; +rb_str_modify_expand(VALUE str, long expand) { - if (FL_TEST(str, STR_ASSOC)) { - /* already associated */ - rb_ary_concat(RSTRING(str)->aux.shared, add); + RUBY_ASSERT(ruby_thread_has_gvl_p()); + + int termlen = TERM_LEN(str); + long len = RSTRING_LEN(str); + + if (expand < 0) { + rb_raise(rb_eArgError, "negative expanding string size"); } - else { - if (FL_TEST(str, ELTS_SHARED)) { - str_make_independent(str); - } - else if (RSTRING(str)->aux.capa != RSTRING(str)->len) { - RESIZE_CAPA(str, RSTRING(str)->len); - } - RSTRING(str)->aux.shared = add; - FL_SET(str, STR_ASSOC); + if (expand >= LONG_MAX - len) { + rb_raise(rb_eArgError, "string size too big"); } + + if (!str_independent(str)) { + str_make_independent_expand(str, len, expand, termlen); + } + else if (expand > 0) { + RESIZE_CAPA_TERM(str, len + expand, termlen); + } + ENC_CODERANGE_CLEAR(str); } -VALUE -rb_str_associated(str) - VALUE str; +/* As rb_str_modify(), but don't clear coderange */ +static void +str_modify_keep_cr(VALUE str) { - if (FL_TEST(str, STR_ASSOC)) { - return RSTRING(str)->aux.shared; + if (!str_independent(str)) + str_make_independent(str); + if (ENC_CODERANGE(str) == ENC_CODERANGE_BROKEN) + /* Force re-scan later */ + ENC_CODERANGE_CLEAR(str); +} + +static inline void +str_discard(VALUE str) +{ + str_modifiable(str); + if (!STR_EMBED_P(str) && !FL_TEST(str, STR_SHARED|STR_NOFREE)) { + ruby_sized_xfree(STR_HEAP_PTR(str), STR_HEAP_SIZE(str)); + RSTRING(str)->as.heap.ptr = 0; + STR_SET_LEN(str, 0); } - return Qfalse; } -static char *null_str = ""; +void +rb_must_asciicompat(VALUE str) +{ + int encindex = rb_enc_get_index(str); + + if (RB_UNLIKELY(encindex == -1)) { + rb_raise(rb_eTypeError, "not encoding capable object"); + } + + if (RB_LIKELY(rb_str_encindex_fastpath(encindex))) { + return; + } + + rb_encoding *enc = rb_enc_from_index(encindex); + if (!rb_enc_asciicompat(enc)) { + rb_raise(rb_eEncCompatError, "ASCII incompatible encoding: %s", rb_enc_name(enc)); + } +} VALUE -rb_string_value(ptr) - volatile VALUE *ptr; +rb_string_value(volatile VALUE *ptr) { + RUBY_ASSERT(ruby_thread_has_gvl_p()); + VALUE s = *ptr; - if (TYPE(s) != T_STRING) { - s = rb_str_to_str(s); - *ptr = s; + if (!RB_TYPE_P(s, T_STRING)) { + s = rb_str_to_str(s); + *ptr = s; + } + return s; +} + +char * +rb_string_value_ptr(volatile VALUE *ptr) +{ + VALUE str = rb_string_value(ptr); + return RSTRING_PTR(str); +} + +static int +zero_filled(const char *s, int n) +{ + for (; n > 0; --n) { + if (*s++) return 0; + } + return 1; +} + +static const char * +str_null_char(const char *s, long len, const int minlen, rb_encoding *enc) +{ + const char *e = s + len; + + for (; s + minlen <= e; s += rb_enc_mbclen(s, e, enc)) { + if (zero_filled(s, minlen)) return s; + } + return 0; +} + +static char * +str_fill_term(VALUE str, char *s, long len, int termlen) +{ + /* This function assumes that (capa + termlen) bytes of memory + * is allocated, like many other functions in this file. + */ + if (str_dependent_p(str)) { + if (!zero_filled(s + len, termlen)) + str_make_independent_expand(str, len, 0L, termlen); + } + else { + TERM_FILL(s + len, termlen); + return s; + } + return RSTRING_PTR(str); +} + +void +rb_str_change_terminator_length(VALUE str, const int oldtermlen, const int termlen) +{ + long capa = str_capacity(str, oldtermlen) + oldtermlen; + long len = RSTRING_LEN(str); + + RUBY_ASSERT(capa >= len); + if (capa - len < termlen) { + rb_check_lockedtmp(str); + str_make_independent_expand(str, len, 0L, termlen); + } + else if (str_dependent_p(str)) { + if (termlen > oldtermlen) + str_make_independent_expand(str, len, 0L, termlen); + } + else { + if (!STR_EMBED_P(str)) { + /* modify capa instead of realloc */ + RUBY_ASSERT(!FL_TEST((str), STR_SHARED)); + RSTRING(str)->as.heap.aux.capa = capa - termlen; + } + if (termlen > oldtermlen) { + TERM_FILL(RSTRING_PTR(str) + len, termlen); + } + } + + return; +} + +static char * +str_null_check(VALUE str, int *w) +{ + char *s = RSTRING_PTR(str); + long len = RSTRING_LEN(str); + int minlen = 1; + + if (RB_UNLIKELY(!rb_str_enc_fastpath(str))) { + rb_encoding *enc = rb_str_enc_get(str); + minlen = rb_enc_mbminlen(enc); + + if (minlen > 1) { + *w = 1; + if (str_null_char(s, len, minlen, enc)) { + return NULL; + } + return str_fill_term(str, s, len, minlen); + } + } + + *w = 0; + if (!s || memchr(s, 0, len)) { + return NULL; } - if (!RSTRING(s)->ptr) { - FL_SET(s, ELTS_SHARED); - RSTRING(s)->ptr = null_str; + if (s[len]) { + s = str_fill_term(str, s, len, minlen); } return s; } +const char * +rb_str_null_check(VALUE str) +{ + RUBY_ASSERT(RB_TYPE_P(str, T_STRING)); + + char *s; + long len; + RSTRING_GETMEM(str, s, len); + + if (RB_LIKELY(rb_str_enc_fastpath(str))) { + if (!s || memchr(s, 0, len)) { + rb_raise(rb_eArgError, "string contains null byte"); + } + } + else { + int w; + const char *s = str_null_check(str, &w); + if (!s) { + if (w) { + rb_raise(rb_eArgError, "string contains null char"); + } + rb_raise(rb_eArgError, "string contains null byte"); + } + } + + return s; +} + char * -rb_string_value_ptr(ptr) - volatile VALUE *ptr; +rb_str_to_cstr(VALUE str) { - return RSTRING(rb_string_value(ptr))->ptr; + int w; + return str_null_check(str, &w); } char * -rb_string_value_cstr(ptr) - volatile VALUE *ptr; +rb_string_value_cstr(volatile VALUE *ptr) { VALUE str = rb_string_value(ptr); - char *s = RSTRING(str)->ptr; - - if (!s || RSTRING(str)->len != strlen(s)) { - rb_raise(rb_eArgError, "string contains null byte"); + int w; + char *s = str_null_check(str, &w); + if (!s) { + if (w) { + rb_raise(rb_eArgError, "string contains null char"); + } + rb_raise(rb_eArgError, "string contains null byte"); } return s; } +char * +rb_str_fill_terminator(VALUE str, const int newminlen) +{ + char *s = RSTRING_PTR(str); + long len = RSTRING_LEN(str); + return str_fill_term(str, s, len, newminlen); +} + VALUE -rb_check_string_type(str) - VALUE str; +rb_check_string_type(VALUE str) { - str = rb_check_convert_type(str, T_STRING, "String", "to_str"); - if (!NIL_P(str) && !RSTRING(str)->ptr) { - FL_SET(str, ELTS_SHARED); - RSTRING(str)->ptr = null_str; - } + str = rb_check_convert_type_with_id(str, T_STRING, "String", idTo_str); return str; } -VALUE -rb_str_substr(str, beg, len) - VALUE str; - long beg, len; +/* + * call-seq: + * String.try_convert(object) -> object, new_string, or nil + * + * Attempts to convert the given +object+ to a string. + * + * If +object+ is already a string, returns +object+, unmodified. + * + * Otherwise if +object+ responds to <tt>:to_str</tt>, + * calls <tt>object.to_str</tt> and returns the result. + * + * Returns +nil+ if +object+ does not respond to <tt>:to_str</tt>. + * + * Raises an exception unless <tt>object.to_str</tt> returns a string. + */ +static VALUE +rb_str_s_try_convert(VALUE dummy, VALUE str) +{ + return rb_check_string_type(str); +} + +static char* +str_nth_len(const char *p, const char *e, long *nthp, rb_encoding *enc) +{ + long nth = *nthp; + if (rb_enc_mbmaxlen(enc) == 1) { + p += nth; + } + else if (rb_enc_mbmaxlen(enc) == rb_enc_mbminlen(enc)) { + p += nth * rb_enc_mbmaxlen(enc); + } + else if (rb_enc_asciicompat(enc)) { + const char *p2, *e2; + int n; + + while (p < e && 0 < nth) { + e2 = p + nth; + if (e < e2) { + *nthp = nth; + return (char *)e; + } + if (ISASCII(*p)) { + p2 = search_nonascii(p, e2); + if (!p2) { + nth -= e2 - p; + *nthp = nth; + return (char *)e2; + } + nth -= p2 - p; + p = p2; + } + n = rb_enc_mbclen(p, e, enc); + p += n; + nth--; + } + *nthp = nth; + if (nth != 0) { + return (char *)e; + } + return (char *)p; + } + else { + while (p < e && nth--) { + p += rb_enc_mbclen(p, e, enc); + } + } + if (p > e) p = e; + *nthp = nth; + return (char*)p; +} + +char* +rb_enc_nth(const char *p, const char *e, long nth, rb_encoding *enc) +{ + return str_nth_len(p, e, &nth, enc); +} + +static char* +str_nth(const char *p, const char *e, long nth, rb_encoding *enc, int singlebyte) +{ + if (singlebyte) + p += nth; + else { + p = str_nth_len(p, e, &nth, enc); + } + if (!p) return 0; + if (p > e) p = e; + return (char *)p; +} + +/* char offset to byte offset */ +static long +str_offset(const char *p, const char *e, long nth, rb_encoding *enc, int singlebyte) +{ + const char *pp = str_nth(p, e, nth, enc, singlebyte); + if (!pp) return e - p; + return pp - p; +} + +long +rb_str_offset(VALUE str, long pos) +{ + return str_offset(RSTRING_PTR(str), RSTRING_END(str), pos, + STR_ENC_GET(str), single_byte_optimizable(str)); +} + +#ifdef NONASCII_MASK +static char * +str_utf8_nth(const char *p, const char *e, long *nthp) +{ + long nth = *nthp; + if ((int)SIZEOF_VOIDP * 2 < e - p && (int)SIZEOF_VOIDP * 2 < nth) { + const uintptr_t *s, *t; + const uintptr_t lowbits = SIZEOF_VOIDP - 1; + s = (const uintptr_t*)(~lowbits & ((uintptr_t)p + lowbits)); + t = (const uintptr_t*)(~lowbits & (uintptr_t)e); + while (p < (const char *)s) { + if (is_utf8_lead_byte(*p)) nth--; + p++; + } + do { + nth -= count_utf8_lead_bytes_with_word(s); + s++; + } while (s < t && (int)SIZEOF_VOIDP <= nth); + p = (char *)s; + } + while (p < e) { + if (is_utf8_lead_byte(*p)) { + if (nth == 0) break; + nth--; + } + p++; + } + *nthp = nth; + return (char *)p; +} + +static long +str_utf8_offset(const char *p, const char *e, long nth) +{ + const char *pp = str_utf8_nth(p, e, &nth); + return pp - p; +} +#endif + +/* byte offset to char offset */ +long +rb_str_sublen(VALUE str, long pos) +{ + if (single_byte_optimizable(str) || pos < 0) + return pos; + else { + char *p = RSTRING_PTR(str); + return enc_strlen(p, p + pos, STR_ENC_GET(str), ENC_CODERANGE(str)); + } +} + +static VALUE +str_subseq(VALUE str, long beg, long len) { VALUE str2; - if (len < 0) return Qnil; - if (beg > RSTRING(str)->len) return Qnil; + RUBY_ASSERT(beg >= 0); + RUBY_ASSERT(len >= 0); + RUBY_ASSERT(beg+len <= RSTRING_LEN(str)); + + const int termlen = TERM_LEN(str); + if (!SHARABLE_SUBSTRING_P(beg, len, RSTRING_LEN(str))) { + str2 = rb_str_new(RSTRING_PTR(str) + beg, len); + RB_GC_GUARD(str); + return str2; + } + + str2 = str_alloc_heap(rb_cString); + if (str_embed_capa(str2) >= len + termlen) { + char *ptr2 = RSTRING(str2)->as.embed.ary; + STR_SET_EMBED(str2); + memcpy(ptr2, RSTRING_PTR(str) + beg, len); + TERM_FILL(ptr2+len, termlen); + + STR_SET_LEN(str2, len); + RB_GC_GUARD(str); + } + else { + str_replace_shared(str2, str); + RUBY_ASSERT(!STR_EMBED_P(str2)); + ENC_CODERANGE_CLEAR(str2); + RSTRING(str2)->as.heap.ptr += beg; + if (RSTRING_LEN(str2) > len) { + STR_SET_LEN(str2, len); + } + } + + return str2; +} + +VALUE +rb_str_subseq(VALUE str, long beg, long len) +{ + VALUE str2 = str_subseq(str, beg, len); + rb_enc_cr_str_copy_for_substr(str2, str); + return str2; +} + +char * +rb_str_subpos(VALUE str, long beg, long *lenp) +{ + long len = *lenp; + long slen = -1L; + const long blen = RSTRING_LEN(str); + rb_encoding *enc = STR_ENC_GET(str); + char *p, *s = RSTRING_PTR(str), *e = s + blen; + + if (len < 0) return 0; + if (beg < 0 && -beg < 0) return 0; + if (!blen) { + len = 0; + } + if (single_byte_optimizable(str)) { + if (beg > blen) return 0; + if (beg < 0) { + beg += blen; + if (beg < 0) return 0; + } + if (len > blen - beg) + len = blen - beg; + if (len < 0) return 0; + p = s + beg; + goto end; + } if (beg < 0) { - beg += RSTRING(str)->len; - if (beg < 0) return Qnil; + if (len > -beg) len = -beg; + if ((ENC_CODERANGE(str) == ENC_CODERANGE_VALID) && + (-beg * rb_enc_mbmaxlen(enc) < blen / 8)) { + beg = -beg; + while (beg-- > len && (e = rb_enc_prev_char(s, e, e, enc)) != 0); + p = e; + if (!p) return 0; + while (len-- > 0 && (p = rb_enc_prev_char(s, p, e, enc)) != 0); + if (!p) return 0; + len = e - p; + goto end; + } + else { + slen = str_strlen(str, enc); + beg += slen; + if (beg < 0) return 0; + p = s + beg; + if (len == 0) goto end; + } } - if (beg + len > RSTRING(str)->len) { - len = RSTRING(str)->len - beg; + else if (beg > 0 && beg > blen) { + return 0; } - if (len < 0) { - len = 0; + if (len == 0) { + if (beg > str_strlen(str, enc)) return 0; /* str's enc */ + p = s + beg; } - if (len == 0) return rb_str_new5(str,0,0); +#ifdef NONASCII_MASK + else if (ENC_CODERANGE(str) == ENC_CODERANGE_VALID && + enc == rb_utf8_encoding()) { + p = str_utf8_nth(s, e, &beg); + if (beg > 0) return 0; + len = str_utf8_offset(p, e, len); + } +#endif + else if (rb_enc_mbmaxlen(enc) == rb_enc_mbminlen(enc)) { + int char_sz = rb_enc_mbmaxlen(enc); - if (len > sizeof(struct RString)/2 && - beg + len == RSTRING(str)->len && !FL_TEST(str, STR_ASSOC)) { - str2 = rb_str_new3(rb_str_new4(str)); - RSTRING(str2)->ptr += RSTRING(str2)->len - len; - RSTRING(str2)->len = len; + p = s + beg * char_sz; + if (p > e) { + return 0; + } + else if (len * char_sz > e - p) + len = e - p; + else + len *= char_sz; + } + else if ((p = str_nth_len(s, e, &beg, enc)) == e) { + if (beg > 0) return 0; + len = 0; } else { - str2 = rb_str_new5(str, RSTRING(str)->ptr+beg, len); + len = str_offset(p, e, len, enc, 0); } - OBJ_INFECT(str2, str); + end: + *lenp = len; + RB_GC_GUARD(str); + return p; +} +static VALUE str_substr(VALUE str, long beg, long len, int empty); + +VALUE +rb_str_substr(VALUE str, long beg, long len) +{ + return str_substr(str, beg, len, TRUE); +} + +VALUE +rb_str_substr_two_fixnums(VALUE str, VALUE beg, VALUE len, int empty) +{ + return str_substr(str, NUM2LONG(beg), NUM2LONG(len), empty); +} + +static VALUE +str_substr(VALUE str, long beg, long len, int empty) +{ + char *p = rb_str_subpos(str, beg, &len); + + if (!p) return Qnil; + if (!len && !empty) return Qnil; + + beg = p - RSTRING_PTR(str); + + VALUE str2 = str_subseq(str, beg, len); + rb_enc_cr_str_copy_for_substr(str2, str); return str2; } +/* :nodoc: */ VALUE -rb_str_freeze(str) - VALUE str; +rb_str_freeze(VALUE str) { + if (CHILLED_STRING_P(str)) { + FL_UNSET_RAW(str, STR_CHILLED); + } + + if (OBJ_FROZEN(str)) return str; + rb_str_resize(str, RSTRING_LEN(str)); return rb_obj_freeze(str); } +/* + * call-seq: + * +string -> new_string or self + * + * Returns +self+ if +self+ is not frozen and can be mutated + * without warning issuance. + * + * Otherwise returns <tt>self.dup</tt>, which is not frozen. + * + * Related: see {Freezing/Unfreezing}[rdoc-ref:String@Freezing-2FUnfreezing]. + */ +static VALUE +str_uplus(VALUE str) +{ + if (OBJ_FROZEN(str) || CHILLED_STRING_P(str)) { + return rb_str_dup(str); + } + else { + return str; + } +} + +/* + * call-seq: + * -self -> frozen_string + * + * Returns a frozen string equal to +self+. + * + * The returned string is +self+ if and only if all of the following are true: + * + * - +self+ is already frozen. + * - +self+ is an instance of \String (rather than of a subclass of \String) + * - +self+ has no instance variables set on it. + * + * Otherwise, the returned string is a frozen copy of +self+. + * + * Returning +self+, when possible, saves duplicating +self+; + * see {Data deduplication}[https://en.wikipedia.org/wiki/Data_deduplication]. + * + * It may also save duplicating other, already-existing, strings: + * + * s0 = 'foo' + * s1 = 'foo' + * s0.object_id == s1.object_id # => false + * (-s0).object_id == (-s1).object_id # => true + * + * Note that method #-@ is convenient for defining a constant: + * + * FileName = -'config/database.yml' + * + * While its alias #dedup is better suited for chaining: + * + * 'foo'.dedup.gsub!('o') + * + * Related: see {Freezing/Unfreezing}[rdoc-ref:String@Freezing-2FUnfreezing]. + */ +static VALUE +str_uminus(VALUE str) +{ + if (!BARE_STRING_P(str) && !rb_obj_frozen_p(str)) { + str = rb_str_dup(str); + } + return rb_fstring(str); +} + +RUBY_ALIAS_FUNCTION(rb_str_dup_frozen(VALUE str), rb_str_new_frozen, (str)) +#define rb_str_dup_frozen rb_str_new_frozen + VALUE -rb_str_dup_frozen(str) - VALUE str; +rb_str_locktmp(VALUE str) { - if (FL_TEST(str, ELTS_SHARED) && RSTRING(str)->aux.shared) { - VALUE shared = RSTRING(str)->aux.shared; - if (RSTRING(shared)->len == RSTRING(str)->len) { - OBJ_FREEZE(shared); - return shared; - } + rb_check_frozen(str); + if (FL_TEST(str, STR_TMPLOCK)) { + rb_raise(rb_eRuntimeError, "temporal locking already locked string"); } - if (OBJ_FROZEN(str)) return str; - str = rb_str_dup(str); - OBJ_FREEZE(str); + FL_SET(str, STR_TMPLOCK); return str; } VALUE -rb_str_resize(str, len) - VALUE str; - long len; +rb_str_unlocktmp(VALUE str) { - if (len < 0) { - rb_raise(rb_eArgError, "negative string size (or size too big)"); - } - - if (len != RSTRING(str)->len) { - rb_str_modify(str); - if (RSTRING(str)->len < len || RSTRING(str)->len - len > 1024) { - REALLOC_N(RSTRING(str)->ptr, char, len+1); - if (!FL_TEST(str, STR_ASSOC|ELTS_SHARED)) { - RSTRING(str)->aux.capa = len; - } - } - RSTRING(str)->len = len; - RSTRING(str)->ptr[len] = '\0'; /* sentinel */ + rb_check_frozen(str); + if (!FL_TEST(str, STR_TMPLOCK)) { + rb_raise(rb_eRuntimeError, "temporal unlocking already unlocked string"); } + FL_UNSET(str, STR_TMPLOCK); return str; } VALUE -rb_str_buf_cat(str, ptr, len) - VALUE str; - const char *ptr; - long len; +rb_str_locktmp_ensure(VALUE str, VALUE (*func)(VALUE), VALUE arg) +{ + rb_str_locktmp(str); + return rb_ensure(func, arg, rb_str_unlocktmp, str); +} + +void +rb_str_set_len(VALUE str, long len) { - long capa, total; + RUBY_ASSERT(ruby_thread_has_gvl_p()); - if (len == 0) return str; + long capa; + const int termlen = TERM_LEN(str); + + str_modifiable(str); + if (STR_SHARED_P(str)) { + rb_raise(rb_eRuntimeError, "can't set length of shared string"); + } + if (len > (capa = (long)str_capacity(str, termlen)) || len < 0) { + rb_bug("probable buffer overflow: %ld for %ld", len, capa); + } + + int cr = ENC_CODERANGE(str); + if (len == 0) { + /* Empty string does not contain non-ASCII */ + ENC_CODERANGE_SET(str, ENC_CODERANGE_7BIT); + } + else if (cr == ENC_CODERANGE_UNKNOWN) { + /* Leave unknown. */ + } + else if (len > RSTRING_LEN(str)) { + if (ENC_CODERANGE_CLEAN_P(cr)) { + /* Update the coderange regarding the extended part. */ + const char *const prev_end = RSTRING_END(str); + const char *const new_end = RSTRING_PTR(str) + len; + rb_encoding *enc = rb_enc_get(str); + rb_str_coderange_scan_restartable(prev_end, new_end, enc, &cr); + ENC_CODERANGE_SET(str, cr); + } + else if (cr == ENC_CODERANGE_BROKEN) { + /* May be valid now, by appended part. */ + ENC_CODERANGE_SET(str, ENC_CODERANGE_UNKNOWN); + } + } + else if (len < RSTRING_LEN(str)) { + if (cr != ENC_CODERANGE_7BIT) { + /* ASCII-only string is keeping after truncated. Valid + * and broken may be invalid or valid, leave unknown. */ + ENC_CODERANGE_SET(str, ENC_CODERANGE_UNKNOWN); + } + } + + STR_SET_LEN(str, len); + TERM_FILL(&RSTRING_PTR(str)[len], termlen); +} + +VALUE +rb_str_resize(VALUE str, long len) +{ if (len < 0) { - rb_raise(rb_eArgError, "negative string size (or size too big)"); + rb_raise(rb_eArgError, "negative string size (or size too big)"); } - rb_str_modify(str); - if (FL_TEST(str, STR_ASSOC)) { - FL_UNSET(str, STR_ASSOC); - capa = RSTRING(str)->aux.capa = RSTRING(str)->len; + + int independent = str_independent(str); + long slen = RSTRING_LEN(str); + const int termlen = TERM_LEN(str); + + if (slen > len || (termlen != 1 && slen < len)) { + ENC_CODERANGE_CLEAR(str); + } + + { + long capa; + if (STR_EMBED_P(str)) { + if (len == slen) return str; + if (str_embed_capa(str) >= len + termlen) { + STR_SET_LEN(str, len); + TERM_FILL(RSTRING(str)->as.embed.ary + len, termlen); + return str; + } + str_make_independent_expand(str, slen, len - slen, termlen); + } + else if (str_embed_capa(str) >= len + termlen) { + char *ptr = STR_HEAP_PTR(str); + STR_SET_EMBED(str); + if (slen > len) slen = len; + if (slen > 0) MEMCPY(RSTRING(str)->as.embed.ary, ptr, char, slen); + TERM_FILL(RSTRING(str)->as.embed.ary + len, termlen); + STR_SET_LEN(str, len); + if (independent) ruby_xfree(ptr); + return str; + } + else if (!independent) { + if (len == slen) return str; + str_make_independent_expand(str, slen, len - slen, termlen); + } + else if ((capa = RSTRING(str)->as.heap.aux.capa) < len || + (capa - len) > (len < 1024 ? len : 1024)) { + SIZED_REALLOC_N(RSTRING(str)->as.heap.ptr, char, + (size_t)len + termlen, STR_HEAP_SIZE(str)); + RSTRING(str)->as.heap.aux.capa = len; + } + else if (len == slen) return str; + STR_SET_LEN(str, len); + TERM_FILL(RSTRING(str)->as.heap.ptr + len, termlen); /* sentinel */ + } + return str; +} + +static void +str_ensure_available_capa(VALUE str, long len) +{ + str_modify_keep_cr(str); + + const int termlen = TERM_LEN(str); + long olen = RSTRING_LEN(str); + + if (RB_UNLIKELY(olen > LONG_MAX - len)) { + rb_raise(rb_eArgError, "string sizes too big"); + } + + long total = olen + len; + long capa = str_capacity(str, termlen); + + if (capa < total) { + if (total >= LONG_MAX / 2) { + capa = total; + } + while (total > capa) { + capa = 2 * capa + termlen; /* == 2*(capa+termlen)-termlen */ + } + RESIZE_CAPA_TERM(str, capa, termlen); + } +} + +static VALUE +str_buf_cat4(VALUE str, const char *ptr, long len, bool keep_cr) +{ + if (keep_cr) { + str_modify_keep_cr(str); } else { - capa = RSTRING(str)->aux.capa; + rb_str_modify(str); } - total = RSTRING(str)->len+len; - if (capa <= total) { - while (total > capa) { - capa = (capa + 1) * 2; - } - RESIZE_CAPA(str, capa); + if (len == 0) return 0; + + long total, olen, off = -1; + char *sptr; + const int termlen = TERM_LEN(str); + + RSTRING_GETMEM(str, sptr, olen); + if (ptr >= sptr && ptr <= sptr + olen) { + off = ptr - sptr; } - memcpy(RSTRING(str)->ptr + RSTRING(str)->len, ptr, len); - RSTRING(str)->len = total; - RSTRING(str)->ptr[total] = '\0'; /* sentinel */ + + long capa = str_capacity(str, termlen); + + if (olen > LONG_MAX - len) { + rb_raise(rb_eArgError, "string sizes too big"); + } + total = olen + len; + if (capa < total) { + if (total >= LONG_MAX / 2) { + capa = total; + } + while (total > capa) { + capa = 2 * capa + termlen; /* == 2*(capa+termlen)-termlen */ + } + RESIZE_CAPA_TERM(str, capa, termlen); + sptr = RSTRING_PTR(str); + } + if (off != -1) { + ptr = sptr + off; + } + memcpy(sptr + olen, ptr, len); + STR_SET_LEN(str, total); + TERM_FILL(sptr + total, termlen); /* sentinel */ return str; } +#define str_buf_cat(str, ptr, len) str_buf_cat4((str), (ptr), len, false) +#define str_buf_cat2(str, ptr) str_buf_cat4((str), (ptr), rb_strlen_lit(ptr), false) + VALUE -rb_str_buf_cat2(str, ptr) - VALUE str; - const char *ptr; +rb_str_cat(VALUE str, const char *ptr, long len) { - return rb_str_buf_cat(str, ptr, strlen(ptr)); + if (len == 0) return str; + if (len < 0) { + rb_raise(rb_eArgError, "negative string size (or size too big)"); + } + return str_buf_cat(str, ptr, len); } VALUE -rb_str_cat(str, ptr, len) - VALUE str; - const char *ptr; - long len; +rb_str_cat_cstr(VALUE str, const char *ptr) { - if (len < 0) { - rb_raise(rb_eArgError, "negative string size (or size too big)"); + must_not_null(ptr); + return rb_str_buf_cat(str, ptr, strlen(ptr)); +} + +static void +rb_str_buf_cat_byte(VALUE str, unsigned char byte) +{ + RUBY_ASSERT(RB_ENCODING_GET_INLINED(str) == ENCINDEX_ASCII_8BIT || RB_ENCODING_GET_INLINED(str) == ENCINDEX_US_ASCII); + + // We can't write directly to shared strings without impacting others, so we must make the string independent. + if (UNLIKELY(!str_independent(str))) { + str_make_independent(str); } - if (FL_TEST(str, STR_ASSOC)) { - rb_str_modify(str); - REALLOC_N(RSTRING(str)->ptr, char, RSTRING(str)->len+len); - memcpy(RSTRING(str)->ptr + RSTRING(str)->len, ptr, len); - RSTRING(str)->len += len; - RSTRING(str)->ptr[RSTRING(str)->len] = '\0'; /* sentinel */ - return str; + + long string_length = -1; + const int null_terminator_length = 1; + char *sptr; + RSTRING_GETMEM(str, sptr, string_length); + + // Ensure the resulting string wouldn't be too long. + if (UNLIKELY(string_length > LONG_MAX - 1)) { + rb_raise(rb_eArgError, "string sizes too big"); } - return rb_str_buf_cat(str, ptr, len); + long string_capacity = str_capacity(str, null_terminator_length); + + // Get the code range before any modifications since those might clear the code range. + int cr = ENC_CODERANGE(str); + + // Check if the string has spare string_capacity to write the new byte. + if (LIKELY(string_capacity >= string_length + 1)) { + // In fast path we can write the new byte and note the string's new length. + sptr[string_length] = byte; + STR_SET_LEN(str, string_length + 1); + TERM_FILL(sptr + string_length + 1, null_terminator_length); + } + else { + // If there's not enough string_capacity, make a call into the general string concatenation function. + str_buf_cat(str, (char *)&byte, 1); + } + + // If the code range is already known, we can derive the resulting code range cheaply by looking at the byte we + // just appended. If the code range is unknown, but the string was empty, then we can also derive the code range + // by looking at the byte we just appended. Otherwise, we'd have to scan the bytes to determine the code range so + // we leave it as unknown. It cannot be broken for binary strings so we don't need to handle that option. + if (cr == ENC_CODERANGE_7BIT || string_length == 0) { + if (ISASCII(byte)) { + ENC_CODERANGE_SET(str, ENC_CODERANGE_7BIT); + } + else { + ENC_CODERANGE_SET(str, ENC_CODERANGE_VALID); + + // Promote a US-ASCII string to ASCII-8BIT when a non-ASCII byte is appended. + if (UNLIKELY(RB_ENCODING_GET_INLINED(str) == ENCINDEX_US_ASCII)) { + rb_enc_associate_index(str, ENCINDEX_ASCII_8BIT); + } + } + } } -VALUE -rb_str_cat2(str, ptr) - VALUE str; - const char *ptr; +RUBY_ALIAS_FUNCTION(rb_str_buf_cat(VALUE str, const char *ptr, long len), rb_str_cat, (str, ptr, len)) +RUBY_ALIAS_FUNCTION(rb_str_buf_cat2(VALUE str, const char *ptr), rb_str_cat_cstr, (str, ptr)) +RUBY_ALIAS_FUNCTION(rb_str_cat2(VALUE str, const char *ptr), rb_str_cat_cstr, (str, ptr)) + +static VALUE +rb_enc_cr_str_buf_cat(VALUE str, const char *ptr, long len, + int ptr_encindex, int ptr_cr, int *ptr_cr_ret) { - return rb_str_cat(str, ptr, strlen(ptr)); + int str_encindex = ENCODING_GET(str); + int res_encindex; + int str_cr, res_cr; + rb_encoding *str_enc, *ptr_enc; + + str_cr = RSTRING_LEN(str) ? ENC_CODERANGE(str) : ENC_CODERANGE_7BIT; + + if (str_encindex == ptr_encindex) { + if (str_cr != ENC_CODERANGE_UNKNOWN && ptr_cr == ENC_CODERANGE_UNKNOWN) { + ptr_cr = coderange_scan(ptr, len, rb_enc_from_index(ptr_encindex)); + } + } + else { + str_enc = rb_enc_from_index(str_encindex); + ptr_enc = rb_enc_from_index(ptr_encindex); + if (!rb_enc_asciicompat(str_enc) || !rb_enc_asciicompat(ptr_enc)) { + if (len == 0) + return str; + if (RSTRING_LEN(str) == 0) { + rb_str_buf_cat(str, ptr, len); + ENCODING_CODERANGE_SET(str, ptr_encindex, ptr_cr); + rb_str_change_terminator_length(str, rb_enc_mbminlen(str_enc), rb_enc_mbminlen(ptr_enc)); + return str; + } + goto incompatible; + } + if (ptr_cr == ENC_CODERANGE_UNKNOWN) { + ptr_cr = coderange_scan(ptr, len, ptr_enc); + } + if (str_cr == ENC_CODERANGE_UNKNOWN) { + if (ENCODING_IS_ASCII8BIT(str) || ptr_cr != ENC_CODERANGE_7BIT) { + str_cr = rb_enc_str_coderange(str); + } + } + } + if (ptr_cr_ret) + *ptr_cr_ret = ptr_cr; + + if (str_encindex != ptr_encindex && + str_cr != ENC_CODERANGE_7BIT && + ptr_cr != ENC_CODERANGE_7BIT) { + str_enc = rb_enc_from_index(str_encindex); + ptr_enc = rb_enc_from_index(ptr_encindex); + goto incompatible; + } + + if (str_cr == ENC_CODERANGE_UNKNOWN) { + res_encindex = str_encindex; + res_cr = ENC_CODERANGE_UNKNOWN; + } + else if (str_cr == ENC_CODERANGE_7BIT) { + if (ptr_cr == ENC_CODERANGE_7BIT) { + res_encindex = str_encindex; + res_cr = ENC_CODERANGE_7BIT; + } + else { + res_encindex = ptr_encindex; + res_cr = ptr_cr; + } + } + else if (str_cr == ENC_CODERANGE_VALID) { + res_encindex = str_encindex; + if (ENC_CODERANGE_CLEAN_P(ptr_cr)) + res_cr = str_cr; + else + res_cr = ptr_cr; + } + else { /* str_cr == ENC_CODERANGE_BROKEN */ + res_encindex = str_encindex; + res_cr = str_cr; + if (0 < len) res_cr = ENC_CODERANGE_UNKNOWN; + } + + if (len < 0) { + rb_raise(rb_eArgError, "negative string size (or size too big)"); + } + str_buf_cat(str, ptr, len); + ENCODING_CODERANGE_SET(str, res_encindex, res_cr); + return str; + + incompatible: + rb_raise(rb_eEncCompatError, "incompatible character encodings: %s and %s", + rb_enc_inspect_name(str_enc), rb_enc_inspect_name(ptr_enc)); + UNREACHABLE_RETURN(Qundef); } VALUE -rb_str_buf_append(str, str2) - VALUE str, str2; +rb_enc_str_buf_cat(VALUE str, const char *ptr, long len, rb_encoding *ptr_enc) { - long capa, len; + return rb_enc_cr_str_buf_cat(str, ptr, len, + rb_enc_to_index(ptr_enc), ENC_CODERANGE_UNKNOWN, NULL); +} - rb_str_modify(str); - if (FL_TEST(str, STR_ASSOC)) { - FL_UNSET(str, STR_ASSOC); - capa = RSTRING(str)->aux.capa = RSTRING(str)->len; +VALUE +rb_str_buf_cat_ascii(VALUE str, const char *ptr) +{ + /* ptr must reference NUL terminated ASCII string. */ + int encindex = ENCODING_GET(str); + rb_encoding *enc = rb_enc_from_index(encindex); + if (rb_enc_asciicompat(enc)) { + return rb_enc_cr_str_buf_cat(str, ptr, strlen(ptr), + encindex, ENC_CODERANGE_7BIT, 0); } else { - capa = RSTRING(str)->aux.capa; + char *buf = ALLOCA_N(char, rb_enc_mbmaxlen(enc)); + while (*ptr) { + unsigned int c = (unsigned char)*ptr; + int len = rb_enc_codelen(c, enc); + rb_enc_mbcput(c, buf, enc); + rb_enc_cr_str_buf_cat(str, buf, len, + encindex, ENC_CODERANGE_VALID, 0); + ptr++; + } + return str; } - len = RSTRING(str)->len+RSTRING(str2)->len; - if (capa <= len) { - while (len > capa) { - capa = (capa + 1) * 2; - } - RESIZE_CAPA(str, capa); +} + +VALUE +rb_str_buf_append(VALUE str, VALUE str2) +{ + int str2_cr = rb_enc_str_coderange(str2); + + if (rb_str_enc_fastpath(str)) { + switch (str2_cr) { + case ENC_CODERANGE_7BIT: + // If RHS is 7bit we can do simple concatenation + str_buf_cat4(str, RSTRING_PTR(str2), RSTRING_LEN(str2), true); + RB_GC_GUARD(str2); + return str; + case ENC_CODERANGE_VALID: + // If RHS is valid, we can do simple concatenation if encodings are the same + if (ENCODING_GET_INLINED(str) == ENCODING_GET_INLINED(str2)) { + str_buf_cat4(str, RSTRING_PTR(str2), RSTRING_LEN(str2), true); + int str_cr = ENC_CODERANGE(str); + if (UNLIKELY(str_cr != ENC_CODERANGE_VALID)) { + ENC_CODERANGE_SET(str, RB_ENC_CODERANGE_AND(str_cr, str2_cr)); + } + RB_GC_GUARD(str2); + return str; + } + } } - memcpy(RSTRING(str)->ptr + RSTRING(str)->len, - RSTRING(str2)->ptr, RSTRING(str2)->len); - RSTRING(str)->len += RSTRING(str2)->len; - RSTRING(str)->ptr[RSTRING(str)->len] = '\0'; /* sentinel */ - OBJ_INFECT(str, str2); + + rb_enc_cr_str_buf_cat(str, RSTRING_PTR(str2), RSTRING_LEN(str2), + ENCODING_GET(str2), str2_cr, &str2_cr); + + ENC_CODERANGE_SET(str2, str2_cr); return str; } VALUE -rb_str_append(str, str2) - VALUE str, str2; +rb_str_append(VALUE str, VALUE str2) { - rb_str_modify(str); StringValue(str2); - if (RSTRING(str2)->len > 0) { - if (FL_TEST(str, STR_ASSOC)) { - long len = RSTRING(str)->len+RSTRING(str2)->len; - REALLOC_N(RSTRING(str)->ptr, char, len+1); - memcpy(RSTRING(str)->ptr + RSTRING(str)->len, - RSTRING(str2)->ptr, RSTRING(str2)->len); - RSTRING(str)->ptr[len] = '\0'; /* sentinel */ - RSTRING(str)->len = len; - } - else { - return rb_str_buf_append(str, str2); - } - } - OBJ_INFECT(str, str2); + return rb_str_buf_append(str, str2); +} + +VALUE +rb_str_concat_literals(size_t num, const VALUE *strary) +{ + VALUE str; + size_t i, s = 0; + unsigned long len = 1; + + if (UNLIKELY(!num)) return rb_str_new(0, 0); + if (UNLIKELY(num == 1)) return rb_str_resurrect(strary[0]); + + for (i = 0; i < num; ++i) { len += RSTRING_LEN(strary[i]); } + str = rb_str_buf_new(len); + str_enc_copy_direct(str, strary[0]); + + for (i = s; i < num; ++i) { + const VALUE v = strary[i]; + int encidx = ENCODING_GET(v); + + rb_str_buf_append(str, v); + if (encidx != ENCINDEX_US_ASCII) { + if (ENCODING_GET_INLINED(str) == ENCINDEX_US_ASCII) + rb_enc_set_index(str, encidx); + } + } return str; } +/* + * call-seq: + * concat(*objects) -> string + * + * :include: doc/string/concat.rdoc + */ +static VALUE +rb_str_concat_multi(int argc, VALUE *argv, VALUE str) +{ + str_modifiable(str); + + if (argc == 1) { + return rb_str_concat(str, argv[0]); + } + else if (argc > 1) { + int i; + VALUE arg_str = rb_str_tmp_new(0); + rb_enc_copy(arg_str, str); + for (i = 0; i < argc; i++) { + rb_str_concat(arg_str, argv[i]); + } + rb_str_buf_append(str, arg_str); + } + + return str; +} /* * call-seq: - * str << fixnum => str - * str.concat(fixnum) => str - * str << obj => str - * str.concat(obj) => str - * - * Append---Concatenates the given object to <i>str</i>. If the object is a - * <code>Fixnum</code> between 0 and 255, it is converted to a character before - * concatenation. - * - * a = "hello " - * a << "world" #=> "hello world" - * a.concat(33) #=> "hello world!" + * append_as_bytes(*objects) -> self + * + * Concatenates each object in +objects+ into +self+; returns +self+; + * performs no encoding validation or conversion: + * + * s = 'foo' + * s.append_as_bytes(" \xE2\x82") # => "foo \xE2\x82" + * s.valid_encoding? # => false + * s.append_as_bytes("\xAC 12") + * s.valid_encoding? # => true + * + * When a given object is an integer, + * the value is considered an 8-bit byte; + * if the integer occupies more than one byte (i.e,. is greater than 255), + * appends only the low-order byte (similar to String#setbyte): + * + * s = "" + * s.append_as_bytes(0, 257) # => "\u0000\u0001" + * s.bytesize # => 2 + * + * Related: see {Modifying}[rdoc-ref:String@Modifying]. */ VALUE -rb_str_concat(str1, str2) - VALUE str1, str2; +rb_str_append_as_bytes(int argc, VALUE *argv, VALUE str) +{ + long needed_capacity = 0; + volatile VALUE t0; + enum ruby_value_type *types = ALLOCV_N(enum ruby_value_type, t0, argc); + + for (int index = 0; index < argc; index++) { + VALUE obj = argv[index]; + enum ruby_value_type type = types[index] = rb_type(obj); + switch (type) { + case T_FIXNUM: + case T_BIGNUM: + needed_capacity++; + break; + case T_STRING: + needed_capacity += RSTRING_LEN(obj); + break; + default: + rb_raise( + rb_eTypeError, + "wrong argument type %"PRIsVALUE" (expected String or Integer)", + rb_obj_class(obj) + ); + break; + } + } + + str_ensure_available_capa(str, needed_capacity); + char *sptr = RSTRING_END(str); + + for (int index = 0; index < argc; index++) { + VALUE obj = argv[index]; + enum ruby_value_type type = types[index]; + switch (type) { + case T_FIXNUM: + case T_BIGNUM: { + argv[index] = obj = rb_int_and(obj, INT2FIX(0xff)); + char byte = (char)(NUM2INT(obj) & 0xFF); + *sptr = byte; + sptr++; + break; + } + case T_STRING: { + const char *ptr; + long len; + RSTRING_GETMEM(obj, ptr, len); + memcpy(sptr, ptr, len); + sptr += len; + break; + } + default: + rb_bug("append_as_bytes arguments should have been validated"); + } + } + + STR_SET_LEN(str, RSTRING_LEN(str) + needed_capacity); + TERM_FILL(sptr, TERM_LEN(str)); /* sentinel */ + + int cr = ENC_CODERANGE(str); + switch (cr) { + case ENC_CODERANGE_7BIT: { + for (int index = 0; index < argc; index++) { + VALUE obj = argv[index]; + enum ruby_value_type type = types[index]; + switch (type) { + case T_FIXNUM: + case T_BIGNUM: { + if (!ISASCII(NUM2INT(obj))) { + goto clear_cr; + } + break; + } + case T_STRING: { + if (ENC_CODERANGE(obj) != ENC_CODERANGE_7BIT) { + goto clear_cr; + } + break; + } + default: + rb_bug("append_as_bytes arguments should have been validated"); + } + } + break; + } + case ENC_CODERANGE_VALID: + if (ENCODING_GET_INLINED(str) == ENCINDEX_ASCII_8BIT) { + goto keep_cr; + } + else { + goto clear_cr; + } + break; + default: + goto clear_cr; + break; + } + + RB_GC_GUARD(t0); + + clear_cr: + // If no fast path was hit, we clear the coderange. + // append_as_bytes is predominantly meant to be used in + // buffering situation, hence it's likely the coderange + // will never be scanned, so it's not worth spending time + // precomputing the coderange except for simple and common + // situations. + ENC_CODERANGE_CLEAR(str); + keep_cr: + return str; +} + +/* + * call-seq: + * self << object -> self + * + * Appends a string representation of +object+ to +self+; + * returns +self+. + * + * If +object+ is a string, appends it to +self+: + * + * s = 'foo' + * s << 'bar' # => "foobar" + * s # => "foobar" + * + * If +object+ is an integer, + * its value is considered a codepoint; + * converts the value to a character before concatenating: + * + * s = 'foo' + * s << 33 # => "foo!" + * + * Additionally, if the codepoint is in range <tt>0..0xff</tt> + * and the encoding of +self+ is Encoding::US_ASCII, + * changes the encoding to Encoding::ASCII_8BIT: + * + * s = 'foo'.encode(Encoding::US_ASCII) + * s.encoding # => #<Encoding:US-ASCII> + * s << 0xff # => "foo\xFF" + * s.encoding # => #<Encoding:BINARY (ASCII-8BIT)> + * + * Raises RangeError if that codepoint is not representable in the encoding of +self+: + * + * s = 'foo' + * s.encoding # => <Encoding:UTF-8> + * s << 0x00110000 # 1114112 out of char range (RangeError) + * s = 'foo'.encode(Encoding::EUC_JP) + * s << 0x00800080 # invalid codepoint 0x800080 in EUC-JP (RangeError) + * + * Related: see {Modifying}[rdoc-ref:String@Modifying]. + */ +VALUE +rb_str_concat(VALUE str1, VALUE str2) { - if (FIXNUM_P(str2)) { - int i = FIX2INT(str2); - if (0 <= i && i <= 0xff) { /* byte */ - char c = i; - return rb_str_cat(str1, &c, 1); - } + unsigned int code; + rb_encoding *enc = STR_ENC_GET(str1); + int encidx; + + if (RB_INTEGER_TYPE_P(str2)) { + if (rb_num_to_uint(str2, &code) == 0) { + } + else if (FIXNUM_P(str2)) { + rb_raise(rb_eRangeError, "%ld out of char range", FIX2LONG(str2)); + } + else { + rb_raise(rb_eRangeError, "bignum out of char range"); + } + } + else { + return rb_str_append(str1, str2); } - str1 = rb_str_append(str1, str2); + encidx = rb_ascii8bit_appendable_encoding_index(enc, code); + + if (encidx >= 0) { + rb_str_buf_cat_byte(str1, (unsigned char)code); + } + else { + long pos = RSTRING_LEN(str1); + int cr = ENC_CODERANGE(str1); + int len; + char *buf; + + switch (len = rb_enc_codelen(code, enc)) { + case ONIGERR_INVALID_CODE_POINT_VALUE: + rb_raise(rb_eRangeError, "invalid codepoint 0x%X in %s", code, rb_enc_name(enc)); + break; + case ONIGERR_TOO_BIG_WIDE_CHAR_VALUE: + case 0: + rb_raise(rb_eRangeError, "%u out of char range", code); + break; + } + buf = ALLOCA_N(char, len + 1); + rb_enc_mbcput(code, buf, enc); + if (rb_enc_precise_mbclen(buf, buf + len + 1, enc) != len) { + rb_raise(rb_eRangeError, "invalid codepoint 0x%X in %s", code, rb_enc_name(enc)); + } + rb_str_resize(str1, pos+len); + memcpy(RSTRING_PTR(str1) + pos, buf, len); + if (cr == ENC_CODERANGE_7BIT && code > 127) { + cr = ENC_CODERANGE_VALID; + } + else if (cr == ENC_CODERANGE_BROKEN) { + cr = ENC_CODERANGE_UNKNOWN; + } + ENC_CODERANGE_SET(str1, cr); + } return str1; } int -rb_str_hash(str) - VALUE str; +rb_ascii8bit_appendable_encoding_index(rb_encoding *enc, unsigned int code) { - register long len = RSTRING(str)->len; - register char *p = RSTRING(str)->ptr; - register int key = 0; + int encidx = rb_enc_to_index(enc); -#ifdef HASH_ELFHASH - register unsigned int g; + if (encidx == ENCINDEX_ASCII_8BIT || encidx == ENCINDEX_US_ASCII) { + /* US-ASCII automatically extended to ASCII-8BIT */ + if (code > 0xFF) { + rb_raise(rb_eRangeError, "%u out of char range", code); + } + if (encidx == ENCINDEX_US_ASCII && code > 127) { + return ENCINDEX_ASCII_8BIT; + } + return encidx; + } + else { + return -1; + } +} + +/* + * call-seq: + * prepend(*other_strings) -> new_string + * + * Prefixes to +self+ the concatenation of the given +other_strings+; returns +self+: + * + * 'baz'.prepend('foo', 'bar') # => "foobarbaz" + * + * Related: see {Modifying}[rdoc-ref:String@Modifying]. + * + */ + +static VALUE +rb_str_prepend_multi(int argc, VALUE *argv, VALUE str) +{ + str_modifiable(str); - while (len--) { - key = (key << 4) + *p++; - if (g = key & 0xF0000000) - key ^= g >> 24; - key &= ~g; + if (argc == 1) { + rb_str_update(str, 0L, 0L, argv[0]); } -#elif HASH_PERL - while (len--) { - key += *p++; - key += (key << 10); - key ^= (key >> 6); + else if (argc > 1) { + int i; + VALUE arg_str = rb_str_tmp_new(0); + rb_enc_copy(arg_str, str); + for (i = 0; i < argc; i++) { + rb_str_append(arg_str, argv[i]); + } + rb_str_update(str, 0L, 0L, arg_str); } - key += (key << 3); - key ^= (key >> 11); - key += (key << 15); -#else - while (len--) { - key = key*65599 + *p; - p++; + + return str; +} + +st_index_t +rb_str_hash(VALUE str) +{ + if (FL_TEST_RAW(str, STR_PRECOMPUTED_HASH)) { + st_index_t precomputed_hash; + memcpy(&precomputed_hash, RSTRING_END(str) + TERM_LEN(str), sizeof(precomputed_hash)); + + RUBY_ASSERT(precomputed_hash == str_do_hash(str)); + return precomputed_hash; } - key = key + (key>>5); -#endif - return key; + + return str_do_hash(str); +} + +int +rb_str_hash_cmp(VALUE str1, VALUE str2) +{ + long len1, len2; + const char *ptr1, *ptr2; + RSTRING_GETMEM(str1, ptr1, len1); + RSTRING_GETMEM(str2, ptr2, len2); + return (len1 != len2 || + !rb_str_comparable(str1, str2) || + memcmp(ptr1, ptr2, len1) != 0); } /* * call-seq: - * str.hash => fixnum + * hash -> integer + * + * :include: doc/string/hash.rdoc * - * Return a hash based on the string's length and content. */ static VALUE -rb_str_hash_m(str) - VALUE str; +rb_str_hash_m(VALUE str) { - int key = rb_str_hash(str); - return INT2FIX(key); + st_index_t hval = rb_str_hash(str); + return ST2FIX(hval); } #define lesser(a,b) (((a)>(b))?(b):(a)) int -rb_str_cmp(str1, str2) - VALUE str1, str2; +rb_str_comparable(VALUE str1, VALUE str2) { - long len; + int idx1, idx2; + int rc1, rc2; + + if (RSTRING_LEN(str1) == 0) return TRUE; + if (RSTRING_LEN(str2) == 0) return TRUE; + idx1 = ENCODING_GET(str1); + idx2 = ENCODING_GET(str2); + if (idx1 == idx2) return TRUE; + rc1 = rb_enc_str_coderange(str1); + rc2 = rb_enc_str_coderange(str2); + if (rc1 == ENC_CODERANGE_7BIT) { + if (rc2 == ENC_CODERANGE_7BIT) return TRUE; + if (rb_enc_asciicompat(rb_enc_from_index(idx2))) + return TRUE; + } + if (rc2 == ENC_CODERANGE_7BIT) { + if (rb_enc_asciicompat(rb_enc_from_index(idx1))) + return TRUE; + } + return FALSE; +} + +int +rb_str_cmp(VALUE str1, VALUE str2) +{ + long len1, len2; + const char *ptr1, *ptr2; int retval; - len = lesser(RSTRING(str1)->len, RSTRING(str2)->len); - retval = rb_memcmp(RSTRING(str1)->ptr, RSTRING(str2)->ptr, len); - if (retval == 0) { - if (RSTRING(str1)->len == RSTRING(str2)->len) return 0; - if (RSTRING(str1)->len > RSTRING(str2)->len) return 1; - return -1; + if (str1 == str2) return 0; + RSTRING_GETMEM(str1, ptr1, len1); + RSTRING_GETMEM(str2, ptr2, len2); + if (ptr1 == ptr2 || (retval = memcmp(ptr1, ptr2, lesser(len1, len2))) == 0) { + if (len1 == len2) { + if (!rb_str_comparable(str1, str2)) { + if (ENCODING_GET(str1) > ENCODING_GET(str2)) + return 1; + return -1; + } + return 0; + } + if (len1 > len2) return 1; + return -1; } if (retval > 0) return 1; return -1; } - /* * call-seq: - * str == obj => true or false - * - * Equality---If <i>obj</i> is not a <code>String</code>, returns - * <code>false</code>. Otherwise, returns <code>true</code> if <i>str</i> - * <code><=></code> <i>obj</i> returns zero. + * self == other -> true or false + * + * Returns whether +other+ is equal to +self+. + * + * When +other+ is a string, returns whether +other+ has the same length and content as +self+: + * + * s = 'foo' + * s == 'foo' # => true + * s == 'food' # => false + * s == 'FOO' # => false + * + * Returns +false+ if the two strings' encodings are not compatible: + * + * "\u{e4 f6 fc}".encode(Encoding::ISO_8859_1) == ("\u{c4 d6 dc}") # => false + * + * When +other+ is not a string: + * + * - If +other+ responds to method <tt>to_str</tt>, + * <tt>other == self</tt> is called and its return value is returned. + * - If +other+ does not respond to <tt>to_str</tt>, + * +false+ is returned. + * + * Related: {Comparing}[rdoc-ref:String@Comparing]. */ -static VALUE -rb_str_equal(str1, str2) - VALUE str1, str2; +VALUE +rb_str_equal(VALUE str1, VALUE str2) { if (str1 == str2) return Qtrue; - if (TYPE(str2) != T_STRING) { - if (!rb_respond_to(str2, rb_intern("to_str"))) { - return Qfalse; - } - return rb_equal(str2, str1); - } - if (RSTRING(str1)->len == RSTRING(str2)->len && - rb_str_cmp(str1, str2) == 0) { - return Qtrue; + if (!RB_TYPE_P(str2, T_STRING)) { + if (!rb_respond_to(str2, idTo_str)) { + return Qfalse; + } + return rb_equal(str2, str1); } - return Qfalse; + return rb_str_eql_internal(str1, str2); } /* * call-seq: - * str.eql?(other) => true or false + * eql?(object) -> true or false + * + * :include: doc/string/eql_p.rdoc * - * Two strings are equal if the have the same length and content. */ -static VALUE -rb_str_eql(str1, str2) - VALUE str1, str2; +VALUE +rb_str_eql(VALUE str1, VALUE str2) { - if (TYPE(str2) != T_STRING || RSTRING(str1)->len != RSTRING(str2)->len) - return Qfalse; + if (str1 == str2) return Qtrue; + if (!RB_TYPE_P(str2, T_STRING)) return Qfalse; + return rb_str_eql_internal(str1, str2); +} - if (memcmp(RSTRING(str1)->ptr, RSTRING(str2)->ptr, - lesser(RSTRING(str1)->len, RSTRING(str2)->len)) == 0) - return Qtrue; +/* + * call-seq: + * self <=> other -> -1, 0, 1, or nil + * + * Compares +self+ and +other+, + * evaluating their _contents_, not their _lengths_. + * + * Returns: + * + * - +-1+, if +self+ is smaller. + * - +0+, if the two are equal. + * - +1+, if +self+ is larger. + * - +nil+, if the two are incomparable. + * + * Examples: + * + * 'a' <=> 'b' # => -1 + * 'a' <=> 'ab' # => -1 + * 'a' <=> 'a' # => 0 + * 'b' <=> 'a' # => 1 + * 'ab' <=> 'a' # => 1 + * 'a' <=> :a # => nil + * + * \Class \String includes module Comparable, + * each of whose methods uses String#<=> for comparison. + * + * Related: see {Comparing}[rdoc-ref:String@Comparing]. + */ - return Qfalse; +static VALUE +rb_str_cmp_m(VALUE str1, VALUE str2) +{ + int result; + VALUE s = rb_check_string_type(str2); + if (NIL_P(s)) { + return rb_invcmp(str1, str2); + } + result = rb_str_cmp(str1, s); + return INT2FIX(result); } +static VALUE str_casecmp(VALUE str1, VALUE str2); +static VALUE str_casecmp_p(VALUE str1, VALUE str2); + /* * call-seq: - * str <=> other_str => -1, 0, +1 - * - * Comparison---Returns -1 if <i>other_str</i> is less than, 0 if - * <i>other_str</i> is equal to, and +1 if <i>other_str</i> is greater than - * <i>str</i>. If the strings are of different lengths, and the strings are - * equal when compared up to the shortest length, then the longer string is - * considered greater than the shorter one. If the variable <code>$=</code> is - * <code>false</code>, the comparison is based on comparing the binary values - * of each character in the string. In older versions of Ruby, setting - * <code>$=</code> allowed case-insensitive comparisons; this is now deprecated - * in favor of using <code>String#casecmp</code>. + * casecmp(other_string) -> -1, 0, 1, or nil + * + * Ignoring case, compares +self+ and +other_string+; returns: + * + * - -1 if <tt>self.downcase</tt> is smaller than <tt>other_string.downcase</tt>. + * - 0 if the two are equal. + * - 1 if <tt>self.downcase</tt> is larger than <tt>other_string.downcase</tt>. + * - +nil+ if the two are incomparable. + * + * See {Case Mapping}[rdoc-ref:case_mapping.rdoc]. + * + * Examples: * - * <code><=></code> is the basis for the methods <code><</code>, - * <code><=</code>, <code>></code>, <code>>=</code>, and <code>between?</code>, - * included from module <code>Comparable</code>. The method - * <code>String#==</code> does not use <code>Comparable#==</code>. - * - * "abcdef" <=> "abcde" #=> 1 - * "abcdef" <=> "abcdef" #=> 0 - * "abcdef" <=> "abcdefg" #=> -1 - * "abcdef" <=> "ABCDEF" #=> 1 + * 'foo'.casecmp('goo') # => -1 + * 'goo'.casecmp('foo') # => 1 + * 'foo'.casecmp('food') # => -1 + * 'food'.casecmp('foo') # => 1 + * 'FOO'.casecmp('foo') # => 0 + * 'foo'.casecmp('FOO') # => 0 + * 'foo'.casecmp(1) # => nil + * + * Related: see {Comparing}[rdoc-ref:String@Comparing]. */ static VALUE -rb_str_cmp_m(str1, str2) - VALUE str1, str2; +rb_str_casecmp(VALUE str1, VALUE str2) { - long result; - - if (TYPE(str2) != T_STRING) { - if (!rb_respond_to(str2, rb_intern("to_str"))) { - return Qfalse; - } - else if (!rb_respond_to(str2, rb_intern("<=>"))) { - return Qnil; - } - else { - VALUE tmp = rb_funcall(str2, rb_intern("<=>"), 1, str1); + VALUE s = rb_check_string_type(str2); + if (NIL_P(s)) { + return Qnil; + } + return str_casecmp(str1, s); +} - if (NIL_P(tmp)) return Qnil; - if (!FIXNUM_P(tmp)) { - return rb_funcall(LONG2FIX(0), '-', 1, tmp); - } - result = -FIX2LONG(tmp); - } +static VALUE +str_casecmp(VALUE str1, VALUE str2) +{ + long len; + rb_encoding *enc; + const char *p1, *p1end, *p2, *p2end; + + enc = rb_enc_compatible(str1, str2); + if (!enc) { + return Qnil; + } + + p1 = RSTRING_PTR(str1); p1end = RSTRING_END(str1); + p2 = RSTRING_PTR(str2); p2end = RSTRING_END(str2); + if (single_byte_optimizable(str1) && single_byte_optimizable(str2)) { + while (p1 < p1end && p2 < p2end) { + if (*p1 != *p2) { + unsigned int c1 = TOLOWER(*p1 & 0xff); + unsigned int c2 = TOLOWER(*p2 & 0xff); + if (c1 != c2) + return INT2FIX(c1 < c2 ? -1 : 1); + } + p1++; + p2++; + } } else { - result = rb_str_cmp(str1, str2); + while (p1 < p1end && p2 < p2end) { + int l1, c1 = rb_enc_ascget(p1, p1end, &l1, enc); + int l2, c2 = rb_enc_ascget(p2, p2end, &l2, enc); + + if (0 <= c1 && 0 <= c2) { + c1 = TOLOWER(c1); + c2 = TOLOWER(c2); + if (c1 != c2) + return INT2FIX(c1 < c2 ? -1 : 1); + } + else { + int r; + l1 = rb_enc_mbclen(p1, p1end, enc); + l2 = rb_enc_mbclen(p2, p2end, enc); + len = l1 < l2 ? l1 : l2; + r = memcmp(p1, p2, len); + if (r != 0) + return INT2FIX(r < 0 ? -1 : 1); + if (l1 != l2) + return INT2FIX(l1 < l2 ? -1 : 1); + } + p1 += l1; + p2 += l2; + } } - return LONG2NUM(result); + if (p1 == p1end && p2 == p2end) return INT2FIX(0); + if (p1 == p1end) return INT2FIX(-1); + return INT2FIX(1); } /* * call-seq: - * str.casecmp(other_str) => -1, 0, +1 - * - * Case-insensitive version of <code>String#<=></code>. - * - * "abcdef".casecmp("abcde") #=> 1 - * "aBcDeF".casecmp("abcdef") #=> 0 - * "abcdef".casecmp("abcdefg") #=> -1 - * "abcdef".casecmp("ABCDEF") #=> 0 + * casecmp?(other_string) -> true, false, or nil + * + * Returns +true+ if +self+ and +other_string+ are equal after + * Unicode case folding, +false+ if unequal, +nil+ if incomparable. + * + * See {Case Mapping}[rdoc-ref:case_mapping.rdoc]. + * + * Examples: + * + * 'foo'.casecmp?('goo') # => false + * 'goo'.casecmp?('foo') # => false + * 'foo'.casecmp?('food') # => false + * 'food'.casecmp?('foo') # => false + * 'FOO'.casecmp?('foo') # => true + * 'foo'.casecmp?('FOO') # => true + * 'foo'.casecmp?(1) # => nil + * + * Related: see {Comparing}[rdoc-ref:String@Comparing]. */ static VALUE -rb_str_casecmp(str1, str2) - VALUE str1, str2; +rb_str_casecmp_p(VALUE str1, VALUE str2) { - long len; - int retval; + VALUE s = rb_check_string_type(str2); + if (NIL_P(s)) { + return Qnil; + } + return str_casecmp_p(str1, s); +} - StringValue(str2); - len = lesser(RSTRING(str1)->len, RSTRING(str2)->len); - retval = rb_memcicmp(RSTRING(str1)->ptr, RSTRING(str2)->ptr, len); - if (retval == 0) { - if (RSTRING(str1)->len == RSTRING(str2)->len) return INT2FIX(0); - if (RSTRING(str1)->len > RSTRING(str2)->len) return INT2FIX(1); - return INT2FIX(-1); +static VALUE +str_casecmp_p(VALUE str1, VALUE str2) +{ + rb_encoding *enc; + VALUE folded_str1, folded_str2; + VALUE fold_opt = sym_fold; + + enc = rb_enc_compatible(str1, str2); + if (!enc) { + return Qnil; } - if (retval == 0) return INT2FIX(0); - if (retval > 0) return INT2FIX(1); - return INT2FIX(-1); + + folded_str1 = rb_str_downcase(1, &fold_opt, str1); + folded_str2 = rb_str_downcase(1, &fold_opt, str2); + + return rb_str_eql(folded_str1, folded_str2); } static long -rb_str_index(str, sub, offset) - VALUE str, sub; - long offset; +strseq_core(const char *str_ptr, const char *str_ptr_end, long str_len, + const char *sub_ptr, long sub_len, long offset, rb_encoding *enc) { - long pos; + const char *search_start = str_ptr; + long pos, search_len = str_len - offset; - if (offset < 0) { - offset += RSTRING(str)->len; - if (offset < 0) return -1; + for (;;) { + const char *t; + pos = rb_memsearch(sub_ptr, sub_len, search_start, search_len, enc); + if (pos < 0) return pos; + t = rb_enc_right_char_head(search_start, search_start+pos, str_ptr_end, enc); + if (t == search_start + pos) break; + search_len -= t - search_start; + if (search_len <= 0) return -1; + offset += t - search_start; + search_start = t; } - if (RSTRING(str)->len - offset < RSTRING(sub)->len) return -1; - if (RSTRING(sub)->len == 0) return offset; - pos = rb_memsearch(RSTRING(sub)->ptr, RSTRING(sub)->len, - RSTRING(str)->ptr+offset, RSTRING(str)->len-offset); - if (pos < 0) return pos; return pos + offset; } +/* found index in byte */ +#define rb_str_index(str, sub, offset) rb_strseq_index(str, sub, offset, 0) +#define rb_str_byteindex(str, sub, offset) rb_strseq_index(str, sub, offset, 1) + +static long +rb_strseq_index(VALUE str, VALUE sub, long offset, int in_byte) +{ + const char *str_ptr, *str_ptr_end, *sub_ptr; + long str_len, sub_len; + rb_encoding *enc; + + enc = rb_enc_check(str, sub); + if (is_broken_string(sub)) return -1; + + str_ptr = RSTRING_PTR(str); + str_ptr_end = RSTRING_END(str); + str_len = RSTRING_LEN(str); + sub_ptr = RSTRING_PTR(sub); + sub_len = RSTRING_LEN(sub); + + if (str_len < sub_len) return -1; + + if (offset != 0) { + long str_len_char, sub_len_char; + int single_byte = single_byte_optimizable(str); + str_len_char = (in_byte || single_byte) ? str_len : str_strlen(str, enc); + sub_len_char = in_byte ? sub_len : str_strlen(sub, enc); + if (offset < 0) { + offset += str_len_char; + if (offset < 0) return -1; + } + if (str_len_char - offset < sub_len_char) return -1; + if (!in_byte) offset = str_offset(str_ptr, str_ptr_end, offset, enc, single_byte); + str_ptr += offset; + } + if (sub_len == 0) return offset; + + /* need proceed one character at a time */ + return strseq_core(str_ptr, str_ptr_end, str_len, sub_ptr, sub_len, offset, enc); +} + /* * call-seq: - * str.index(substring [, offset]) => fixnum or nil - * str.index(fixnum [, offset]) => fixnum or nil - * str.index(regexp [, offset]) => fixnum or nil - * - * Returns the index of the first occurrence of the given <i>substring</i>, - * character (<i>fixnum</i>), or pattern (<i>regexp</i>) in <i>str</i>. Returns - * <code>nil</code> if not found. If the second parameter is present, it - * specifies the position in the string to begin the search. - * - * "hello".index('e') #=> 1 - * "hello".index('lo') #=> 3 - * "hello".index('a') #=> nil - * "hello".index(101) #=> 1 - * "hello".index(/[aeiou]/, -3) #=> 4 + * index(pattern, offset = 0) -> integer or nil + * + * :include: doc/string/index.rdoc + * */ static VALUE -rb_str_index_m(argc, argv, str) - int argc; - VALUE *argv; - VALUE str; +rb_str_index_m(int argc, VALUE *argv, VALUE str) { VALUE sub; VALUE initpos; + rb_encoding *enc = STR_ENC_GET(str); long pos; if (rb_scan_args(argc, argv, "11", &sub, &initpos) == 2) { - pos = NUM2LONG(initpos); + long slen = str_strlen(str, enc); /* str's enc */ + pos = NUM2LONG(initpos); + if (pos < 0 ? (pos += slen) < 0 : pos > slen) { + if (RB_TYPE_P(sub, T_REGEXP)) { + rb_backref_set(Qnil); + } + return Qnil; + } + } + else { + pos = 0; + } + + if (RB_TYPE_P(sub, T_REGEXP)) { + pos = str_offset(RSTRING_PTR(str), RSTRING_END(str), pos, + enc, single_byte_optimizable(str)); + + if (rb_reg_search(sub, str, pos, 0) >= 0) { + VALUE match = rb_backref_get(); + struct re_registers *regs = RMATCH_REGS(match); + pos = rb_str_sublen(str, BEG(0)); + return LONG2NUM(pos); + } } else { - pos = 0; + StringValue(sub); + pos = rb_str_index(str, sub, pos); + if (pos >= 0) { + pos = rb_str_sublen(str, pos); + return LONG2NUM(pos); + } } - if (pos < 0) { - pos += RSTRING(str)->len; - if (pos < 0) { - if (TYPE(sub) == T_REGEXP) { - rb_backref_set(Qnil); - } - return Qnil; - } + return Qnil; +} + +/* Ensure that the given pos is a valid character boundary. + * Note that in this function, "character" means a code point + * (Unicode scalar value), not a grapheme cluster. + */ +static void +str_ensure_byte_pos(VALUE str, long pos) +{ + if (!single_byte_optimizable(str)) { + const char *s = RSTRING_PTR(str); + const char *e = RSTRING_END(str); + const char *p = s + pos; + if (!at_char_boundary(s, p, e, rb_enc_get(str))) { + rb_raise(rb_eIndexError, + "offset %ld does not land on character boundary", pos); + } } +} - switch (TYPE(sub)) { - case T_REGEXP: - pos = rb_reg_adjust_startpos(sub, str, pos, 0); - pos = rb_reg_search(sub, str, pos, 0); - break; +/* + * call-seq: + * byteindex(object, offset = 0) -> integer or nil + * + * Returns the 0-based integer index of a substring of +self+ + * specified by +object+ (a string or Regexp) and +offset+, + * or +nil+ if there is no such substring; + * the returned index is the count of _bytes_ (not characters). + * + * When +object+ is a string, + * returns the index of the first found substring equal to +object+: + * + * s = 'foo' # => "foo" + * s.size # => 3 # Three 1-byte characters. + * s.bytesize # => 3 # Three bytes. + * s.byteindex('f') # => 0 + * s.byteindex('o') # => 1 + * s.byteindex('oo') # => 1 + * s.byteindex('ooo') # => nil + * + * When +object+ is a Regexp, + * returns the index of the first found substring matching +object+; + * updates {Regexp-related global variables}[rdoc-ref:Regexp@Global+Variables]: + * + * s = 'foo' + * s.byteindex(/f/) # => 0 + * $~ # => #<MatchData "f"> + * s.byteindex(/o/) # => 1 + * s.byteindex(/oo/) # => 1 + * s.byteindex(/ooo/) # => nil + * $~ # => nil + * + * \Integer argument +offset+, if given, specifies the 0-based index + * of the byte where searching is to begin. + * + * When +offset+ is non-negative, + * searching begins at byte position +offset+: + * + * s = 'foo' + * s.byteindex('o', 1) # => 1 + * s.byteindex('o', 2) # => 2 + * s.byteindex('o', 3) # => nil + * + * When +offset+ is negative, counts backward from the end of +self+: + * + * s = 'foo' + * s.byteindex('o', -1) # => 2 + * s.byteindex('o', -2) # => 1 + * s.byteindex('o', -3) # => 1 + * s.byteindex('o', -4) # => nil + * + * Raises IndexError if the byte at +offset+ is not the first byte of a character: + * + * s = "\uFFFF\uFFFF" # => "\uFFFF\uFFFF" + * s.size # => 2 # Two 3-byte characters. + * s.bytesize # => 6 # Six bytes. + * s.byteindex("\uFFFF") # => 0 + * s.byteindex("\uFFFF", 1) # Raises IndexError + * s.byteindex("\uFFFF", 2) # Raises IndexError + * s.byteindex("\uFFFF", 3) # => 3 + * s.byteindex("\uFFFF", 4) # Raises IndexError + * s.byteindex("\uFFFF", 5) # Raises IndexError + * s.byteindex("\uFFFF", 6) # => nil + * + * Related: see {Querying}[rdoc-ref:String@Querying]. + */ - case T_FIXNUM: - { - int c = FIX2INT(sub); - long len = RSTRING(str)->len; - unsigned char *p = RSTRING(str)->ptr; - - for (;pos<len;pos++) { - if (p[pos] == c) return LONG2NUM(pos); - } - return Qnil; - } +static VALUE +rb_str_byteindex_m(int argc, VALUE *argv, VALUE str) +{ + VALUE sub; + VALUE initpos; + long pos; - default: { - VALUE tmp; + if (rb_scan_args(argc, argv, "11", &sub, &initpos) == 2) { + long slen = RSTRING_LEN(str); + pos = NUM2LONG(initpos); + if (pos < 0 ? (pos += slen) < 0 : pos > slen) { + if (RB_TYPE_P(sub, T_REGEXP)) { + rb_backref_set(Qnil); + } + return Qnil; + } + } + else { + pos = 0; + } - tmp = rb_check_string_type(sub); - if (NIL_P(tmp)) { - rb_raise(rb_eTypeError, "type mismatch: %s given", - rb_obj_classname(sub)); - } - sub = tmp; - } - /* fall through */ - case T_STRING: - pos = rb_str_index(str, sub, pos); - break; + str_ensure_byte_pos(str, pos); + + if (RB_TYPE_P(sub, T_REGEXP)) { + if (rb_reg_search(sub, str, pos, 0) >= 0) { + VALUE match = rb_backref_get(); + struct re_registers *regs = RMATCH_REGS(match); + pos = BEG(0); + return LONG2NUM(pos); + } + } + else { + StringValue(sub); + pos = rb_str_byteindex(str, sub, pos); + if (pos >= 0) return LONG2NUM(pos); } + return Qnil; +} - if (pos == -1) return Qnil; - return LONG2NUM(pos); +#ifndef HAVE_MEMRCHR +static void* +memrchr(const char *search_str, int chr, long search_len) +{ + const char *ptr = search_str + search_len; + while (ptr > search_str) { + if ((unsigned char)*(--ptr) == chr) return (void *)ptr; + } + + return ((void *)0); } +#endif static long -rb_str_rindex(str, sub, pos) - VALUE str, sub; - long pos; +str_rindex(VALUE str, VALUE sub, const char *s, rb_encoding *enc) +{ + char *hit, *adjusted; + int c; + long slen, searchlen; + char *sbeg, *e, *t; + + sbeg = RSTRING_PTR(str); + slen = RSTRING_LEN(sub); + if (slen == 0) return s - sbeg; + e = RSTRING_END(str); + t = RSTRING_PTR(sub); + c = *t & 0xff; + searchlen = s - sbeg + 1; + + if (memcmp(s, t, slen) == 0) { + return s - sbeg; + } + + do { + hit = memrchr(sbeg, c, searchlen); + if (!hit) break; + adjusted = rb_enc_left_char_head(sbeg, hit, e, enc); + if (hit != adjusted) { + searchlen = adjusted - sbeg; + continue; + } + if (memcmp(hit, t, slen) == 0) + return hit - sbeg; + searchlen = adjusted - sbeg; + } while (searchlen > 0); + + return -1; +} + +/* found index in byte */ +static long +rb_str_rindex(VALUE str, VALUE sub, long pos) { - long len = RSTRING(sub)->len; - char *s, *sbeg, *t; + long len, slen; + char *sbeg, *s; + rb_encoding *enc; + int singlebyte; + + enc = rb_enc_check(str, sub); + if (is_broken_string(sub)) return -1; + singlebyte = single_byte_optimizable(str); + len = singlebyte ? RSTRING_LEN(str) : str_strlen(str, enc); /* rb_enc_check */ + slen = str_strlen(sub, enc); /* rb_enc_check */ /* substring longer than string */ - if (RSTRING(str)->len < len) return -1; - if (RSTRING(str)->len - pos < len) { - pos = RSTRING(str)->len - len; + if (len < slen) return -1; + if (len - pos < slen) pos = len - slen; + if (len == 0) return pos; + + sbeg = RSTRING_PTR(str); + + if (pos == 0) { + if (memcmp(sbeg, RSTRING_PTR(sub), RSTRING_LEN(sub)) == 0) + return 0; + else + return -1; } - sbeg = RSTRING(str)->ptr; - s = RSTRING(str)->ptr + pos; - t = RSTRING(sub)->ptr; - if (len) { - while (sbeg <= s) { - if (rb_memcmp(s, t, len) == 0) { - return s - RSTRING(str)->ptr; - } - s--; - } - return -1; + + s = str_nth(sbeg, RSTRING_END(str), pos, enc, singlebyte); + return str_rindex(str, sub, s, enc); +} + +/* + * call-seq: + * rindex(pattern, offset = self.length) -> integer or nil + * + * :include:doc/string/rindex.rdoc + * + */ + +static VALUE +rb_str_rindex_m(int argc, VALUE *argv, VALUE str) +{ + VALUE sub; + VALUE initpos; + rb_encoding *enc = STR_ENC_GET(str); + long pos, len = str_strlen(str, enc); /* str's enc */ + + if (rb_scan_args(argc, argv, "11", &sub, &initpos) == 2) { + pos = NUM2LONG(initpos); + if (pos < 0 && (pos += len) < 0) { + if (RB_TYPE_P(sub, T_REGEXP)) { + rb_backref_set(Qnil); + } + return Qnil; + } + if (pos > len) pos = len; + } + else { + pos = len; + } + + if (RB_TYPE_P(sub, T_REGEXP)) { + /* enc = rb_enc_check(str, sub); */ + pos = str_offset(RSTRING_PTR(str), RSTRING_END(str), pos, + enc, single_byte_optimizable(str)); + + if (rb_reg_search(sub, str, pos, 1) >= 0) { + VALUE match = rb_backref_get(); + struct re_registers *regs = RMATCH_REGS(match); + pos = rb_str_sublen(str, BEG(0)); + return LONG2NUM(pos); + } } else { - return pos; + StringValue(sub); + pos = rb_str_rindex(str, sub, pos); + if (pos >= 0) { + pos = rb_str_sublen(str, pos); + return LONG2NUM(pos); + } } + return Qnil; } +static long +rb_str_byterindex(VALUE str, VALUE sub, long pos) +{ + long len, slen; + char *sbeg, *s; + rb_encoding *enc; + + enc = rb_enc_check(str, sub); + if (is_broken_string(sub)) return -1; + len = RSTRING_LEN(str); + slen = RSTRING_LEN(sub); + + /* substring longer than string */ + if (len < slen) return -1; + if (len - pos < slen) pos = len - slen; + if (len == 0) return pos; + + sbeg = RSTRING_PTR(str); + + if (pos == 0) { + if (memcmp(sbeg, RSTRING_PTR(sub), RSTRING_LEN(sub)) == 0) + return 0; + else + return -1; + } + + s = sbeg + pos; + return str_rindex(str, sub, s, enc); +} /* * call-seq: - * str.rindex(substring [, fixnum]) => fixnum or nil - * str.rindex(fixnum [, fixnum]) => fixnum or nil - * str.rindex(regexp [, fixnum]) => fixnum or nil - * - * Returns the index of the last occurrence of the given <i>substring</i>, - * character (<i>fixnum</i>), or pattern (<i>regexp</i>) in <i>str</i>. Returns - * <code>nil</code> if not found. If the second parameter is present, it - * specifies the position in the string to end the search---characters beyond - * this point will not be considered. - * - * "hello".rindex('e') #=> 1 - * "hello".rindex('l') #=> 3 - * "hello".rindex('a') #=> nil - * "hello".rindex(101) #=> 1 - * "hello".rindex(/[aeiou]/, -2) #=> 1 + * byterindex(object, offset = self.bytesize) -> integer or nil + * + * Returns the 0-based integer index of a substring of +self+ + * that is the _last_ match for the given +object+ (a string or Regexp) and +offset+, + * or +nil+ if there is no such substring; + * the returned index is the count of _bytes_ (not characters). + * + * When +object+ is a string, + * returns the index of the _last_ found substring equal to +object+: + * + * s = 'foo' # => "foo" + * s.size # => 3 # Three 1-byte characters. + * s.bytesize # => 3 # Three bytes. + * s.byterindex('f') # => 0 + s.byterindex('o') # => 2 + s.byterindex('oo') # => 1 + s.byterindex('ooo') # => nil + * + * When +object+ is a Regexp, + * returns the index of the last found substring matching +object+; + * updates {Regexp-related global variables}[rdoc-ref:Regexp@Global+Variables]: + * + * s = 'foo' + * s.byterindex(/f/) # => 0 + * $~ # => #<MatchData "f"> + * s.byterindex(/o/) # => 2 + * s.byterindex(/oo/) # => 1 + * s.byterindex(/ooo/) # => nil + * $~ # => nil + * + * The last match means starting at the possible last position, + * not the last of the longest matches: + * + * s = 'foo' + * s.byterindex(/o+/) # => 2 + * $~ #=> #<MatchData "o"> + * + * To get the last longest match, use a negative lookbehind: + * + * s = 'foo' + * s.byterindex(/(?<!o)o+/) # => 1 + * $~ # => #<MatchData "oo"> + * + * Or use method #byteindex with negative lookahead: + * + * s = 'foo' + * s.byteindex(/o+(?!.*o)/) # => 1 + * $~ #=> #<MatchData "oo"> + * + * \Integer argument +offset+, if given, specifies the 0-based index + * of the byte where searching is to end. + * + * When +offset+ is non-negative, + * searching ends at byte position +offset+: + * + * s = 'foo' + * s.byterindex('o', 0) # => nil + * s.byterindex('o', 1) # => 1 + * s.byterindex('o', 2) # => 2 + * s.byterindex('o', 3) # => 2 + * + * When +offset+ is negative, counts backward from the end of +self+: + * + * s = 'foo' + * s.byterindex('o', -1) # => 2 + * s.byterindex('o', -2) # => 1 + * s.byterindex('o', -3) # => nil + * + * Raises IndexError if the byte at +offset+ is not the first byte of a character: + * + * s = "\uFFFF\uFFFF" # => "\uFFFF\uFFFF" + * s.size # => 2 # Two 3-byte characters. + * s.bytesize # => 6 # Six bytes. + * s.byterindex("\uFFFF") # => 3 + * s.byterindex("\uFFFF", 1) # Raises IndexError + * s.byterindex("\uFFFF", 2) # Raises IndexError + * s.byterindex("\uFFFF", 3) # => 3 + * s.byterindex("\uFFFF", 4) # Raises IndexError + * s.byterindex("\uFFFF", 5) # Raises IndexError + * s.byterindex("\uFFFF", 6) # => nil + * + * Related: see {Querying}[rdoc-ref:String@Querying]. */ static VALUE -rb_str_rindex_m(argc, argv, str) - int argc; - VALUE *argv; - VALUE str; +rb_str_byterindex_m(int argc, VALUE *argv, VALUE str) { VALUE sub; - VALUE position; - long pos; + VALUE initpos; + long pos, len = RSTRING_LEN(str); - if (rb_scan_args(argc, argv, "11", &sub, &position) == 2) { - pos = NUM2LONG(position); - if (pos < 0) { - pos += RSTRING(str)->len; - if (pos < 0) { - if (TYPE(sub) == T_REGEXP) { - rb_backref_set(Qnil); - } - return Qnil; - } + if (rb_scan_args(argc, argv, "11", &sub, &initpos) == 2) { + pos = NUM2LONG(initpos); + if (pos < 0 && (pos += len) < 0) { + if (RB_TYPE_P(sub, T_REGEXP)) { + rb_backref_set(Qnil); + } + return Qnil; } - if (pos > RSTRING(str)->len) pos = RSTRING(str)->len; + if (pos > len) pos = len; } else { - pos = RSTRING(str)->len; + pos = len; } - switch (TYPE(sub)) { - case T_REGEXP: - if (RREGEXP(sub)->len) { - pos = rb_reg_adjust_startpos(sub, str, pos, 1); - pos = rb_reg_search(sub, str, pos, 1); - } - if (pos >= 0) return LONG2NUM(pos); - break; + str_ensure_byte_pos(str, pos); - case T_STRING: - pos = rb_str_rindex(str, sub, pos); - if (pos >= 0) return LONG2NUM(pos); - break; - - case T_FIXNUM: - { - int c = FIX2INT(sub); - unsigned char *p = RSTRING(str)->ptr + pos; - unsigned char *pbeg = RSTRING(str)->ptr; - - if (pos == RSTRING(str)->len) { - if (pos == 0) return Qnil; - --p; - } - while (pbeg <= p) { - if (*p == c) return LONG2NUM((char*)p - RSTRING(str)->ptr); - p--; - } - return Qnil; - } - - default: - rb_raise(rb_eTypeError, "type mismatch: %s given", - rb_obj_classname(sub)); + if (RB_TYPE_P(sub, T_REGEXP)) { + if (rb_reg_search(sub, str, pos, 1) >= 0) { + VALUE match = rb_backref_get(); + struct re_registers *regs = RMATCH_REGS(match); + pos = BEG(0); + return LONG2NUM(pos); + } + } + else { + StringValue(sub); + pos = rb_str_byterindex(str, sub, pos); + if (pos >= 0) return LONG2NUM(pos); } return Qnil; } /* * call-seq: - * str =~ obj => fixnum or nil - * - * Match---If <i>obj</i> is a <code>Regexp</code>, use it as a pattern to match - * against <i>str</i>. If <i>obj</i> is a <code>String</code>, look for it in - * <i>str</i> (similar to <code>String#index</code>). Returns the position the - * match starts, or <code>nil</code> if there is no match. Otherwise, invokes - * <i>obj.=~</i>, passing <i>str</i> as an argument. The default - * <code>=~</code> in <code>Object</code> returns <code>false</code>. - * - * "cat o' 9 tails" =~ '\d' #=> nil - * "cat o' 9 tails" =~ /\d/ #=> 7 - * "cat o' 9 tails" =~ 9 #=> false + * self =~ other -> integer or nil + * + * When +other+ is a Regexp: + * + * - Returns the integer index (in characters) of the first match + * for +self+ and +other+, or +nil+ if none; + * - Updates {Regexp-related global variables}[rdoc-ref:Regexp@Global+Variables]. + * + * Examples: + * + * 'foo' =~ /f/ # => 0 + * $~ # => #<MatchData "f"> + * 'foo' =~ /o/ # => 1 + * $~ # => #<MatchData "o"> + * 'foo' =~ /x/ # => nil + * $~ # => nil + * + * Note that <tt>string =~ regexp</tt> is different from <tt>regexp =~ string</tt> + * (see Regexp#=~): + * + * number = nil + * 'no. 9' =~ /(?<number>\d+)/ # => 4 + * number # => nil # Not assigned. + * /(?<number>\d+)/ =~ 'no. 9' # => 4 + * number # => "9" # Assigned. + * + * When +other+ is not a Regexp, returns the value + * returned by <tt>other =~ self</tt>. + * + * Related: see {Querying}[rdoc-ref:String@Querying]. */ static VALUE -rb_str_match(x, y) - VALUE x, y; +rb_str_match(VALUE x, VALUE y) { - VALUE reg; - long start; - - switch (TYPE(y)) { + switch (OBJ_BUILTIN_TYPE(y)) { case T_STRING: -#if RUBY_VERSION_CODE < 182 - rb_warn("string =~ string will be obsolete; use explicit regexp"); -#endif - y = rb_reg_regcomp(y); - /* fall through */ + rb_raise(rb_eTypeError, "type mismatch: String given"); + case T_REGEXP: - return rb_reg_match(y, x); + return rb_reg_match(y, x); default: - return rb_funcall(y, rb_intern("=~"), 1, x); + return rb_funcall(y, idEqTilde, 1, x); } } +static VALUE get_pat(VALUE); + + /* * call-seq: - * ~str => fixnum or nil - * - * Equivalent to <code>$_</code><code>=~ <i>str</i></code>. + * match(pattern, offset = 0) -> matchdata or nil + * match(pattern, offset = 0) {|matchdata| ... } -> object + * + * Creates a MatchData object based on +self+ and the given arguments; + * updates {Regexp Global Variables}[rdoc-ref:Regexp@Global+Variables]. + * + * - Computes +regexp+ by converting +pattern+ (if not already a Regexp). + * + * regexp = Regexp.new(pattern) + * + * - Computes +matchdata+, which will be either a MatchData object or +nil+ + * (see Regexp#match): + * + * matchdata = regexp.match(self[offset..]) + * + * With no block given, returns the computed +matchdata+ or +nil+: + * + * 'foo'.match('f') # => #<MatchData "f"> + * 'foo'.match('o') # => #<MatchData "o"> + * 'foo'.match('x') # => nil + * 'foo'.match('f', 1) # => nil + * 'foo'.match('o', 1) # => #<MatchData "o"> + * + * With a block given and computed +matchdata+ non-nil, calls the block with +matchdata+; + * returns the block's return value: + * + * 'foo'.match(/o/) {|matchdata| matchdata } # => #<MatchData "o"> + * + * With a block given and +nil+ +matchdata+, does not call the block: + * + * 'foo'.match(/x/) {|matchdata| fail 'Cannot happen' } # => nil + * + * Related: see {Querying}[rdoc-ref:String@Querying]. */ static VALUE -rb_str_match2(str) - VALUE str; +rb_str_match_m(int argc, VALUE *argv, VALUE str) { - StringValue(str); -#if RUBY_VERSION_CODE < 182 - rb_warn("~string will be obsolete; use explicit regexp"); -#endif - return rb_reg_match2(rb_reg_regcomp(rb_reg_quote(str))); + VALUE re, result; + if (argc < 1) + rb_check_arity(argc, 1, 2); + re = argv[0]; + argv[0] = str; + result = rb_funcallv(get_pat(re), rb_intern("match"), argc, argv); + if (!NIL_P(result) && rb_block_given_p()) { + return rb_yield(result); + } + return result; } -static VALUE get_pat _((VALUE, int)); - - /* * call-seq: - * str.match(pattern) => matchdata or nil - * - * Converts <i>pattern</i> to a <code>Regexp</code> (if it isn't already one), - * then invokes its <code>match</code> method on <i>str</i>. - * - * 'hello'.match('(.)\1') #=> #<MatchData:0x401b3d30> - * 'hello'.match('(.)\1')[0] #=> "ll" - * 'hello'.match(/(.)\1/)[0] #=> "ll" - * 'hello'.match('xx') #=> nil + * match?(pattern, offset = 0) -> true or false + * + * Returns whether a match is found for +self+ and the given arguments; + * does not update {Regexp Global Variables}[rdoc-ref:Regexp@Global+Variables]. + * + * Computes +regexp+ by converting +pattern+ (if not already a Regexp): + * + * regexp = Regexp.new(pattern) + * + * Returns +true+ if <tt>self[offset..].match(regexp)</tt> returns a MatchData object, + * +false+ otherwise: + * + * 'foo'.match?(/o/) # => true + * 'foo'.match?('o') # => true + * 'foo'.match?(/x/) # => false + * 'foo'.match?('f', 1) # => false + * 'foo'.match?('o', 1) # => true + * + * Related: see {Querying}[rdoc-ref:String@Querying]. */ static VALUE -rb_str_match_m(str, re) - VALUE str, re; +rb_str_match_m_p(int argc, VALUE *argv, VALUE str) { - return rb_funcall(get_pat(re, 0), rb_intern("match"), 1, str); + VALUE re; + rb_check_arity(argc, 1, 2); + re = get_pat(argv[0]); + return rb_reg_match_p(re, str, argc > 1 ? NUM2LONG(argv[1]) : 0); } -static char -succ_char(s) - char *s; +enum neighbor_char { + NEIGHBOR_NOT_CHAR, + NEIGHBOR_FOUND, + NEIGHBOR_WRAPPED +}; + +static enum neighbor_char +enc_succ_char(char *p, long len, rb_encoding *enc) +{ + long i; + int l; + + if (rb_enc_mbminlen(enc) > 1) { + /* wchar, trivial case */ + int r = rb_enc_precise_mbclen(p, p + len, enc), c; + if (!MBCLEN_CHARFOUND_P(r)) { + return NEIGHBOR_NOT_CHAR; + } + c = rb_enc_mbc_to_codepoint(p, p + len, enc) + 1; + l = rb_enc_code_to_mbclen(c, enc); + if (!l) return NEIGHBOR_NOT_CHAR; + if (l != len) return NEIGHBOR_WRAPPED; + rb_enc_mbcput(c, p, enc); + r = rb_enc_precise_mbclen(p, p + len, enc); + if (!MBCLEN_CHARFOUND_P(r)) { + return NEIGHBOR_NOT_CHAR; + } + return NEIGHBOR_FOUND; + } + while (1) { + for (i = len-1; 0 <= i && (unsigned char)p[i] == 0xff; i--) + p[i] = '\0'; + if (i < 0) + return NEIGHBOR_WRAPPED; + ++((unsigned char*)p)[i]; + l = rb_enc_precise_mbclen(p, p+len, enc); + if (MBCLEN_CHARFOUND_P(l)) { + l = MBCLEN_CHARFOUND_LEN(l); + if (l == len) { + return NEIGHBOR_FOUND; + } + else { + memset(p+l, 0xff, len-l); + } + } + if (MBCLEN_INVALID_P(l) && i < len-1) { + long len2; + int l2; + for (len2 = len-1; 0 < len2; len2--) { + l2 = rb_enc_precise_mbclen(p, p+len2, enc); + if (!MBCLEN_INVALID_P(l2)) + break; + } + memset(p+len2+1, 0xff, len-(len2+1)); + } + } +} + +static enum neighbor_char +enc_pred_char(char *p, long len, rb_encoding *enc) { - char c = *s; + long i; + int l; + if (rb_enc_mbminlen(enc) > 1) { + /* wchar, trivial case */ + int r = rb_enc_precise_mbclen(p, p + len, enc), c; + if (!MBCLEN_CHARFOUND_P(r)) { + return NEIGHBOR_NOT_CHAR; + } + c = rb_enc_mbc_to_codepoint(p, p + len, enc); + if (!c) return NEIGHBOR_NOT_CHAR; + --c; + l = rb_enc_code_to_mbclen(c, enc); + if (!l) return NEIGHBOR_NOT_CHAR; + if (l != len) return NEIGHBOR_WRAPPED; + rb_enc_mbcput(c, p, enc); + r = rb_enc_precise_mbclen(p, p + len, enc); + if (!MBCLEN_CHARFOUND_P(r)) { + return NEIGHBOR_NOT_CHAR; + } + return NEIGHBOR_FOUND; + } + while (1) { + for (i = len-1; 0 <= i && (unsigned char)p[i] == 0; i--) + p[i] = '\xff'; + if (i < 0) + return NEIGHBOR_WRAPPED; + --((unsigned char*)p)[i]; + l = rb_enc_precise_mbclen(p, p+len, enc); + if (MBCLEN_CHARFOUND_P(l)) { + l = MBCLEN_CHARFOUND_LEN(l); + if (l == len) { + return NEIGHBOR_FOUND; + } + else { + memset(p+l, 0, len-l); + } + } + if (MBCLEN_INVALID_P(l) && i < len-1) { + long len2; + int l2; + for (len2 = len-1; 0 < len2; len2--) { + l2 = rb_enc_precise_mbclen(p, p+len2, enc); + if (!MBCLEN_INVALID_P(l2)) + break; + } + memset(p+len2+1, 0, len-(len2+1)); + } + } +} - /* numerics */ - if ('0' <= c && c < '9') (*s)++; - else if (c == '9') { - *s = '0'; - return '1'; +/* + overwrite +p+ by succeeding letter in +enc+ and returns + NEIGHBOR_FOUND or NEIGHBOR_WRAPPED. + When NEIGHBOR_WRAPPED, carried-out letter is stored into carry. + assuming each ranges are successive, and mbclen + never change in each ranges. + NEIGHBOR_NOT_CHAR is returned if invalid character or the range has only one + character. + */ +static enum neighbor_char +enc_succ_alnum_char(char *p, long len, rb_encoding *enc, char *carry) +{ + enum neighbor_char ret; + unsigned int c; + int ctype; + int range; + char save[ONIGENC_CODE_TO_MBC_MAXLEN]; + + /* skip 03A2, invalid char between GREEK CAPITAL LETTERS */ + int try; + const int max_gaps = 1; + + c = rb_enc_mbc_to_codepoint(p, p+len, enc); + if (rb_enc_isctype(c, ONIGENC_CTYPE_DIGIT, enc)) + ctype = ONIGENC_CTYPE_DIGIT; + else if (rb_enc_isctype(c, ONIGENC_CTYPE_ALPHA, enc)) + ctype = ONIGENC_CTYPE_ALPHA; + else + return NEIGHBOR_NOT_CHAR; + + MEMCPY(save, p, char, len); + for (try = 0; try <= max_gaps; ++try) { + ret = enc_succ_char(p, len, enc); + if (ret == NEIGHBOR_FOUND) { + c = rb_enc_mbc_to_codepoint(p, p+len, enc); + if (rb_enc_isctype(c, ctype, enc)) + return NEIGHBOR_FOUND; + } } - /* small alphabets */ - else if ('a' <= c && c < 'z') (*s)++; - else if (c == 'z') { - return *s = 'a'; + MEMCPY(p, save, char, len); + range = 1; + while (1) { + MEMCPY(save, p, char, len); + ret = enc_pred_char(p, len, enc); + if (ret == NEIGHBOR_FOUND) { + c = rb_enc_mbc_to_codepoint(p, p+len, enc); + if (!rb_enc_isctype(c, ctype, enc)) { + MEMCPY(p, save, char, len); + break; + } + } + else { + MEMCPY(p, save, char, len); + break; + } + range++; } - /* capital alphabets */ - else if ('A' <= c && c < 'Z') (*s)++; - else if (c == 'Z') { - return *s = 'A'; + if (range == 1) { + return NEIGHBOR_NOT_CHAR; } - return 0; + + if (ctype != ONIGENC_CTYPE_DIGIT) { + MEMCPY(carry, p, char, len); + return NEIGHBOR_WRAPPED; + } + + MEMCPY(carry, p, char, len); + enc_succ_char(carry, len, enc); + return NEIGHBOR_WRAPPED; } +static VALUE str_succ(VALUE str); + /* * call-seq: - * str.succ => new_str - * str.next => new_str - * - * Returns the successor to <i>str</i>. The successor is calculated by - * incrementing characters starting from the rightmost alphanumeric (or - * the rightmost character if there are no alphanumerics) in the - * string. Incrementing a digit always results in another digit, and - * incrementing a letter results in another letter of the same case. - * Incrementing nonalphanumerics uses the underlying character set's - * collating sequence. - * - * If the increment generates a ``carry,'' the character to the left of - * it is incremented. This process repeats until there is no carry, - * adding an additional character if necessary. - * - * "abcd".succ #=> "abce" - * "THX1138".succ #=> "THX1139" - * "<<koala>>".succ #=> "<<koalb>>" - * "1999zzz".succ #=> "2000aaa" - * "ZZZ9999".succ #=> "AAAA0000" - * "***".succ #=> "**+" + * succ -> new_str + * + * :include: doc/string/succ.rdoc + * */ -static VALUE -rb_str_succ(orig) - VALUE orig; +VALUE +rb_str_succ(VALUE orig) { VALUE str; - char *sbeg, *s; - int c = -1; - long n = 0; - - str = rb_str_new5(orig,RSTRING(orig)->ptr, RSTRING(orig)->len); - OBJ_INFECT(str, orig); - if (RSTRING(str)->len == 0) return str; - - sbeg = RSTRING(str)->ptr; s = sbeg + RSTRING(str)->len - 1; - - while (sbeg <= s) { - if (ISALNUM(*s)) { - if ((c = succ_char(s)) == 0) break; - n = s - sbeg; - } - s--; - } - if (c == -1) { /* str contains no alnum */ - sbeg = RSTRING(str)->ptr; s = sbeg + RSTRING(str)->len - 1; - c = '\001'; - while (sbeg <= s) { - if ((*s += 1) != 0) break; - s--; - } - } - if (s < sbeg) { - RESIZE_CAPA(str, RSTRING(str)->len + 1); - s = RSTRING(str)->ptr + n; - memmove(s+1, s, RSTRING(str)->len - n); - *s = c; - RSTRING(str)->len += 1; - RSTRING(str)->ptr[RSTRING(str)->len] = '\0'; - } + str = rb_str_new(RSTRING_PTR(orig), RSTRING_LEN(orig)); + rb_enc_cr_str_copy_for_substr(str, orig); + return str_succ(str); +} +static VALUE +str_succ(VALUE str) +{ + rb_encoding *enc; + char *sbeg, *s, *e, *last_alnum = 0; + int found_alnum = 0; + long l, slen; + char carry[ONIGENC_CODE_TO_MBC_MAXLEN] = "\1"; + long carry_pos = 0, carry_len = 1; + enum neighbor_char neighbor = NEIGHBOR_FOUND; + + slen = RSTRING_LEN(str); + if (slen == 0) return str; + + enc = STR_ENC_GET(str); + sbeg = RSTRING_PTR(str); + s = e = sbeg + slen; + + while ((s = rb_enc_prev_char(sbeg, s, e, enc)) != 0) { + if (neighbor == NEIGHBOR_NOT_CHAR && last_alnum) { + if (ISALPHA(*last_alnum) ? ISDIGIT(*s) : + ISDIGIT(*last_alnum) ? ISALPHA(*s) : 0) { + break; + } + } + l = rb_enc_precise_mbclen(s, e, enc); + if (!ONIGENC_MBCLEN_CHARFOUND_P(l)) continue; + l = ONIGENC_MBCLEN_CHARFOUND_LEN(l); + neighbor = enc_succ_alnum_char(s, l, enc, carry); + switch (neighbor) { + case NEIGHBOR_NOT_CHAR: + continue; + case NEIGHBOR_FOUND: + return str; + case NEIGHBOR_WRAPPED: + last_alnum = s; + break; + } + found_alnum = 1; + carry_pos = s - sbeg; + carry_len = l; + } + if (!found_alnum) { /* str contains no alnum */ + s = e; + while ((s = rb_enc_prev_char(sbeg, s, e, enc)) != 0) { + enum neighbor_char neighbor; + char tmp[ONIGENC_CODE_TO_MBC_MAXLEN]; + l = rb_enc_precise_mbclen(s, e, enc); + if (!ONIGENC_MBCLEN_CHARFOUND_P(l)) continue; + l = ONIGENC_MBCLEN_CHARFOUND_LEN(l); + MEMCPY(tmp, s, char, l); + neighbor = enc_succ_char(tmp, l, enc); + switch (neighbor) { + case NEIGHBOR_FOUND: + MEMCPY(s, tmp, char, l); + return str; + break; + case NEIGHBOR_WRAPPED: + MEMCPY(s, tmp, char, l); + break; + case NEIGHBOR_NOT_CHAR: + break; + } + if (rb_enc_precise_mbclen(s, s+l, enc) != l) { + /* wrapped to \0...\0. search next valid char. */ + enc_succ_char(s, l, enc); + } + if (!rb_enc_asciicompat(enc)) { + MEMCPY(carry, s, char, l); + carry_len = l; + } + carry_pos = s - sbeg; + } + ENC_CODERANGE_SET(str, ENC_CODERANGE_UNKNOWN); + } + RESIZE_CAPA(str, slen + carry_len); + sbeg = RSTRING_PTR(str); + s = sbeg + carry_pos; + memmove(s + carry_len, s, slen - carry_pos); + memmove(s, carry, carry_len); + slen += carry_len; + STR_SET_LEN(str, slen); + TERM_FILL(&sbeg[slen], rb_enc_mbminlen(enc)); + rb_enc_str_coderange(str); return str; } /* * call-seq: - * str.succ! => str - * str.next! => str - * - * Equivalent to <code>String#succ</code>, but modifies the receiver in - * place. + * succ! -> self + * + * Like String#succ, but modifies +self+ in place; returns +self+. + * + * Related: see {Modifying}[rdoc-ref:String@Modifying]. */ static VALUE -rb_str_succ_bang(str) - VALUE str; +rb_str_succ_bang(VALUE str) { - rb_str_shared_replace(str, rb_str_succ(str)); - + rb_str_modify(str); + str_succ(str); return str; } +static int +all_digits_p(const char *s, long len) +{ + while (len-- > 0) { + if (!ISDIGIT(*s)) return 0; + s++; + } + return 1; +} + +static int +str_upto_i(VALUE str, VALUE arg) +{ + rb_yield(str); + return 0; +} + +/* + * call-seq: + * upto(other_string, exclusive = false) {|string| ... } -> self + * upto(other_string, exclusive = false) -> new_enumerator + * + * :include: doc/string/upto.rdoc + * + */ + +static VALUE +rb_str_upto(int argc, VALUE *argv, VALUE beg) +{ + VALUE end, exclusive; + + rb_scan_args(argc, argv, "11", &end, &exclusive); + RETURN_ENUMERATOR(beg, argc, argv); + return rb_str_upto_each(beg, end, RTEST(exclusive), str_upto_i, Qnil); +} + VALUE -rb_str_upto(beg, end, excl) - VALUE beg, end; - int excl; +rb_str_upto_each(VALUE beg, VALUE end, int excl, int (*each)(VALUE, VALUE), VALUE arg) { VALUE current, after_end; - ID succ = rb_intern("succ"); - int n; + ID succ; + int n, ascii; + rb_encoding *enc; + CONST_ID(succ, "succ"); StringValue(end); + enc = rb_enc_check(beg, end); + ascii = (is_ascii_string(beg) && is_ascii_string(end)); + /* single character */ + if (RSTRING_LEN(beg) == 1 && RSTRING_LEN(end) == 1 && ascii) { + char c = RSTRING_PTR(beg)[0]; + char e = RSTRING_PTR(end)[0]; + + if (c > e || (excl && c == e)) return beg; + for (;;) { + VALUE str = rb_enc_str_new(&c, 1, enc); + ENC_CODERANGE_SET(str, RUBY_ENC_CODERANGE_7BIT); + if ((*each)(str, arg)) break; + if (!excl && c == e) break; + c++; + if (excl && c == e) break; + } + return beg; + } + /* both edges are all digits */ + if (ascii && ISDIGIT(RSTRING_PTR(beg)[0]) && ISDIGIT(RSTRING_PTR(end)[0]) && + all_digits_p(RSTRING_PTR(beg), RSTRING_LEN(beg)) && + all_digits_p(RSTRING_PTR(end), RSTRING_LEN(end))) { + VALUE b, e; + int width; + + width = RSTRING_LENINT(beg); + b = rb_str_to_inum(beg, 10, FALSE); + e = rb_str_to_inum(end, 10, FALSE); + if (FIXNUM_P(b) && FIXNUM_P(e)) { + long bi = FIX2LONG(b); + long ei = FIX2LONG(e); + rb_encoding *usascii = rb_usascii_encoding(); + + while (bi <= ei) { + if (excl && bi == ei) break; + if ((*each)(rb_enc_sprintf(usascii, "%.*ld", width, bi), arg)) break; + bi++; + } + } + else { + ID op = excl ? '<' : idLE; + VALUE args[2], fmt = rb_fstring_lit("%.*d"); + + args[0] = INT2FIX(width); + while (rb_funcall(b, op, 1, e)) { + args[1] = b; + if ((*each)(rb_str_format(numberof(args), args, fmt), arg)) break; + b = rb_funcallv(b, succ, 0, 0); + } + } + return beg; + } + /* normal case */ n = rb_str_cmp(beg, end); if (n > 0 || (excl && n == 0)) return beg; - after_end = rb_funcall(end, succ, 0, 0); - current = beg; + + after_end = rb_funcallv(end, succ, 0, 0); + current = str_duplicate(rb_cString, beg); while (!rb_str_equal(current, after_end)) { - rb_yield(current); - if (!excl && rb_str_equal(current, end)) break; - current = rb_funcall(current, succ, 0, 0); - if (excl && rb_str_equal(current, end)) break; - if (RSTRING(current)->len > RSTRING(end)->len) - break; + VALUE next = Qnil; + if (excl || !rb_str_equal(current, end)) + next = rb_funcallv(current, succ, 0, 0); + if ((*each)(current, arg)) break; + if (NIL_P(next)) break; + current = next; + StringValue(current); + if (excl && rb_str_equal(current, end)) break; + if (RSTRING_LEN(current) > RSTRING_LEN(end) || RSTRING_LEN(current) == 0) + break; } return beg; } +VALUE +rb_str_upto_endless_each(VALUE beg, int (*each)(VALUE, VALUE), VALUE arg) +{ + VALUE current; + ID succ; + + CONST_ID(succ, "succ"); + /* both edges are all digits */ + if (is_ascii_string(beg) && ISDIGIT(RSTRING_PTR(beg)[0]) && + all_digits_p(RSTRING_PTR(beg), RSTRING_LEN(beg))) { + VALUE b, args[2], fmt = rb_fstring_lit("%.*d"); + int width = RSTRING_LENINT(beg); + b = rb_str_to_inum(beg, 10, FALSE); + if (FIXNUM_P(b)) { + long bi = FIX2LONG(b); + rb_encoding *usascii = rb_usascii_encoding(); + + while (FIXABLE(bi)) { + if ((*each)(rb_enc_sprintf(usascii, "%.*ld", width, bi), arg)) break; + bi++; + } + b = LONG2NUM(bi); + } + args[0] = INT2FIX(width); + while (1) { + args[1] = b; + if ((*each)(rb_str_format(numberof(args), args, fmt), arg)) break; + b = rb_funcallv(b, succ, 0, 0); + } + } + /* normal case */ + current = str_duplicate(rb_cString, beg); + while (1) { + VALUE next = rb_funcallv(current, succ, 0, 0); + if ((*each)(current, arg)) break; + current = next; + StringValue(current); + if (RSTRING_LEN(current) == 0) + break; + } -/* - * call-seq: - * str.upto(other_str) {|s| block } => str - * - * Iterates through successive values, starting at <i>str</i> and - * ending at <i>other_str</i> inclusive, passing each value in turn to - * the block. The <code>String#succ</code> method is used to generate - * each value. - * - * "a8".upto("b6") {|s| print s, ' ' } - * for s in "a8".."b6" - * print s, ' ' - * end - * - * <em>produces:</em> - * - * a8 a9 b0 b1 b2 b3 b4 b5 b6 - * a8 a9 b0 b1 b2 b3 b4 b5 b6 - */ + return beg; +} -static VALUE -rb_str_upto_m(beg, end) - VALUE beg, end; +static int +include_range_i(VALUE str, VALUE arg) { - return rb_str_upto(beg, end, Qfalse); + VALUE *argp = (VALUE *)arg; + if (!rb_equal(str, *argp)) return 0; + *argp = Qnil; + return 1; +} + +VALUE +rb_str_include_range_p(VALUE beg, VALUE end, VALUE val, VALUE exclusive) +{ + beg = rb_str_new_frozen(beg); + StringValue(end); + end = rb_str_new_frozen(end); + if (NIL_P(val)) return Qfalse; + val = rb_check_string_type(val); + if (NIL_P(val)) return Qfalse; + if (rb_enc_asciicompat(STR_ENC_GET(beg)) && + rb_enc_asciicompat(STR_ENC_GET(end)) && + rb_enc_asciicompat(STR_ENC_GET(val))) { + const char *bp = RSTRING_PTR(beg); + const char *ep = RSTRING_PTR(end); + const char *vp = RSTRING_PTR(val); + if (RSTRING_LEN(beg) == 1 && RSTRING_LEN(end) == 1) { + if (RSTRING_LEN(val) == 0 || RSTRING_LEN(val) > 1) + return Qfalse; + else { + char b = *bp; + char e = *ep; + char v = *vp; + + if (ISASCII(b) && ISASCII(e) && ISASCII(v)) { + if (b <= v && v < e) return Qtrue; + return RBOOL(!RTEST(exclusive) && v == e); + } + } + } +#if 0 + /* both edges are all digits */ + if (ISDIGIT(*bp) && ISDIGIT(*ep) && + all_digits_p(bp, RSTRING_LEN(beg)) && + all_digits_p(ep, RSTRING_LEN(end))) { + /* TODO */ + } +#endif + } + rb_str_upto_each(beg, end, RTEST(exclusive), include_range_i, (VALUE)&val); + + return RBOOL(NIL_P(val)); } static VALUE -rb_str_subpat(str, re, nth) - VALUE str, re; - int nth; +rb_str_subpat(VALUE str, VALUE re, VALUE backref) { if (rb_reg_search(re, str, 0, 0) >= 0) { - return rb_reg_nth_match(nth, rb_backref_get()); + VALUE match = rb_backref_get(); + int nth = rb_reg_backref_number(match, backref); + return rb_reg_nth_match(nth, match); } return Qnil; } - + static VALUE -rb_str_aref(str, indx) - VALUE str; - VALUE indx; +rb_str_aref(VALUE str, VALUE indx) { long idx; - switch (TYPE(indx)) { - case T_FIXNUM: - idx = FIX2LONG(indx); + if (FIXNUM_P(indx)) { + idx = FIX2LONG(indx); + } + else if (RB_TYPE_P(indx, T_REGEXP)) { + return rb_str_subpat(str, indx, INT2FIX(0)); + } + else if (RB_TYPE_P(indx, T_STRING)) { + if (rb_str_index(str, indx, 0) != -1) + return str_duplicate(rb_cString, indx); + return Qnil; + } + else { + /* check if indx is Range */ + long beg, len = str_strlen(str, NULL); + switch (rb_range_beg_len(indx, &beg, &len, len, 0)) { + case Qfalse: + break; + case Qnil: + return Qnil; + default: + return rb_str_substr(str, beg, len); + } + idx = NUM2LONG(indx); + } - num_index: - if (idx < 0) { - idx = RSTRING(str)->len + idx; - } - if (idx < 0 || RSTRING(str)->len <= idx) { - return Qnil; - } - return INT2FIX(RSTRING(str)->ptr[idx] & 0xff); + return str_substr(str, idx, 1, FALSE); +} - case T_REGEXP: - return rb_str_subpat(str, indx, 0); - case T_STRING: - if (rb_str_index(str, indx, 0) != -1) - return rb_str_dup(indx); - return Qnil; +/* + * call-seq: + * self[offset] -> new_string or nil + * self[offset, size] -> new_string or nil + * self[range] -> new_string or nil + * self[regexp, capture = 0] -> new_string or nil + * self[substring] -> new_string or nil + * + * :include: doc/string/aref.rdoc + * + */ - default: - /* check if indx is Range */ - { - long beg, len; - switch (rb_range_beg_len(indx, &beg, &len, RSTRING(str)->len, 0)) { - case Qfalse: - break; - case Qnil: - return Qnil; - default: - return rb_str_substr(str, beg, len); - } - } - idx = NUM2LONG(indx); - goto num_index; - } - return Qnil; /* not reached */ -} - - -/* - * call-seq: - * str[fixnum] => fixnum or nil - * str[fixnum, fixnum] => new_str or nil - * str[range] => new_str or nil - * str[regexp] => new_str or nil - * str[regexp, fixnum] => new_str or nil - * str[other_str] => new_str or nil - * str.slice(fixnum) => fixnum or nil - * str.slice(fixnum, fixnum) => new_str or nil - * str.slice(range) => new_str or nil - * str.slice(regexp) => new_str or nil - * str.slice(regexp, fixnum) => new_str or nil - * str.slice(other_str) => new_str or nil - * - * Element Reference---If passed a single <code>Fixnum</code>, returns the code - * of the character at that position. If passed two <code>Fixnum</code> - * objects, returns a substring starting at the offset given by the first, and - * a length given by the second. If given a range, a substring containing - * characters at offsets given by the range is returned. In all three cases, if - * an offset is negative, it is counted from the end of <i>str</i>. Returns - * <code>nil</code> if the initial offset falls outside the string, the length - * is negative, or the beginning of the range is greater than the end. - * - * If a <code>Regexp</code> is supplied, the matching portion of <i>str</i> is - * returned. If a numeric parameter follows the regular expression, that - * component of the <code>MatchData</code> is returned instead. If a - * <code>String</code> is given, that string is returned if it occurs in - * <i>str</i>. In both cases, <code>nil</code> is returned if there is no - * match. - * - * a = "hello there" - * a[1] #=> 101 - * a[1,3] #=> "ell" - * a[1..3] #=> "ell" - * a[-3,2] #=> "er" - * a[-4..-2] #=> "her" - * a[-2..-4] #=> nil - * a[/[aeiou](.)\1/] #=> "ell" - * a[/[aeiou](.)\1/, 0] #=> "ell" - * a[/[aeiou](.)\1/, 1] #=> "l" - * a[/[aeiou](.)\1/, 2] #=> nil - * a["lo"] #=> "lo" - * a["bye"] #=> nil - */ - -static VALUE -rb_str_aref_m(argc, argv, str) - int argc; - VALUE *argv; - VALUE str; +static VALUE +rb_str_aref_m(int argc, VALUE *argv, VALUE str) { if (argc == 2) { - if (TYPE(argv[0]) == T_REGEXP) { - return rb_str_subpat(str, argv[0], NUM2INT(argv[1])); - } - return rb_str_substr(str, NUM2LONG(argv[0]), NUM2LONG(argv[1])); - } - if (argc != 1) { - rb_raise(rb_eArgError, "wrong number of arguments (%d for 1)", argc); + if (RB_TYPE_P(argv[0], T_REGEXP)) { + return rb_str_subpat(str, argv[0], argv[1]); + } + else { + return rb_str_substr_two_fixnums(str, argv[0], argv[1], TRUE); + } } + rb_check_arity(argc, 1, 2); return rb_str_aref(str, argv[0]); } -static void -rb_str_splice(str, beg, len, val) - VALUE str; - long beg, len; - VALUE val; -{ - if (len < 0) rb_raise(rb_eIndexError, "negative length %ld", len); - if (RSTRING(str)->len < beg) { - out_of_range: - rb_raise(rb_eIndexError, "index %ld out of string", beg); +VALUE +rb_str_drop_bytes(VALUE str, long len) +{ + char *ptr = RSTRING_PTR(str); + long olen = RSTRING_LEN(str), nlen; + + str_modifiable(str); + if (len > olen) len = olen; + nlen = olen - len; + if (str_embed_capa(str) >= nlen + TERM_LEN(str)) { + char *oldptr = ptr; + int fl = (int)(RBASIC(str)->flags & (STR_NOEMBED|STR_SHARED|STR_NOFREE)); + STR_SET_EMBED(str); + ptr = RSTRING(str)->as.embed.ary; + memmove(ptr, oldptr + len, nlen); + if (fl == STR_NOEMBED) xfree(oldptr); } - if (beg < 0) { - if (-beg > RSTRING(str)->len) { - goto out_of_range; - } - beg += RSTRING(str)->len; + else { + if (!STR_SHARED_P(str)) { + VALUE shared = heap_str_make_shared(rb_obj_class(str), str); + rb_enc_cr_str_exact_copy(shared, str); + OBJ_FREEZE(shared); + } + ptr = RSTRING(str)->as.heap.ptr += len; } - if (RSTRING(str)->len < beg + len) { - len = RSTRING(str)->len - beg; + STR_SET_LEN(str, nlen); + + if (!SHARABLE_MIDDLE_SUBSTRING) { + TERM_FILL(ptr + nlen, TERM_LEN(str)); } + ENC_CODERANGE_CLEAR(str); + return str; +} - StringValue(val); - if (len < RSTRING(val)->len) { - /* expand string */ - RESIZE_CAPA(str, RSTRING(str)->len + RSTRING(val)->len - len); +static void +rb_str_update_1(VALUE str, long beg, long len, VALUE val, long vbeg, long vlen) +{ + char *sptr; + long slen; + int cr; + + if (beg == 0 && vlen == 0) { + rb_str_drop_bytes(str, len); + return; } - if (RSTRING(val)->len != len) { - memmove(RSTRING(str)->ptr + beg + RSTRING(val)->len, - RSTRING(str)->ptr + beg + len, - RSTRING(str)->len - (beg + len)); + str_modify_keep_cr(str); + RSTRING_GETMEM(str, sptr, slen); + if (len < vlen) { + /* expand string */ + RESIZE_CAPA(str, slen + vlen - len); + sptr = RSTRING_PTR(str); } - if (RSTRING(str)->len < beg && len < 0) { - MEMZERO(RSTRING(str)->ptr + RSTRING(str)->len, char, -len); + + if (ENC_CODERANGE(str) == ENC_CODERANGE_7BIT) + cr = rb_enc_str_coderange(val); + else + cr = ENC_CODERANGE_UNKNOWN; + + if (vlen != len) { + memmove(sptr + beg + vlen, + sptr + beg + len, + slen - (beg + len)); } - if (RSTRING(val)->len > 0) { - memmove(RSTRING(str)->ptr+beg, RSTRING(val)->ptr, RSTRING(val)->len); + if (vlen < beg && len < 0) { + MEMZERO(sptr + slen, char, -len); } - RSTRING(str)->len += RSTRING(val)->len - len; - if (RSTRING(str)->ptr) { - RSTRING(str)->ptr[RSTRING(str)->len] = '\0'; + if (vlen > 0) { + memmove(sptr + beg, RSTRING_PTR(val) + vbeg, vlen); } - OBJ_INFECT(str, val); + slen += vlen - len; + STR_SET_LEN(str, slen); + TERM_FILL(&sptr[slen], TERM_LEN(str)); + ENC_CODERANGE_SET(str, cr); +} + +static inline void +rb_str_update_0(VALUE str, long beg, long len, VALUE val) +{ + rb_str_update_1(str, beg, len, val, 0, RSTRING_LEN(val)); } void -rb_str_update(str, beg, len, val) - VALUE str; - long beg, len; - VALUE val; +rb_str_update(VALUE str, long beg, long len, VALUE val) { - rb_str_modify(str); - rb_str_splice(str, beg, len, val); + long slen; + char *p, *e; + rb_encoding *enc; + int singlebyte = single_byte_optimizable(str); + int cr; + + if (len < 0) rb_raise(rb_eIndexError, "negative length %ld", len); + + StringValue(val); + enc = rb_enc_check(str, val); + slen = str_strlen(str, enc); /* rb_enc_check */ + + if ((slen < beg) || ((beg < 0) && (beg + slen < 0))) { + rb_raise(rb_eIndexError, "index %ld out of string", beg); + } + if (beg < 0) { + beg += slen; + } + RUBY_ASSERT(beg >= 0); + RUBY_ASSERT(beg <= slen); + + if (len > slen - beg) { + len = slen - beg; + } + p = str_nth(RSTRING_PTR(str), RSTRING_END(str), beg, enc, singlebyte); + if (!p) p = RSTRING_END(str); + e = str_nth(p, RSTRING_END(str), len, enc, singlebyte); + if (!e) e = RSTRING_END(str); + /* error check */ + beg = p - RSTRING_PTR(str); /* physical position */ + len = e - p; /* physical length */ + rb_str_update_0(str, beg, len, val); + rb_enc_associate(str, enc); + cr = ENC_CODERANGE_AND(ENC_CODERANGE(str), ENC_CODERANGE(val)); + if (cr != ENC_CODERANGE_BROKEN) + ENC_CODERANGE_SET(str, cr); } static void -rb_str_subpat_set(str, re, nth, val) - VALUE str, re; - int nth; - VALUE val; +rb_str_subpat_set(VALUE str, VALUE re, VALUE backref, VALUE val) { + int nth; VALUE match; long start, end, len; + rb_encoding *enc; + struct re_registers *regs; if (rb_reg_search(re, str, 0, 0) < 0) { - rb_raise(rb_eIndexError, "regexp not matched"); + rb_raise(rb_eIndexError, "regexp not matched"); } match = rb_backref_get(); - if (nth >= RMATCH(match)->regs->num_regs) { - out_of_range: - rb_raise(rb_eIndexError, "index %d out of regexp", nth); + nth = rb_reg_backref_number(match, backref); + regs = RMATCH_REGS(match); + if ((nth >= regs->num_regs) || ((nth < 0) && (-nth >= regs->num_regs))) { + rb_raise(rb_eIndexError, "index %d out of regexp", nth); } if (nth < 0) { - if (-nth >= RMATCH(match)->regs->num_regs) { - goto out_of_range; - } - nth += RMATCH(match)->regs->num_regs; + nth += regs->num_regs; } - start = RMATCH(match)->BEG(nth); + start = BEG(nth); if (start == -1) { - rb_raise(rb_eIndexError, "regexp group %d not matched", nth); + rb_raise(rb_eIndexError, "regexp group %d not matched", nth); } - end = RMATCH(match)->END(nth); + end = END(nth); len = end - start; - rb_str_modify(str); - rb_str_splice(str, start, len, val); + StringValue(val); + enc = rb_enc_check_str(str, val); + rb_str_update_0(str, start, len, val); + rb_enc_associate(str, enc); } static VALUE -rb_str_aset(str, indx, val) - VALUE str; - VALUE indx, val; +rb_str_aset(VALUE str, VALUE indx, VALUE val) { long idx, beg; switch (TYPE(indx)) { - case T_FIXNUM: - num_index: - idx = FIX2LONG(indx); - if (RSTRING(str)->len <= idx) { - out_of_range: - rb_raise(rb_eIndexError, "index %ld out of string", idx); - } - if (idx < 0) { - if (-idx > RSTRING(str)->len) - goto out_of_range; - idx += RSTRING(str)->len; - } - if (FIXNUM_P(val)) { - if (RSTRING(str)->len == idx) { - RSTRING(str)->len += 1; - RESIZE_CAPA(str, RSTRING(str)->len); - } - RSTRING(str)->ptr[idx] = FIX2INT(val) & 0xff; - } - else { - rb_str_splice(str, idx, 1, val); - } - return val; - case T_REGEXP: - rb_str_subpat_set(str, indx, 0, val); - return val; + rb_str_subpat_set(str, indx, INT2FIX(0), val); + return val; case T_STRING: - beg = rb_str_index(str, indx, 0); - if (beg < 0) { - rb_raise(rb_eIndexError, "string not matched"); - } - rb_str_splice(str, beg, RSTRING(indx)->len, val); - return val; + beg = rb_str_index(str, indx, 0); + if (beg < 0) { + rb_raise(rb_eIndexError, "string not matched"); + } + beg = rb_str_sublen(str, beg); + rb_str_update(str, beg, str_strlen(indx, NULL), val); + return val; default: - /* check if indx is Range */ - { - long beg, len; - if (rb_range_beg_len(indx, &beg, &len, RSTRING(str)->len, 2)) { - rb_str_splice(str, beg, len, val); - return val; - } - } - idx = NUM2LONG(indx); - goto num_index; - } -} - -/* - * call-seq: - * str[fixnum] = fixnum - * str[fixnum] = new_str - * str[fixnum, fixnum] = new_str - * str[range] = aString - * str[regexp] = new_str - * str[regexp, fixnum] = new_str - * str[other_str] = new_str - * - * Element Assignment---Replaces some or all of the content of <i>str</i>. The - * portion of the string affected is determined using the same criteria as - * <code>String#[]</code>. If the replacement string is not the same length as - * the text it is replacing, the string will be adjusted accordingly. If the - * regular expression or string is used as the index doesn't match a position - * in the string, <code>IndexError</code> is raised. If the regular expression - * form is used, the optional second <code>Fixnum</code> allows you to specify - * which portion of the match to replace (effectively using the - * <code>MatchData</code> indexing rules. The forms that take a - * <code>Fixnum</code> will raise an <code>IndexError</code> if the value is - * out of range; the <code>Range</code> form will raise a - * <code>RangeError</code>, and the <code>Regexp</code> and <code>String</code> - * forms will silently ignore the assignment. - */ - -static VALUE -rb_str_aset_m(argc, argv, str) - int argc; - VALUE *argv; - VALUE str; + /* check if indx is Range */ + { + long beg, len; + if (rb_range_beg_len(indx, &beg, &len, str_strlen(str, NULL), 2)) { + rb_str_update(str, beg, len, val); + return val; + } + } + /* FALLTHROUGH */ + + case T_FIXNUM: + idx = NUM2LONG(indx); + rb_str_update(str, idx, 1, val); + return val; + } +} + +/* + * call-seq: + * self[index] = other_string -> new_string + * self[start, length] = other_string -> new_string + * self[range] = other_string -> new_string + * self[regexp, capture = 0] = other_string -> new_string + * self[substring] = other_string -> new_string + * + * :include: doc/string/aset.rdoc + * + */ + +static VALUE +rb_str_aset_m(int argc, VALUE *argv, VALUE str) { - rb_str_modify(str); if (argc == 3) { - if (TYPE(argv[0]) == T_REGEXP) { - rb_str_subpat_set(str, argv[0], NUM2INT(argv[1]), argv[2]); - } - else { - rb_str_splice(str, NUM2LONG(argv[0]), NUM2LONG(argv[1]), argv[2]); - } - return argv[2]; - } - if (argc != 2) { - rb_raise(rb_eArgError, "wrong number of arguments (%d for 2)", argc); + if (RB_TYPE_P(argv[0], T_REGEXP)) { + rb_str_subpat_set(str, argv[0], argv[1], argv[2]); + } + else { + rb_str_update(str, NUM2LONG(argv[0]), NUM2LONG(argv[1]), argv[2]); + } + return argv[2]; } + rb_check_arity(argc, 2, 3); return rb_str_aset(str, argv[0], argv[1]); } /* * call-seq: - * str.insert(index, other_str) => str - * - * Inserts <i>other_str</i> before the character at the given - * <i>index</i>, modifying <i>str</i>. Negative indices count from the - * end of the string, and insert <em>after</em> the given character. - * The intent is insert <i>aString</i> so that it starts at the given - * <i>index</i>. - * - * "abcd".insert(0, 'X') #=> "Xabcd" - * "abcd".insert(3, 'X') #=> "abcXd" - * "abcd".insert(4, 'X') #=> "abcdX" - * "abcd".insert(-3, 'X') #=> "abXcd" - * "abcd".insert(-1, 'X') #=> "abcdX" + * insert(offset, other_string) -> self + * + * :include: doc/string/insert.rdoc + * */ static VALUE -rb_str_insert(str, idx, str2) - VALUE str, idx, str2; +rb_str_insert(VALUE str, VALUE idx, VALUE str2) { long pos = NUM2LONG(idx); - rb_str_modify(str); if (pos == -1) { - pos = RSTRING(str)->len; + return rb_str_append(str, str2); } else if (pos < 0) { - pos++; + pos++; } - rb_str_splice(str, pos, 0, str2); + rb_str_update(str, pos, 0, str2); return str; } + /* * call-seq: - * str.slice!(fixnum) => fixnum or nil - * str.slice!(fixnum, fixnum) => new_str or nil - * str.slice!(range) => new_str or nil - * str.slice!(regexp) => new_str or nil - * str.slice!(other_str) => new_str or nil - * - * Deletes the specified portion from <i>str</i>, and returns the portion - * deleted. The forms that take a <code>Fixnum</code> will raise an - * <code>IndexError</code> if the value is out of range; the <code>Range</code> - * form will raise a <code>RangeError</code>, and the <code>Regexp</code> and - * <code>String</code> forms will silently ignore the assignment. - * - * string = "this is a string" - * string.slice!(2) #=> 105 - * string.slice!(3..6) #=> " is " - * string.slice!(/s.*t/) #=> "sa st" - * string.slice!("r") #=> "r" - * string #=> "thing" + * slice!(index) -> new_string or nil + * slice!(start, length) -> new_string or nil + * slice!(range) -> new_string or nil + * slice!(regexp, capture = 0) -> new_string or nil + * slice!(substring) -> new_string or nil + * + * Like String#[] (and its alias String#slice), except that: + * + * - Performs substitutions in +self+ (not in a copy of +self+). + * - Returns the removed substring if any modifications were made, +nil+ otherwise. + * + * A few examples: + * + * s = 'hello' + * s.slice!('e') # => "e" + * s # => "hllo" + * s.slice!('e') # => nil + * s # => "hllo" + * + * Related: see {Modifying}[rdoc-ref:String@Modifying]. */ static VALUE -rb_str_slice_bang(argc, argv, str) - int argc; - VALUE *argv; - VALUE str; +rb_str_slice_bang(int argc, VALUE *argv, VALUE str) { - VALUE result; - VALUE buf[3]; - int i; - - if (argc < 1 || 2 < argc) { - rb_raise(rb_eArgError, "wrong number of arguments (%d for 1)", argc); + VALUE result = Qnil; + VALUE indx; + long beg, len = 1; + char *p; + + rb_check_arity(argc, 1, 2); + str_modify_keep_cr(str); + indx = argv[0]; + if (RB_TYPE_P(indx, T_REGEXP)) { + if (rb_reg_search(indx, str, 0, 0) < 0) return Qnil; + VALUE match = rb_backref_get(); + struct re_registers *regs = RMATCH_REGS(match); + int nth = 0; + if (argc > 1 && (nth = rb_reg_backref_number(match, argv[1])) < 0) { + if ((nth += regs->num_regs) <= 0) return Qnil; + } + else if (nth >= regs->num_regs) return Qnil; + beg = BEG(nth); + len = END(nth) - beg; + goto subseq; } - for (i=0; i<argc; i++) { - buf[i] = argv[i]; + else if (argc == 2) { + beg = NUM2LONG(indx); + len = NUM2LONG(argv[1]); + goto num_index; + } + else if (FIXNUM_P(indx)) { + beg = FIX2LONG(indx); + if (!(p = rb_str_subpos(str, beg, &len))) return Qnil; + if (!len) return Qnil; + beg = p - RSTRING_PTR(str); + goto subseq; + } + else if (RB_TYPE_P(indx, T_STRING)) { + beg = rb_str_index(str, indx, 0); + if (beg == -1) return Qnil; + len = RSTRING_LEN(indx); + result = str_duplicate(rb_cString, indx); + goto squash; } - buf[i] = rb_str_new(0,0); - result = rb_str_aref_m(argc, buf, str); - if (!NIL_P(result)) { - rb_str_aset_m(argc+1, buf, str); + else { + switch (rb_range_beg_len(indx, &beg, &len, str_strlen(str, NULL), 0)) { + case Qnil: + return Qnil; + case Qfalse: + beg = NUM2LONG(indx); + if (!(p = rb_str_subpos(str, beg, &len))) return Qnil; + if (!len) return Qnil; + beg = p - RSTRING_PTR(str); + goto subseq; + default: + goto num_index; + } + } + + num_index: + if (!(p = rb_str_subpos(str, beg, &len))) return Qnil; + beg = p - RSTRING_PTR(str); + + subseq: + result = rb_str_new(RSTRING_PTR(str)+beg, len); + rb_enc_cr_str_copy_for_substr(result, str); + + squash: + if (len > 0) { + if (beg == 0) { + rb_str_drop_bytes(str, len); + } + else { + char *sptr = RSTRING_PTR(str); + long slen = RSTRING_LEN(str); + if (beg + len > slen) /* pathological check */ + len = slen - beg; + memmove(sptr + beg, + sptr + beg + len, + slen - (beg + len)); + slen -= len; + STR_SET_LEN(str, slen); + TERM_FILL(&sptr[slen], TERM_LEN(str)); + } } return result; } static VALUE -get_pat(pat, quote) - VALUE pat; - int quote; +get_pat(VALUE pat) +{ + VALUE val; + + switch (OBJ_BUILTIN_TYPE(pat)) { + case T_REGEXP: + return pat; + + case T_STRING: + break; + + default: + val = rb_check_string_type(pat); + if (NIL_P(val)) { + Check_Type(pat, T_REGEXP); + } + pat = val; + } + + return rb_reg_regcomp(pat); +} + +static VALUE +get_pat_quoted(VALUE pat, int check) { VALUE val; - switch (TYPE(pat)) { + switch (OBJ_BUILTIN_TYPE(pat)) { case T_REGEXP: - return pat; + return pat; case T_STRING: - break; + break; default: - val = rb_check_string_type(pat); - if (NIL_P(val)) { - Check_Type(pat, T_REGEXP); - } - pat = val; + val = rb_check_string_type(pat); + if (NIL_P(val)) { + Check_Type(pat, T_REGEXP); + } + pat = val; + } + if (check && is_broken_string(pat)) { + rb_exc_raise(rb_reg_check_preprocess(pat)); } + return pat; +} - if (quote) { - pat = rb_reg_quote(pat); +static long +rb_pat_search0(VALUE pat, VALUE str, long pos, int set_backref_str, VALUE *match) +{ + if (BUILTIN_TYPE(pat) == T_STRING) { + pos = rb_str_byteindex(str, pat, pos); + if (set_backref_str) { + if (pos >= 0) { + str = rb_str_new_frozen_String(str); + VALUE match_data = rb_backref_set_string(str, pos, RSTRING_LEN(pat)); + if (match) { + *match = match_data; + } + } + else { + rb_backref_set(Qnil); + } + } + return pos; + } + else { + return rb_reg_search0(pat, str, pos, 0, set_backref_str, match); } +} - return rb_reg_regcomp(pat); +static long +rb_pat_search(VALUE pat, VALUE str, long pos, int set_backref_str) +{ + return rb_pat_search0(pat, str, pos, set_backref_str, NULL); } /* * call-seq: - * str.sub!(pattern, replacement) => str or nil - * str.sub!(pattern) {|match| block } => str or nil - * - * Performs the substitutions of <code>String#sub</code> in place, - * returning <i>str</i>, or <code>nil</code> if no substitutions were - * performed. + * sub!(pattern, replacement) -> self or nil + * sub!(pattern) {|match| ... } -> self or nil + * + * Like String#sub, except that: + * + * - Changes are made to +self+, not to copy of +self+. + * - Returns +self+ if any changes are made, +nil+ otherwise. + * + * Related: see {Modifying}[rdoc-ref:String@Modifying]. */ static VALUE -rb_str_sub_bang(argc, argv, str) - int argc; - VALUE *argv; - VALUE str; +rb_str_sub_bang(int argc, VALUE *argv, VALUE str) { - VALUE pat, repl, match; - struct re_registers *regs; + VALUE pat, repl, hash = Qnil; int iter = 0; - int tainted = 0; long plen; + int min_arity = rb_block_given_p() ? 1 : 2; + long beg; - if (argc == 1 && rb_block_given_p()) { - iter = 1; - } - else if (argc == 2) { - repl = argv[1]; - StringValue(repl); - if (OBJ_TAINTED(repl)) tainted = 1; + rb_check_arity(argc, min_arity, 2); + if (argc == 1) { + iter = 1; } else { - rb_raise(rb_eArgError, "wrong number of arguments (%d for 2)", argc); - } - - pat = get_pat(argv[0], 1); - if (rb_reg_search(pat, str, 0, 0) >= 0) { - rb_str_modify(str); - match = rb_backref_get(); - regs = RMATCH(match)->regs; - - if (iter) { - rb_match_busy(match); - repl = rb_obj_as_string(rb_yield(rb_reg_nth_match(0, match))); - rb_backref_set(match); - } - else { - repl = rb_reg_regsub(repl, str, regs); - } - if (OBJ_TAINTED(repl)) tainted = 1; - plen = END(0) - BEG(0); - if (RSTRING(repl)->len > plen) { - RESIZE_CAPA(str, RSTRING(str)->len + RSTRING(repl)->len - plen); - } - if (RSTRING(repl)->len != plen) { - memmove(RSTRING(str)->ptr + BEG(0) + RSTRING(repl)->len, - RSTRING(str)->ptr + BEG(0) + plen, - RSTRING(str)->len - BEG(0) - plen); - } - memcpy(RSTRING(str)->ptr + BEG(0), - RSTRING(repl)->ptr, RSTRING(repl)->len); - RSTRING(str)->len += RSTRING(repl)->len - plen; - RSTRING(str)->ptr[RSTRING(str)->len] = '\0'; - if (tainted) OBJ_TAINT(str); - - return str; + repl = argv[1]; + hash = rb_check_hash_type(argv[1]); + if (NIL_P(hash)) { + StringValue(repl); + } + } + + pat = get_pat_quoted(argv[0], 1); + + str_modifiable(str); + beg = rb_pat_search(pat, str, 0, 1); + if (beg >= 0) { + rb_encoding *enc; + int cr = ENC_CODERANGE(str); + long beg0, end0; + VALUE match, match0 = Qnil; + struct re_registers *regs; + char *p, *rp; + long len, rlen; + + match = rb_backref_get(); + regs = RMATCH_REGS(match); + if (RB_TYPE_P(pat, T_STRING)) { + beg0 = beg; + end0 = beg0 + RSTRING_LEN(pat); + match0 = pat; + } + else { + beg0 = BEG(0); + end0 = END(0); + if (iter) match0 = rb_reg_nth_match(0, match); + } + + if (iter || !NIL_P(hash)) { + p = RSTRING_PTR(str); len = RSTRING_LEN(str); + + if (iter) { + repl = rb_obj_as_string(rb_yield(match0)); + } + else { + repl = rb_hash_aref(hash, rb_str_subseq(str, beg0, end0 - beg0)); + repl = rb_obj_as_string(repl); + } + str_mod_check(str, p, len); + rb_check_frozen(str); + } + else { + repl = rb_reg_regsub(repl, str, regs, RB_TYPE_P(pat, T_STRING) ? Qnil : pat); + } + + enc = rb_enc_compatible(str, repl); + if (!enc) { + rb_encoding *str_enc = STR_ENC_GET(str); + p = RSTRING_PTR(str); len = RSTRING_LEN(str); + if (coderange_scan(p, beg0, str_enc) != ENC_CODERANGE_7BIT || + coderange_scan(p+end0, len-end0, str_enc) != ENC_CODERANGE_7BIT) { + rb_raise(rb_eEncCompatError, "incompatible character encodings: %s and %s", + rb_enc_inspect_name(str_enc), + rb_enc_inspect_name(STR_ENC_GET(repl))); + } + enc = STR_ENC_GET(repl); + } + rb_str_modify(str); + rb_enc_associate(str, enc); + if (ENC_CODERANGE_UNKNOWN < cr && cr < ENC_CODERANGE_BROKEN) { + int cr2 = ENC_CODERANGE(repl); + if (cr2 == ENC_CODERANGE_BROKEN || + (cr == ENC_CODERANGE_VALID && cr2 == ENC_CODERANGE_7BIT)) + cr = ENC_CODERANGE_UNKNOWN; + else + cr = cr2; + } + plen = end0 - beg0; + rlen = RSTRING_LEN(repl); + len = RSTRING_LEN(str); + if (rlen > plen) { + RESIZE_CAPA(str, len + rlen - plen); + } + p = RSTRING_PTR(str); + if (rlen != plen) { + memmove(p + beg0 + rlen, p + beg0 + plen, len - beg0 - plen); + } + rp = RSTRING_PTR(repl); + memmove(p + beg0, rp, rlen); + len += rlen - plen; + STR_SET_LEN(str, len); + TERM_FILL(&RSTRING_PTR(str)[len], TERM_LEN(str)); + ENC_CODERANGE_SET(str, cr); + + RB_GC_GUARD(match); + + return str; } return Qnil; } @@ -1968,213 +6316,231 @@ rb_str_sub_bang(argc, argv, str) /* * call-seq: - * str.sub(pattern, replacement) => new_str - * str.sub(pattern) {|match| block } => new_str - * - * Returns a copy of <i>str</i> with the <em>first</em> occurrence of - * <i>pattern</i> replaced with either <i>replacement</i> or the value of the - * block. The <i>pattern</i> will typically be a <code>Regexp</code>; if it is - * a <code>String</code> then no regular expression metacharacters will be - * interpreted (that is <code>/\d/</code> will match a digit, but - * <code>'\d'</code> will match a backslash followed by a 'd'). - * - * If the method call specifies <i>replacement</i>, special variables such as - * <code>$&</code> will not be useful, as substitution into the string occurs - * before the pattern match starts. However, the sequences <code>\1</code>, - * <code>\2</code>, etc., may be used. - * - * In the block form, the current match string is passed in as a parameter, and - * variables such as <code>$1</code>, <code>$2</code>, <code>$`</code>, - * <code>$&</code>, and <code>$'</code> will be set appropriately. The value - * returned by the block will be substituted for the match on each call. - * - * The result inherits any tainting in the original string or any supplied - * replacement string. - * - * "hello".sub(/[aeiou]/, '*') #=> "h*llo" - * "hello".sub(/([aeiou])/, '<\1>') #=> "h<e>llo" - * "hello".sub(/./) {|s| s[0].to_s + ' ' } #=> "104 ello" - */ - -static VALUE -rb_str_sub(argc, argv, str) - int argc; - VALUE *argv; - VALUE str; + * sub(pattern, replacement) -> new_string + * sub(pattern) {|match| ... } -> new_string + * + * :include: doc/string/sub.rdoc + */ + +static VALUE +rb_str_sub(int argc, VALUE *argv, VALUE str) { - str = rb_str_dup(str); + str = str_duplicate(rb_cString, str); rb_str_sub_bang(argc, argv, str); return str; } static VALUE -str_gsub(argc, argv, str, bang) - int argc; - VALUE *argv; - VALUE str; - int bang; -{ - VALUE pat, val, repl, match; - struct re_registers *regs; - long beg, n; - long offset, blen, len; - int iter = 0; - char *buf, *bp, *cp; - int tainted = 0; - - if (argc == 1 && rb_block_given_p()) { - iter = 1; - } - else if (argc == 2) { - repl = argv[1]; - StringValue(repl); - if (OBJ_TAINTED(repl)) tainted = 1; - } - else { - rb_raise(rb_eArgError, "wrong number of arguments (%d for 2)", argc); +str_gsub(int argc, VALUE *argv, VALUE str, int bang) +{ + VALUE pat, val = Qnil, repl, match0 = Qnil, dest, hash = Qnil, match = Qnil; + long beg, beg0, end0; + long offset, blen, slen, len, last; + enum {STR, ITER, FAST_MAP, MAP} mode = STR; + char *sp, *cp; + int need_backref_str = -1; + rb_encoding *str_enc; + + switch (argc) { + case 1: + RETURN_ENUMERATOR(str, argc, argv); + mode = ITER; + break; + case 2: + repl = argv[1]; + hash = rb_check_hash_type(argv[1]); + if (NIL_P(hash)) { + StringValue(repl); + } + else if (rb_hash_default_unredefined(hash) && !FL_TEST_RAW(hash, RHASH_PROC_DEFAULT)) { + mode = FAST_MAP; + } + else { + mode = MAP; + } + break; + default: + rb_error_arity(argc, 1, 2); } - pat = get_pat(argv[0], 1); - offset=0; n=0; - beg = rb_reg_search(pat, str, 0, 0); + pat = get_pat_quoted(argv[0], 1); + beg = rb_pat_search0(pat, str, 0, need_backref_str, &match); + if (beg < 0) { - if (bang) return Qnil; /* no match, no substitution */ - return rb_str_dup(str); - } - - blen = RSTRING(str)->len + 30; /* len + margin */ - buf = ALLOC_N(char, blen); - bp = buf; - cp = RSTRING(str)->ptr; - - while (beg >= 0) { - n++; - match = rb_backref_get(); - regs = RMATCH(match)->regs; - if (iter) { - rb_match_busy(match); - val = rb_obj_as_string(rb_yield(rb_reg_nth_match(0, match))); - rb_backref_set(match); - } - else { - val = rb_reg_regsub(repl, str, regs); - } - if (OBJ_TAINTED(val)) tainted = 1; - len = (bp - buf) + (beg - offset) + RSTRING(val)->len + 3; - if (blen < len) { - while (blen < len) blen *= 2; - len = bp - buf; - REALLOC_N(buf, char, blen); - bp = buf + len; - } - len = beg - offset; /* copy pre-match substr */ - memcpy(bp, cp, len); - bp += len; - memcpy(bp, RSTRING(val)->ptr, RSTRING(val)->len); - bp += RSTRING(val)->len; - offset = END(0); - if (BEG(0) == END(0)) { - /* - * Always consume at least one character of the input string - * in order to prevent infinite loops. - */ - if (RSTRING(str)->len <= END(0)) break; - len = mbclen2(RSTRING(str)->ptr[END(0)], pat); - memcpy(bp, RSTRING(str)->ptr+END(0), len); - bp += len; - offset = END(0) + len; - } - cp = RSTRING(str)->ptr + offset; - if (offset > RSTRING(str)->len) break; - beg = rb_reg_search(pat, str, offset, 0); - } - if (RSTRING(str)->len > offset) { - len = bp - buf; - if (blen - len < RSTRING(str)->len - offset + 1) { - REALLOC_N(buf, char, len + RSTRING(str)->len - offset + 1); - bp = buf + len; - } - memcpy(bp, cp, RSTRING(str)->len - offset); - bp += RSTRING(str)->len - offset; - } - rb_backref_set(match); + if (bang) return Qnil; /* no match, no substitution */ + return str_duplicate(rb_cString, str); + } + + offset = 0; + blen = RSTRING_LEN(str) + 30; /* len + margin */ + dest = rb_str_buf_new(blen); + sp = RSTRING_PTR(str); + slen = RSTRING_LEN(str); + cp = sp; + str_enc = STR_ENC_GET(str); + rb_enc_associate(dest, str_enc); + ENC_CODERANGE_SET(dest, rb_enc_asciicompat(str_enc) ? ENC_CODERANGE_7BIT : ENC_CODERANGE_VALID); + + do { + struct re_registers *regs = RMATCH_REGS(match); + if (RB_TYPE_P(pat, T_STRING)) { + beg0 = beg; + end0 = beg0 + RSTRING_LEN(pat); + match0 = pat; + } + else { + beg0 = BEG(0); + end0 = END(0); + if (mode == ITER) match0 = rb_reg_nth_match(0, match); + } + + if (mode != STR) { + if (mode == ITER) { + val = rb_obj_as_string(rb_yield(match0)); + } + else { + struct RString fake_str = {RBASIC_INIT}; + VALUE key; + if (mode == FAST_MAP) { + // It is safe to use a fake_str here because we established that it won't escape, + // as it's only used for `rb_hash_aref` and we checked the hash doesn't have a + // default proc. + key = setup_fake_str(&fake_str, sp + beg0, end0 - beg0, ENCODING_GET_INLINED(str)); + } + else { + key = rb_str_subseq(str, beg0, end0 - beg0); + } + val = rb_hash_aref(hash, key); + val = rb_obj_as_string(val); + } + str_mod_check(str, sp, slen); + if (val == dest) { /* paranoid check [ruby-dev:24827] */ + rb_raise(rb_eRuntimeError, "block should not cheat"); + } + } + else if (need_backref_str) { + val = rb_reg_regsub(repl, str, regs, RB_TYPE_P(pat, T_STRING) ? Qnil : pat); + if (need_backref_str < 0) { + need_backref_str = val != repl; + } + } + else { + val = repl; + } + + len = beg0 - offset; /* copy pre-match substr */ + if (len) { + rb_enc_str_buf_cat(dest, cp, len, str_enc); + } + + rb_str_buf_append(dest, val); + + last = offset; + offset = end0; + if (beg0 == end0) { + /* + * Always consume at least one character of the input string + * in order to prevent infinite loops. + */ + if (RSTRING_LEN(str) <= end0) break; + len = rb_enc_fast_mbclen(RSTRING_PTR(str)+end0, RSTRING_END(str), str_enc); + rb_enc_str_buf_cat(dest, RSTRING_PTR(str)+end0, len, str_enc); + offset = end0 + len; + } + cp = RSTRING_PTR(str) + offset; + if (offset > RSTRING_LEN(str)) break; + + // In FAST_MAP and STR mode the backref can't escape so we can re-use the MatchData safely. + if (mode != FAST_MAP && mode != STR) { + match = Qnil; + } + beg = rb_pat_search0(pat, str, offset, need_backref_str, &match); + + RB_GC_GUARD(match); + } while (beg >= 0); + + if (RSTRING_LEN(str) > offset) { + rb_enc_str_buf_cat(dest, cp, RSTRING_LEN(str) - offset, str_enc); + } + rb_pat_search0(pat, str, last, 1, &match); if (bang) { - if (str_independent(str)) { - free(RSTRING(str)->ptr); - } - FL_UNSET(str, ELTS_SHARED|STR_ASSOC); + str_shared_replace(str, dest); } else { - VALUE dup = str_alloc(rb_obj_class(str)); - - OBJ_INFECT(dup, str); - str = dup; + str = dest; } - RSTRING(str)->ptr = buf; - RSTRING(str)->len = len = bp - buf; - RSTRING(str)->ptr[len] = '\0'; - RSTRING(str)->aux.capa = len; - if (tainted) OBJ_TAINT(str); return str; } /* * call-seq: - * str.gsub!(pattern, replacement) => str or nil - * str.gsub!(pattern) {|match| block } => str or nil - * - * Performs the substitutions of <code>String#gsub</code> in place, returning - * <i>str</i>, or <code>nil</code> if no substitutions were performed. + * gsub!(pattern, replacement) -> self or nil + * gsub!(pattern) {|match| ... } -> self or nil + * gsub!(pattern) -> an_enumerator + * + * Like String#gsub, except that: + * + * - Performs substitutions in +self+ (not in a copy of +self+). + * - Returns +self+ if any characters are removed, +nil+ otherwise. + * + * Related: see {Modifying}[rdoc-ref:String@Modifying]. */ static VALUE -rb_str_gsub_bang(argc, argv, str) - int argc; - VALUE *argv; - VALUE str; +rb_str_gsub_bang(int argc, VALUE *argv, VALUE str) { + str_modify_keep_cr(str); return str_gsub(argc, argv, str, 1); } /* * call-seq: - * str.gsub(pattern, replacement) => new_str - * str.gsub(pattern) {|match| block } => new_str - * - * Returns a copy of <i>str</i> with <em>all</em> occurrences of <i>pattern</i> - * replaced with either <i>replacement</i> or the value of the block. The - * <i>pattern</i> will typically be a <code>Regexp</code>; if it is a - * <code>String</code> then no regular expression metacharacters will be - * interpreted (that is <code>/\d/</code> will match a digit, but - * <code>'\d'</code> will match a backslash followed by a 'd'). - * - * If a string is used as the replacement, special variables from the match - * (such as <code>$&</code> and <code>$1</code>) cannot be substituted into it, - * as substitution into the string occurs before the pattern match - * starts. However, the sequences <code>\1</code>, <code>\2</code>, and so on - * may be used to interpolate successive groups in the match. - * - * In the block form, the current match string is passed in as a parameter, and - * variables such as <code>$1</code>, <code>$2</code>, <code>$`</code>, - * <code>$&</code>, and <code>$'</code> will be set appropriately. The value - * returned by the block will be substituted for the match on each call. - * - * The result inherits any tainting in the original string or any supplied - * replacement string. - * - * "hello".gsub(/[aeiou]/, '*') #=> "h*ll*" - * "hello".gsub(/([aeiou])/, '<\1>') #=> "h<e>ll<o>" - * "hello".gsub(/./) {|s| s[0].to_s + ' '} #=> "104 101 108 108 111 " - */ - -static VALUE -rb_str_gsub(argc, argv, str) - int argc; - VALUE *argv; - VALUE str; + * gsub(pattern, replacement) -> new_string + * gsub(pattern) {|match| ... } -> new_string + * gsub(pattern) -> enumerator + * + * Returns a copy of +self+ with zero or more substrings replaced. + * + * Argument +pattern+ may be a string or a Regexp; + * argument +replacement+ may be a string or a Hash. + * Varying types for the argument values makes this method very versatile. + * + * Below are some simple examples; + * for many more examples, see {Substitution Methods}[rdoc-ref:String@Substitution+Methods]. + * + * With arguments +pattern+ and string +replacement+ given, + * replaces each matching substring with the given +replacement+ string: + * + * s = 'abracadabra' + * s.gsub('ab', 'AB') # => "ABracadABra" + * s.gsub(/[a-c]/, 'X') # => "XXrXXXdXXrX" + * + * With arguments +pattern+ and hash +replacement+ given, + * replaces each matching substring with a value from the given +replacement+ hash, + * or removes it: + * + * h = {'a' => 'A', 'b' => 'B', 'c' => 'C'} + * s.gsub(/[a-c]/, h) # => "ABrACAdABrA" # 'a', 'b', 'c' replaced. + * s.gsub(/[a-d]/, h) # => "ABrACAABrA" # 'd' removed. + * + * With argument +pattern+ and a block given, + * calls the block with each matching substring; + * replaces that substring with the block's return value: + * + * s.gsub(/[a-d]/) {|substring| substring.upcase } + * # => "ABrACADABrA" + * + * With argument +pattern+ and no block given, + * returns a new Enumerator. + * + * Related: see {Converting to New String}[rdoc-ref:String@Converting+to+New+String]. + */ + +static VALUE +rb_str_gsub(int argc, VALUE *argv, VALUE str) { return str_gsub(argc, argv, str, 0); } @@ -2182,882 +6548,2031 @@ rb_str_gsub(argc, argv, str) /* * call-seq: - * str.replace(other_str) => str - * - * Replaces the contents and taintedness of <i>str</i> with the corresponding - * values in <i>other_str</i>. - * - * s = "hello" #=> "hello" - * s.replace "world" #=> "world" + * replace(other_string) -> self + * + * Replaces the contents of +self+ with the contents of +other_string+; + * returns +self+: + * + * s = 'foo' # => "foo" + * s.replace('bar') # => "bar" + * + * Related: see {Modifying}[rdoc-ref:String@Modifying]. */ -static VALUE -rb_str_replace(str, str2) - VALUE str, str2; +VALUE +rb_str_replace(VALUE str, VALUE str2) { + str_modifiable(str); if (str == str2) return str; StringValue(str2); - if (FL_TEST(str2, ELTS_SHARED)) { - if (str_independent(str)) { - free(RSTRING(str)->ptr); - } - RSTRING(str)->len = RSTRING(str2)->len; - RSTRING(str)->ptr = RSTRING(str2)->ptr; - FL_SET(str, ELTS_SHARED); - FL_UNSET(str, STR_ASSOC); - RSTRING(str)->aux.shared = RSTRING(str2)->aux.shared; - } - else { - rb_str_modify(str); - rb_str_resize(str, RSTRING(str2)->len); - memcpy(RSTRING(str)->ptr, RSTRING(str2)->ptr, RSTRING(str2)->len); - if (FL_TEST(str2, STR_ASSOC)) { - FL_SET(str, STR_ASSOC); - RSTRING(str)->aux.shared = RSTRING(str2)->aux.shared; - } - } - - OBJ_INFECT(str, str2); - return str; + str_discard(str); + return str_replace(str, str2); } -static VALUE -uscore_get() -{ - VALUE line; +/* + * call-seq: + * clear -> self + * + * Removes the contents of +self+: + * + * s = 'foo' + * s.clear # => "" + * s # => "" + * + * Related: see {Modifying}[rdoc-ref:String@Modifying]. + */ - line = rb_lastline_get(); - if (TYPE(line) != T_STRING) { - rb_raise(rb_eTypeError, "$_ value need to be String (%s given)", - NIL_P(line) ? "nil" : rb_obj_classname(line)); - } - return line; +static VALUE +rb_str_clear(VALUE str) +{ + str_discard(str); + STR_SET_EMBED(str); + STR_SET_LEN(str, 0); + RSTRING_PTR(str)[0] = 0; + if (rb_enc_asciicompat(STR_ENC_GET(str))) + ENC_CODERANGE_SET(str, ENC_CODERANGE_7BIT); + else + ENC_CODERANGE_SET(str, ENC_CODERANGE_VALID); + return str; } /* * call-seq: - * sub!(pattern, replacement) => $_ or nil - * sub!(pattern) {|...| block } => $_ or nil - * - * Equivalent to <code>$_.sub!(<i>args</i>)</code>. + * chr -> string + * + * :include: doc/string/chr.rdoc + * */ static VALUE -rb_f_sub_bang(argc, argv) - int argc; - VALUE *argv; +rb_str_chr(VALUE str) { - return rb_str_sub_bang(argc, argv, uscore_get()); + return rb_str_substr(str, 0, 1); } /* * call-seq: - * sub(pattern, replacement) => $_ - * sub(pattern) { block } => $_ - * - * Equivalent to <code>$_.sub(<i>args</i>)</code>, except that - * <code>$_</code> will be updated if substitution occurs. + * getbyte(index) -> integer or nil + * + * :include: doc/string/getbyte.rdoc + * */ - -static VALUE -rb_f_sub(argc, argv) - int argc; - VALUE *argv; +VALUE +rb_str_getbyte(VALUE str, VALUE index) { - VALUE str = rb_str_dup(uscore_get()); + long pos = NUM2LONG(index); - if (NIL_P(rb_str_sub_bang(argc, argv, str))) - return str; - rb_lastline_set(str); - return str; + if (pos < 0) + pos += RSTRING_LEN(str); + if (pos < 0 || RSTRING_LEN(str) <= pos) + return Qnil; + + return INT2FIX((unsigned char)RSTRING_PTR(str)[pos]); } /* * call-seq: - * gsub!(pattern, replacement) => string or nil - * gsub!(pattern) {|...| block } => string or nil - * - * Equivalent to <code>Kernel::gsub</code>, except <code>nil</code> is - * returned if <code>$_</code> is not modified. - * - * $_ = "quick brown fox" - * gsub! /cat/, '*' #=> nil - * $_ #=> "quick brown fox" + * setbyte(index, integer) -> integer + * + * Sets the byte at zero-based offset +index+ to the value of the given +integer+; + * returns +integer+: + * + * s = 'xyzzy' + * s.setbyte(2, 129) # => 129 + * s # => "xy\x81zy" + * + * Related: see {Modifying}[rdoc-ref:String@Modifying]. */ +VALUE +rb_str_setbyte(VALUE str, VALUE index, VALUE value) +{ + long pos = NUM2LONG(index); + long len = RSTRING_LEN(str); + char *ptr, *head, *left = 0; + rb_encoding *enc; + int cr = ENC_CODERANGE_UNKNOWN, width, nlen; + + if (pos < -len || len <= pos) + rb_raise(rb_eIndexError, "index %ld out of string", pos); + if (pos < 0) + pos += len; + + VALUE v = rb_to_int(value); + VALUE w = rb_int_and(v, INT2FIX(0xff)); + char byte = (char)(NUM2INT(w) & 0xFF); + + if (!str_independent(str)) + str_make_independent(str); + enc = STR_ENC_GET(str); + head = RSTRING_PTR(str); + ptr = &head[pos]; + if (!STR_EMBED_P(str)) { + cr = ENC_CODERANGE(str); + switch (cr) { + case ENC_CODERANGE_7BIT: + left = ptr; + *ptr = byte; + if (ISASCII(byte)) goto end; + nlen = rb_enc_precise_mbclen(left, head+len, enc); + if (!MBCLEN_CHARFOUND_P(nlen)) + ENC_CODERANGE_SET(str, ENC_CODERANGE_BROKEN); + else + ENC_CODERANGE_SET(str, ENC_CODERANGE_VALID); + goto end; + case ENC_CODERANGE_VALID: + left = rb_enc_left_char_head(head, ptr, head+len, enc); + width = rb_enc_precise_mbclen(left, head+len, enc); + *ptr = byte; + nlen = rb_enc_precise_mbclen(left, head+len, enc); + if (!MBCLEN_CHARFOUND_P(nlen)) + ENC_CODERANGE_SET(str, ENC_CODERANGE_BROKEN); + else if (MBCLEN_CHARFOUND_LEN(nlen) != width || ISASCII(byte)) + ENC_CODERANGE_CLEAR(str); + goto end; + } + } + ENC_CODERANGE_CLEAR(str); + *ptr = byte; + + end: + return value; +} static VALUE -rb_f_gsub_bang(argc, argv) - int argc; - VALUE *argv; +str_byte_substr(VALUE str, long beg, long len, int empty) { - return rb_str_gsub_bang(argc, argv, uscore_get()); + long n = RSTRING_LEN(str); + + if (beg > n || len < 0) return Qnil; + if (beg < 0) { + beg += n; + if (beg < 0) return Qnil; + } + if (len > n - beg) + len = n - beg; + if (len <= 0) { + if (!empty) return Qnil; + len = 0; + } + + VALUE str2 = str_subseq(str, beg, len); + + str_enc_copy_direct(str2, str); + + if (RSTRING_LEN(str2) == 0) { + if (!rb_enc_asciicompat(STR_ENC_GET(str))) + ENC_CODERANGE_SET(str2, ENC_CODERANGE_VALID); + else + ENC_CODERANGE_SET(str2, ENC_CODERANGE_7BIT); + } + else { + switch (ENC_CODERANGE(str)) { + case ENC_CODERANGE_7BIT: + ENC_CODERANGE_SET(str2, ENC_CODERANGE_7BIT); + break; + default: + ENC_CODERANGE_SET(str2, ENC_CODERANGE_UNKNOWN); + break; + } + } + + return str2; +} + +VALUE +rb_str_byte_substr(VALUE str, VALUE beg, VALUE len) +{ + return str_byte_substr(str, NUM2LONG(beg), NUM2LONG(len), TRUE); +} + +static VALUE +str_byte_aref(VALUE str, VALUE indx) +{ + long idx; + if (FIXNUM_P(indx)) { + idx = FIX2LONG(indx); + } + else { + /* check if indx is Range */ + long beg, len = RSTRING_LEN(str); + + switch (rb_range_beg_len(indx, &beg, &len, len, 0)) { + case Qfalse: + break; + case Qnil: + return Qnil; + default: + return str_byte_substr(str, beg, len, TRUE); + } + + idx = NUM2LONG(indx); + } + return str_byte_substr(str, idx, 1, FALSE); } /* * call-seq: - * gsub(pattern, replacement) => string - * gsub(pattern) {|...| block } => string - * - * Equivalent to <code>$_.gsub...</code>, except that <code>$_</code> - * receives the modified result. - * - * $_ = "quick brown fox" - * gsub /[aeiou]/, '*' #=> "q**ck br*wn f*x" - * $_ #=> "q**ck br*wn f*x" + * byteslice(offset, length = 1) -> string or nil + * byteslice(range) -> string or nil + * + * :include: doc/string/byteslice.rdoc */ static VALUE -rb_f_gsub(argc, argv) - int argc; - VALUE *argv; +rb_str_byteslice(int argc, VALUE *argv, VALUE str) { - VALUE str = rb_str_dup(uscore_get()); - - if (NIL_P(rb_str_gsub_bang(argc, argv, str))) - return str; - rb_lastline_set(str); - return str; + if (argc == 2) { + long beg = NUM2LONG(argv[0]); + long len = NUM2LONG(argv[1]); + return str_byte_substr(str, beg, len, TRUE); + } + rb_check_arity(argc, 1, 2); + return str_byte_aref(str, argv[0]); } +static void +str_check_beg_len(VALUE str, long *beg, long *len) +{ + long end, slen = RSTRING_LEN(str); + + if (*len < 0) rb_raise(rb_eIndexError, "negative length %ld", *len); + if ((slen < *beg) || ((*beg < 0) && (*beg + slen < 0))) { + rb_raise(rb_eIndexError, "index %ld out of string", *beg); + } + if (*beg < 0) { + *beg += slen; + } + RUBY_ASSERT(*beg >= 0); + RUBY_ASSERT(*beg <= slen); + + if (*len > slen - *beg) { + *len = slen - *beg; + } + end = *beg + *len; + str_ensure_byte_pos(str, *beg); + str_ensure_byte_pos(str, end); +} /* * call-seq: - * str.reverse! => str - * - * Reverses <i>str</i> in place. + * bytesplice(offset, length, str) -> self + * bytesplice(offset, length, str, str_offset, str_length) -> self + * bytesplice(range, str) -> self + * bytesplice(range, str, str_range) -> self + * + * :include: doc/string/bytesplice.rdoc */ static VALUE -rb_str_reverse_bang(str) - VALUE str; +rb_str_bytesplice(int argc, VALUE *argv, VALUE str) { - char *s, *e; - char c; + long beg, len, vbeg, vlen; + VALUE val; + int cr; + + rb_check_arity(argc, 2, 5); + if (!(argc == 2 || argc == 3 || argc == 5)) { + rb_raise(rb_eArgError, "wrong number of arguments (given %d, expected 2, 3, or 5)", argc); + } + if (argc == 2 || (argc == 3 && !RB_INTEGER_TYPE_P(argv[0]))) { + if (!rb_range_beg_len(argv[0], &beg, &len, RSTRING_LEN(str), 2)) { + rb_raise(rb_eTypeError, "wrong argument type %s (expected Range)", + rb_builtin_class_name(argv[0])); + } + val = argv[1]; + StringValue(val); + if (argc == 2) { + /* bytesplice(range, str) */ + vbeg = 0; + vlen = RSTRING_LEN(val); + } + else { + /* bytesplice(range, str, str_range) */ + if (!rb_range_beg_len(argv[2], &vbeg, &vlen, RSTRING_LEN(val), 2)) { + rb_raise(rb_eTypeError, "wrong argument type %s (expected Range)", + rb_builtin_class_name(argv[2])); + } + } + } + else { + beg = NUM2LONG(argv[0]); + len = NUM2LONG(argv[1]); + val = argv[2]; + StringValue(val); + if (argc == 3) { + /* bytesplice(index, length, str) */ + vbeg = 0; + vlen = RSTRING_LEN(val); + } + else { + /* bytesplice(index, length, str, str_index, str_length) */ + vbeg = NUM2LONG(argv[3]); + vlen = NUM2LONG(argv[4]); + } + } + str_check_beg_len(str, &beg, &len); + str_check_beg_len(val, &vbeg, &vlen); + str_modify_keep_cr(str); - if (RSTRING(str)->len > 1) { - rb_str_modify(str); - s = RSTRING(str)->ptr; - e = s + RSTRING(str)->len - 1; - while (s < e) { - c = *s; - *s++ = *e; - *e-- = c; - } + if (RB_UNLIKELY(ENCODING_GET_INLINED(str) != ENCODING_GET_INLINED(val))) { + rb_enc_associate(str, rb_enc_check(str, val)); } + + rb_str_update_1(str, beg, len, val, vbeg, vlen); + cr = ENC_CODERANGE_AND(ENC_CODERANGE(str), ENC_CODERANGE(val)); + if (cr != ENC_CODERANGE_BROKEN) + ENC_CODERANGE_SET(str, cr); return str; } - /* * call-seq: - * str.reverse => new_str - * - * Returns a new string with the characters from <i>str</i> in reverse order. - * - * "stressed".reverse #=> "desserts" + * reverse -> new_string + * + * Returns a new string with the characters from +self+ in reverse order. + * + * 'drawer'.reverse # => "reward" + * 'reviled'.reverse # => "deliver" + * 'stressed'.reverse # => "desserts" + * 'semordnilaps'.reverse # => "spalindromes" + * + * Related: see {Converting to New String}[rdoc-ref:String@Converting+to+New+String]. */ static VALUE -rb_str_reverse(str) - VALUE str; +rb_str_reverse(VALUE str) { - VALUE obj; + rb_encoding *enc; + VALUE rev; char *s, *e, *p; + int cr; + + if (RSTRING_LEN(str) <= 1) return str_duplicate(rb_cString, str); + enc = STR_ENC_GET(str); + rev = rb_str_new(0, RSTRING_LEN(str)); + s = RSTRING_PTR(str); e = RSTRING_END(str); + p = RSTRING_END(rev); + cr = ENC_CODERANGE(str); + + if (RSTRING_LEN(str) > 1) { + if (single_byte_optimizable(str)) { + while (s < e) { + *--p = *s++; + } + } + else if (cr == ENC_CODERANGE_VALID) { + while (s < e) { + int clen = rb_enc_fast_mbclen(s, e, enc); - if (RSTRING(str)->len <= 1) return rb_str_dup(str); - - obj = rb_str_new5(str, 0, RSTRING(str)->len); - s = RSTRING(str)->ptr; e = s + RSTRING(str)->len - 1; - p = RSTRING(obj)->ptr; - - while (e >= s) { - *p++ = *e--; + p -= clen; + memcpy(p, s, clen); + s += clen; + } + } + else { + cr = rb_enc_asciicompat(enc) ? + ENC_CODERANGE_7BIT : ENC_CODERANGE_VALID; + while (s < e) { + int clen = rb_enc_mbclen(s, e, enc); + + if (clen > 1 || (*s & 0x80)) cr = ENC_CODERANGE_UNKNOWN; + p -= clen; + memcpy(p, s, clen); + s += clen; + } + } } - OBJ_INFECT(obj, str); + STR_SET_LEN(rev, RSTRING_LEN(str)); + str_enc_copy_direct(rev, str); + ENC_CODERANGE_SET(rev, cr); - return obj; + return rev; } /* * call-seq: - * str.include? other_str => true or false - * str.include? fixnum => true or false - * - * Returns <code>true</code> if <i>str</i> contains the given string or - * character. - * - * "hello".include? "lo" #=> true - * "hello".include? "ol" #=> false - * "hello".include? ?h #=> true + * reverse! -> self + * + * Returns +self+ with its characters reversed: + * + * 'drawer'.reverse! # => "reward" + * 'reviled'.reverse! # => "deliver" + * 'stressed'.reverse! # => "desserts" + * 'semordnilaps'.reverse! # => "spalindromes" + * + * Related: see {Modifying}[rdoc-ref:String@Modifying]. */ static VALUE -rb_str_include(str, arg) - VALUE str, arg; -{ - long i; +rb_str_reverse_bang(VALUE str) +{ + if (RSTRING_LEN(str) > 1) { + if (single_byte_optimizable(str)) { + char *s, *e, c; + + str_modify_keep_cr(str); + s = RSTRING_PTR(str); + e = RSTRING_END(str) - 1; + while (s < e) { + c = *s; + *s++ = *e; + *e-- = c; + } + } + else { + str_shared_replace(str, rb_str_reverse(str)); + } + } + else { + str_modify_keep_cr(str); + } + return str; +} - if (FIXNUM_P(arg)) { - int c = FIX2INT(arg); - long len = RSTRING(str)->len; - char *p = RSTRING(str)->ptr; - for (i=0; i<len; i++) { - if (p[i] == c) { - return Qtrue; - } - } - return Qfalse; - } +/* + * call-seq: + * include?(other_string) -> true or false + * + * Returns whether +self+ contains +other_string+: + * + * s = 'bar' + * s.include?('ba') # => true + * s.include?('ar') # => true + * s.include?('bar') # => true + * s.include?('a') # => true + * s.include?('') # => true + * s.include?('foo') # => false + * + * Related: see {Querying}[rdoc-ref:String@Querying]. + */ + +VALUE +rb_str_include(VALUE str, VALUE arg) +{ + long i; StringValue(arg); i = rb_str_index(str, arg, 0); - if (i == -1) return Qfalse; - return Qtrue; + return RBOOL(i != -1); } /* * call-seq: - * str.to_i(base=10) => integer - * - * Returns the result of interpreting leading characters in <i>str</i> as an - * integer base <i>base</i> (2, 8, 10, or 16). Extraneous characters past the - * end of a valid number are ignored. If there is not a valid number at the - * start of <i>str</i>, <code>0</code> is returned. This method never raises an - * exception. - * - * "12345".to_i #=> 12345 - * "99 red balloons".to_i #=> 99 - * "0a".to_i #=> 0 - * "0a".to_i(16) #=> 10 - * "hello".to_i #=> 0 - * "1100101".to_i(2) #=> 101 - * "1100101".to_i(8) #=> 294977 - * "1100101".to_i(10) #=> 1100101 - * "1100101".to_i(16) #=> 17826049 + * to_i(base = 10) -> integer + * + * Returns the result of interpreting leading characters in +self+ + * as an integer in the given +base+; + * +base+ must be either +0+ or in range <tt>(2..36)</tt>: + * + * '123456'.to_i # => 123456 + * '123def'.to_i(16) # => 1195503 + * + * With +base+ zero given, string +object+ may contain leading characters + * to specify the actual base: + * + * '123def'.to_i(0) # => 123 + * '0123def'.to_i(0) # => 83 + * '0b123def'.to_i(0) # => 1 + * '0o123def'.to_i(0) # => 83 + * '0d123def'.to_i(0) # => 123 + * '0x123def'.to_i(0) # => 1195503 + * + * Characters past a leading valid number (in the given +base+) are ignored: + * + * '12.345'.to_i # => 12 + * '12345'.to_i(2) # => 1 + * + * Returns zero if there is no leading valid number: + * + * 'abcdef'.to_i # => 0 + * '2'.to_i(2) # => 0 + * + * Related: see {Converting to Non-String}[rdoc-ref:String@Converting+to+Non--5CString]. */ static VALUE -rb_str_to_i(argc, argv, str) - int argc; - VALUE *argv; - VALUE str; +rb_str_to_i(int argc, VALUE *argv, VALUE str) { - VALUE b; - int base; + int base = 10; - rb_scan_args(argc, argv, "01", &b); - if (argc == 0) base = 10; - else base = NUM2INT(b); - - if (base < 0) { - rb_raise(rb_eArgError, "illegal radix %d", base); + if (rb_check_arity(argc, 0, 1) && (base = NUM2INT(argv[0])) < 0) { + rb_raise(rb_eArgError, "invalid radix %d", base); } - return rb_str_to_inum(str, base, Qfalse); + return rb_str_to_inum(str, base, FALSE); } /* * call-seq: - * str.to_f => float - * - * Returns the result of interpreting leading characters in <i>str</i> as a - * floating point number. Extraneous characters past the end of a valid number - * are ignored. If there is not a valid number at the start of <i>str</i>, - * <code>0.0</code> is returned. This method never raises an exception. - * - * "123.45e1".to_f #=> 1234.5 - * "45.67 degrees".to_f #=> 45.67 - * "thx1138".to_f #=> 0.0 + * to_f -> float + * + * Returns the result of interpreting leading characters in +self+ as a Float: + * + * '3.14159'.to_f # => 3.14159 + * '1.234e-2'.to_f # => 0.01234 + * + * Characters past a leading valid number are ignored: + * + * '3.14 (pi to two places)'.to_f # => 3.14 + * + * Returns zero if there is no leading valid number: + * + * 'abcdef'.to_f # => 0.0 + * + * See {Converting to Non-String}[rdoc-ref:String@Converting+to+Non--5CString]. */ static VALUE -rb_str_to_f(str) - VALUE str; +rb_str_to_f(VALUE str) { - return rb_float_new(rb_str_to_dbl(str, Qfalse)); + return DBL2NUM(rb_str_to_dbl(str, FALSE)); } /* * call-seq: - * str.to_s => str - * str.to_str => str - * - * Returns the receiver. + * to_s -> self or new_string + * + * Returns +self+ if +self+ is a +String+, + * or +self+ converted to a +String+ if +self+ is a subclass of +String+. + * + * Related: see {Converting to New String}[rdoc-ref:String@Converting+to+New+String]. */ static VALUE -rb_str_to_s(str) - VALUE str; +rb_str_to_s(VALUE str) { if (rb_obj_class(str) != rb_cString) { - VALUE dup = str_alloc(rb_cString); - rb_str_replace(dup, str); - return dup; + return str_duplicate(rb_cString, str); } return str; } +#if 0 +static void +str_cat_char(VALUE str, unsigned int c, rb_encoding *enc) +{ + char s[RUBY_MAX_CHAR_LEN]; + int n = rb_enc_codelen(c, enc); + + rb_enc_mbcput(c, s, enc); + rb_enc_str_buf_cat(str, s, n, enc); +} +#endif + +#define CHAR_ESC_LEN 13 /* sizeof(\x{ hex of 32bit unsigned int } \0) */ + +int +rb_str_buf_cat_escaped_char(VALUE result, unsigned int c, int unicode_p) +{ + char buf[CHAR_ESC_LEN + 1]; + int l; + +#if SIZEOF_INT > 4 + c &= 0xffffffff; +#endif + if (unicode_p) { + if (c < 0x7F && ISPRINT(c)) { + snprintf(buf, CHAR_ESC_LEN, "%c", c); + } + else if (c < 0x10000) { + snprintf(buf, CHAR_ESC_LEN, "\\u%04X", c); + } + else { + snprintf(buf, CHAR_ESC_LEN, "\\u{%X}", c); + } + } + else { + if (c < 0x100) { + snprintf(buf, CHAR_ESC_LEN, "\\x%02X", c); + } + else { + snprintf(buf, CHAR_ESC_LEN, "\\x{%X}", c); + } + } + l = (int)strlen(buf); /* CHAR_ESC_LEN cannot exceed INT_MAX */ + rb_str_buf_cat(result, buf, l); + return l; +} + +const char * +ruby_escaped_char(int c) +{ + switch (c) { + case '\0': return "\\0"; + case '\n': return "\\n"; + case '\r': return "\\r"; + case '\t': return "\\t"; + case '\f': return "\\f"; + case '\013': return "\\v"; + case '\010': return "\\b"; + case '\007': return "\\a"; + case '\033': return "\\e"; + case '\x7f': return "\\c?"; + } + return NULL; +} + VALUE +rb_str_escape(VALUE str) +{ + int encidx = ENCODING_GET(str); + rb_encoding *enc = rb_enc_from_index(encidx); + const char *p = RSTRING_PTR(str); + const char *pend = RSTRING_END(str); + const char *prev = p; + char buf[CHAR_ESC_LEN + 1]; + VALUE result = rb_str_buf_new(0); + int unicode_p = rb_enc_unicode_p(enc); + int asciicompat = rb_enc_asciicompat(enc); + + while (p < pend) { + unsigned int c; + const char *cc; + int n = rb_enc_precise_mbclen(p, pend, enc); + if (!MBCLEN_CHARFOUND_P(n)) { + if (p > prev) str_buf_cat(result, prev, p - prev); + n = rb_enc_mbminlen(enc); + if (pend < p + n) + n = (int)(pend - p); + while (n--) { + snprintf(buf, CHAR_ESC_LEN, "\\x%02X", *p & 0377); + str_buf_cat(result, buf, strlen(buf)); + prev = ++p; + } + continue; + } + n = MBCLEN_CHARFOUND_LEN(n); + c = rb_enc_mbc_to_codepoint(p, pend, enc); + p += n; + cc = ruby_escaped_char(c); + if (cc) { + if (p - n > prev) str_buf_cat(result, prev, p - n - prev); + str_buf_cat(result, cc, strlen(cc)); + prev = p; + } + else if (asciicompat && rb_enc_isascii(c, enc) && ISPRINT(c)) { + } + else { + if (p - n > prev) str_buf_cat(result, prev, p - n - prev); + rb_str_buf_cat_escaped_char(result, c, unicode_p); + prev = p; + } + } + if (p > prev) str_buf_cat(result, prev, p - prev); + ENCODING_CODERANGE_SET(result, rb_usascii_encindex(), ENC_CODERANGE_7BIT); + + return result; +} + /* - * call-seq: - * str.inspect => string + * call-seq: + * inspect -> string * - * Returns a printable version of _str_, with special characters - * escaped. + * :include: doc/string/inspect.rdoc * - * str = "hello" - * str[3] = 8 - * str.inspect #=> "hel\010o" */ -rb_str_inspect(str) - VALUE str; -{ - char *p, *pend; - VALUE result = rb_str_buf_new2("\""); - char s[5]; - - p = RSTRING(str)->ptr; pend = p + RSTRING(str)->len; +VALUE +rb_str_inspect(VALUE str) +{ + int encidx = ENCODING_GET(str); + rb_encoding *enc = rb_enc_from_index(encidx); + const char *p, *pend, *prev; + char buf[CHAR_ESC_LEN + 1]; + VALUE result = rb_str_buf_new(0); + rb_encoding *resenc = rb_default_internal_encoding(); + int unicode_p = rb_enc_unicode_p(enc); + int asciicompat = rb_enc_asciicompat(enc); + + if (resenc == NULL) resenc = rb_default_external_encoding(); + if (!rb_enc_asciicompat(resenc)) resenc = rb_usascii_encoding(); + rb_enc_associate(result, resenc); + str_buf_cat2(result, "\""); + + p = RSTRING_PTR(str); pend = RSTRING_END(str); + prev = p; while (p < pend) { - char c = *p++; - if (ismbchar(c) && p < pend) { - int len = mbclen(c); - rb_str_buf_cat(result, p - 1, len); - p += len - 1; - } - else if (c == '"'|| c == '\\') { - s[0] = '\\'; s[1] = c; - rb_str_buf_cat(result, s, 2); - } - else if (ISPRINT(c)) { - s[0] = c; - rb_str_buf_cat(result, s, 1); - } - else if (c == '\n') { - s[0] = '\\'; s[1] = 'n'; - rb_str_buf_cat(result, s, 2); - } - else if (c == '\r') { - s[0] = '\\'; s[1] = 'r'; - rb_str_buf_cat(result, s, 2); - } - else if (c == '\t') { - s[0] = '\\'; s[1] = 't'; - rb_str_buf_cat(result, s, 2); - } - else if (c == '\f') { - s[0] = '\\'; s[1] = 'f'; - rb_str_buf_cat(result, s, 2); - } - else if (c == '\013') { - s[0] = '\\'; s[1] = 'v'; - rb_str_buf_cat(result, s, 2); - } - else if (c == '\007') { - s[0] = '\\'; s[1] = 'a'; - rb_str_buf_cat(result, s, 2); - } - else if (c == 033) { - s[0] = '\\'; s[1] = 'e'; - rb_str_buf_cat(result, s, 2); - } - else { - sprintf(s, "\\%03o", c & 0377); - rb_str_buf_cat2(result, s); - } - } - rb_str_buf_cat2(result, "\""); - - OBJ_INFECT(result, str); + unsigned int c, cc; + int n; + + n = rb_enc_precise_mbclen(p, pend, enc); + if (!MBCLEN_CHARFOUND_P(n)) { + if (p > prev) str_buf_cat(result, prev, p - prev); + n = rb_enc_mbminlen(enc); + if (pend < p + n) + n = (int)(pend - p); + while (n--) { + snprintf(buf, CHAR_ESC_LEN, "\\x%02X", *p & 0377); + str_buf_cat(result, buf, strlen(buf)); + prev = ++p; + } + continue; + } + n = MBCLEN_CHARFOUND_LEN(n); + c = rb_enc_mbc_to_codepoint(p, pend, enc); + p += n; + if ((asciicompat || unicode_p) && + (c == '"'|| c == '\\' || + (c == '#' && + p < pend && + MBCLEN_CHARFOUND_P(rb_enc_precise_mbclen(p,pend,enc)) && + (cc = rb_enc_codepoint(p,pend,enc), + (cc == '$' || cc == '@' || cc == '{'))))) { + if (p - n > prev) str_buf_cat(result, prev, p - n - prev); + str_buf_cat2(result, "\\"); + if (asciicompat || enc == resenc) { + prev = p - n; + continue; + } + } + switch (c) { + case '\n': cc = 'n'; break; + case '\r': cc = 'r'; break; + case '\t': cc = 't'; break; + case '\f': cc = 'f'; break; + case '\013': cc = 'v'; break; + case '\010': cc = 'b'; break; + case '\007': cc = 'a'; break; + case 033: cc = 'e'; break; + default: cc = 0; break; + } + if (cc) { + if (p - n > prev) str_buf_cat(result, prev, p - n - prev); + buf[0] = '\\'; + buf[1] = (char)cc; + str_buf_cat(result, buf, 2); + prev = p; + continue; + } + /* The special casing of 0x85 (NEXT_LINE) here is because + * Oniguruma historically treats it as printable, but it + * doesn't match the print POSIX bracket class or character + * property in regexps. + * + * See Ruby Bug #16842 for details: + * https://bugs.ruby-lang.org/issues/16842 + */ + if ((enc == resenc && rb_enc_isprint(c, enc) && c != 0x85) || + (asciicompat && rb_enc_isascii(c, enc) && ISPRINT(c))) { + continue; + } + else { + if (p - n > prev) str_buf_cat(result, prev, p - n - prev); + rb_str_buf_cat_escaped_char(result, c, unicode_p); + prev = p; + continue; + } + } + if (p > prev) str_buf_cat(result, prev, p - prev); + str_buf_cat2(result, "\""); + return result; } +#define IS_EVSTR(p,e) ((p) < (e) && (*(p) == '$' || *(p) == '@' || *(p) == '{')) /* * call-seq: - * str.dump => new_str - * - * Produces a version of <i>str</i> with all nonprinting characters replaced by - * <code>\nnn</code> notation and all special characters escaped. + * dump -> new_string + * + * :include: doc/string/dump.rdoc + * */ VALUE -rb_str_dump(str) - VALUE str; +rb_str_dump(VALUE str) { + int encidx = rb_enc_get_index(str); + rb_encoding *enc = rb_enc_from_index(encidx); long len; - char *p, *pend; + const char *p, *pend; char *q, *qend; VALUE result; + int u8 = (encidx == rb_utf8_encindex()); + static const char nonascii_suffix[] = ".dup.force_encoding(\"%s\")"; len = 2; /* "" */ - p = RSTRING(str)->ptr; pend = p + RSTRING(str)->len; + if (!rb_enc_asciicompat(enc)) { + len += strlen(nonascii_suffix) - rb_strlen_lit("%s"); + len += strlen(enc->name); + } + + p = RSTRING_PTR(str); pend = p + RSTRING_LEN(str); while (p < pend) { - char c = *p++; - switch (c) { - case '"': case '\\': - case '\n': case '\r': - case '\t': case '\f': case '#': - case '\013': case '\007': case '\033': - len += 2; - break; - - default: - if (ISPRINT(c)) { - len++; - } - else { - len += 4; /* \nnn */ - } - break; - } - } - - result = rb_str_new5(str, 0, len); - p = RSTRING(str)->ptr; pend = p + RSTRING(str)->len; - q = RSTRING(result)->ptr; qend = q + len; + int clen; + unsigned char c = *p++; + + switch (c) { + case '"': case '\\': + case '\n': case '\r': + case '\t': case '\f': + case '\013': case '\010': case '\007': case '\033': + clen = 2; + break; + + case '#': + clen = IS_EVSTR(p, pend) ? 2 : 1; + break; + + default: + if (ISPRINT(c)) { + clen = 1; + } + else { + if (u8 && c > 0x7F) { /* \u notation */ + int n = rb_enc_precise_mbclen(p-1, pend, enc); + if (MBCLEN_CHARFOUND_P(n)) { + unsigned int cc = rb_enc_mbc_to_codepoint(p-1, pend, enc); + if (cc <= 0xFFFF) + clen = 6; /* \uXXXX */ + else if (cc <= 0xFFFFF) + clen = 9; /* \u{XXXXX} */ + else + clen = 10; /* \u{XXXXXX} */ + p += MBCLEN_CHARFOUND_LEN(n)-1; + break; + } + } + clen = 4; /* \xNN */ + } + break; + } + + if (clen > LONG_MAX - len) { + rb_raise(rb_eRuntimeError, "string size too big"); + } + len += clen; + } + + result = rb_str_new(0, len); + p = RSTRING_PTR(str); pend = p + RSTRING_LEN(str); + q = RSTRING_PTR(result); qend = q + len + 1; *q++ = '"'; while (p < pend) { - char c = *p++; - - if (c == '"' || c == '\\') { - *q++ = '\\'; - *q++ = c; - } - else if (c == '#') { - *q++ = '\\'; - *q++ = '#'; - } - else if (ISPRINT(c)) { - *q++ = c; - } - else if (c == '\n') { - *q++ = '\\'; - *q++ = 'n'; - } - else if (c == '\r') { - *q++ = '\\'; - *q++ = 'r'; - } - else if (c == '\t') { - *q++ = '\\'; - *q++ = 't'; - } - else if (c == '\f') { - *q++ = '\\'; - *q++ = 'f'; - } - else if (c == '\013') { - *q++ = '\\'; - *q++ = 'v'; - } - else if (c == '\007') { - *q++ = '\\'; - *q++ = 'a'; - } - else if (c == '\033') { - *q++ = '\\'; - *q++ = 'e'; - } - else { - *q++ = '\\'; - sprintf(q, "%03o", c&0xff); - q += 3; - } + unsigned char c = *p++; + + if (c == '"' || c == '\\') { + *q++ = '\\'; + *q++ = c; + } + else if (c == '#') { + if (IS_EVSTR(p, pend)) *q++ = '\\'; + *q++ = '#'; + } + else if (c == '\n') { + *q++ = '\\'; + *q++ = 'n'; + } + else if (c == '\r') { + *q++ = '\\'; + *q++ = 'r'; + } + else if (c == '\t') { + *q++ = '\\'; + *q++ = 't'; + } + else if (c == '\f') { + *q++ = '\\'; + *q++ = 'f'; + } + else if (c == '\013') { + *q++ = '\\'; + *q++ = 'v'; + } + else if (c == '\010') { + *q++ = '\\'; + *q++ = 'b'; + } + else if (c == '\007') { + *q++ = '\\'; + *q++ = 'a'; + } + else if (c == '\033') { + *q++ = '\\'; + *q++ = 'e'; + } + else if (ISPRINT(c)) { + *q++ = c; + } + else { + *q++ = '\\'; + if (u8) { + int n = rb_enc_precise_mbclen(p-1, pend, enc) - 1; + if (MBCLEN_CHARFOUND_P(n)) { + int cc = rb_enc_mbc_to_codepoint(p-1, pend, enc); + p += n; + if (cc <= 0xFFFF) + snprintf(q, qend-q, "u%04X", cc); /* \uXXXX */ + else + snprintf(q, qend-q, "u{%X}", cc); /* \u{XXXXX} or \u{XXXXXX} */ + q += strlen(q); + continue; + } + } + snprintf(q, qend-q, "x%02X", c); + q += 3; + } } *q++ = '"'; - - OBJ_INFECT(result, str); + *q = '\0'; + if (!rb_enc_asciicompat(enc)) { + snprintf(q, qend-q, nonascii_suffix, enc->name); + encidx = rb_ascii8bit_encindex(); + } + /* result from dump is ASCII */ + rb_enc_associate_index(result, encidx); + ENC_CODERANGE_SET(result, ENC_CODERANGE_7BIT); return result; } +static int +unescape_ascii(unsigned int c) +{ + switch (c) { + case 'n': + return '\n'; + case 'r': + return '\r'; + case 't': + return '\t'; + case 'f': + return '\f'; + case 'v': + return '\13'; + case 'b': + return '\010'; + case 'a': + return '\007'; + case 'e': + return 033; + } + UNREACHABLE_RETURN(-1); +} + +static void +undump_after_backslash(VALUE undumped, const char **ss, const char *s_end, rb_encoding **penc, bool *utf8, bool *binary) +{ + const char *s = *ss; + unsigned int c; + int codelen; + size_t hexlen; + unsigned char buf[6]; + static rb_encoding *enc_utf8 = NULL; + + switch (*s) { + case '\\': + case '"': + case '#': + rb_str_cat(undumped, s, 1); /* cat itself */ + s++; + break; + case 'n': + case 'r': + case 't': + case 'f': + case 'v': + case 'b': + case 'a': + case 'e': + *buf = unescape_ascii(*s); + rb_str_cat(undumped, (char *)buf, 1); + s++; + break; + case 'u': + if (*binary) { + rb_raise(rb_eRuntimeError, "hex escape and Unicode escape are mixed"); + } + *utf8 = true; + if (++s >= s_end) { + rb_raise(rb_eRuntimeError, "invalid Unicode escape"); + } + if (enc_utf8 == NULL) enc_utf8 = rb_utf8_encoding(); + if (*penc != enc_utf8) { + *penc = enc_utf8; + rb_enc_associate(undumped, enc_utf8); + } + if (*s == '{') { /* handle \u{...} form */ + s++; + for (;;) { + if (s >= s_end) { + rb_raise(rb_eRuntimeError, "unterminated Unicode escape"); + } + if (*s == '}') { + s++; + break; + } + if (ISSPACE(*s)) { + s++; + continue; + } + c = scan_hex(s, s_end-s, &hexlen); + if (hexlen == 0 || hexlen > 6) { + rb_raise(rb_eRuntimeError, "invalid Unicode escape"); + } + if (c > 0x10ffff) { + rb_raise(rb_eRuntimeError, "invalid Unicode codepoint (too large)"); + } + if (0xd800 <= c && c <= 0xdfff) { + rb_raise(rb_eRuntimeError, "invalid Unicode codepoint"); + } + codelen = rb_enc_mbcput(c, (char *)buf, *penc); + rb_str_cat(undumped, (char *)buf, codelen); + s += hexlen; + } + } + else { /* handle \uXXXX form */ + c = scan_hex(s, 4, &hexlen); + if (hexlen != 4) { + rb_raise(rb_eRuntimeError, "invalid Unicode escape"); + } + if (0xd800 <= c && c <= 0xdfff) { + rb_raise(rb_eRuntimeError, "invalid Unicode codepoint"); + } + codelen = rb_enc_mbcput(c, (char *)buf, *penc); + rb_str_cat(undumped, (char *)buf, codelen); + s += hexlen; + } + break; + case 'x': + if (++s >= s_end) { + rb_raise(rb_eRuntimeError, "invalid hex escape"); + } + *buf = scan_hex(s, 2, &hexlen); + if (hexlen != 2) { + rb_raise(rb_eRuntimeError, "invalid hex escape"); + } + if (!ISASCII(*buf)) { + if (*utf8) { + rb_raise(rb_eRuntimeError, "hex escape and Unicode escape are mixed"); + } + *binary = true; + } + rb_str_cat(undumped, (char *)buf, 1); + s += hexlen; + break; + default: + rb_str_cat(undumped, s-1, 2); + s++; + } + + *ss = s; +} + +static VALUE rb_str_is_ascii_only_p(VALUE str); /* * call-seq: - * str.upcase! => str or nil - * - * Upcases the contents of <i>str</i>, returning <code>nil</code> if no changes - * were made. + * undump -> new_string + * + * Inverse of String#dump; returns a copy of +self+ with changes of the kinds made by String#dump "undone." + * + * Related: see {Converting to New String}[rdoc-ref:String@Converting+to+New+String]. */ static VALUE -rb_str_upcase_bang(str) - VALUE str; +str_undump(VALUE str) { - char *s, *send; - int modify = 0; + const char *s = RSTRING_PTR(str); + const char *s_end = RSTRING_END(str); + rb_encoding *enc = rb_enc_get(str); + VALUE undumped = rb_enc_str_new(s, 0L, enc); + bool utf8 = false; + bool binary = false; + int w; + + rb_must_asciicompat(str); + if (rb_str_is_ascii_only_p(str) == Qfalse) { + rb_raise(rb_eRuntimeError, "non-ASCII character detected"); + } + if (!str_null_check(str, &w)) { + rb_raise(rb_eRuntimeError, "string contains null byte"); + } + if (RSTRING_LEN(str) < 2) goto invalid_format; + if (*s != '"') goto invalid_format; + + /* strip '"' at the start */ + s++; + + for (;;) { + if (s >= s_end) { + rb_raise(rb_eRuntimeError, "unterminated dumped string"); + } + + if (*s == '"') { + /* epilogue */ + s++; + if (s == s_end) { + /* ascii compatible dumped string */ + break; + } + else { + static const char force_encoding_suffix[] = ".force_encoding(\""; /* "\")" */ + static const char dup_suffix[] = ".dup"; + const char *encname; + int encidx; + ptrdiff_t size; + + /* check separately for strings dumped by older versions */ + size = sizeof(dup_suffix) - 1; + if (s_end - s > size && memcmp(s, dup_suffix, size) == 0) s += size; + + size = sizeof(force_encoding_suffix) - 1; + if (s_end - s <= size) goto invalid_format; + if (memcmp(s, force_encoding_suffix, size) != 0) goto invalid_format; + s += size; + + if (utf8) { + rb_raise(rb_eRuntimeError, "dumped string contained Unicode escape but used force_encoding"); + } + + encname = s; + s = memchr(s, '"', s_end-s); + size = s - encname; + if (!s) goto invalid_format; + if (s_end - s != 2) goto invalid_format; + if (s[0] != '"' || s[1] != ')') goto invalid_format; + + encidx = rb_enc_find_index2(encname, (long)size); + if (encidx < 0) { + rb_raise(rb_eRuntimeError, "dumped string has unknown encoding name"); + } + rb_enc_associate_index(undumped, encidx); + } + break; + } + + if (*s == '\\') { + s++; + if (s >= s_end) { + rb_raise(rb_eRuntimeError, "invalid escape"); + } + undump_after_backslash(undumped, &s, s_end, &enc, &utf8, &binary); + } + else { + rb_str_cat(undumped, s++, 1); + } + } + + RB_GC_GUARD(str); + + return undumped; +invalid_format: + rb_raise(rb_eRuntimeError, "invalid dumped string; not wrapped with '\"' nor '\"...\".force_encoding(\"...\")' form"); +} + +static void +rb_str_check_dummy_enc(rb_encoding *enc) +{ + if (rb_enc_dummy_p(enc)) { + rb_raise(rb_eEncCompatError, "incompatible encoding with this operation: %s", + rb_enc_name(enc)); + } +} + +static rb_encoding * +str_true_enc(VALUE str) +{ + rb_encoding *enc = STR_ENC_GET(str); + rb_str_check_dummy_enc(enc); + return enc; +} + +static OnigCaseFoldType +check_case_options(int argc, VALUE *argv, OnigCaseFoldType flags) +{ + if (argc==0) + return flags; + if (argc>2) + rb_raise(rb_eArgError, "too many options"); + if (argv[0]==sym_turkic) { + flags |= ONIGENC_CASE_FOLD_TURKISH_AZERI; + if (argc==2) { + if (argv[1]==sym_lithuanian) + flags |= ONIGENC_CASE_FOLD_LITHUANIAN; + else + rb_raise(rb_eArgError, "invalid second option"); + } + } + else if (argv[0]==sym_lithuanian) { + flags |= ONIGENC_CASE_FOLD_LITHUANIAN; + if (argc==2) { + if (argv[1]==sym_turkic) + flags |= ONIGENC_CASE_FOLD_TURKISH_AZERI; + else + rb_raise(rb_eArgError, "invalid second option"); + } + } + else if (argc>1) + rb_raise(rb_eArgError, "too many options"); + else if (argv[0]==sym_ascii) + flags |= ONIGENC_CASE_ASCII_ONLY; + else if (argv[0]==sym_fold) { + if ((flags & (ONIGENC_CASE_UPCASE|ONIGENC_CASE_DOWNCASE)) == ONIGENC_CASE_DOWNCASE) + flags ^= ONIGENC_CASE_FOLD|ONIGENC_CASE_DOWNCASE; + else + rb_raise(rb_eArgError, "option :fold only allowed for downcasing"); + } + else + rb_raise(rb_eArgError, "invalid option"); + return flags; +} + +static inline bool +case_option_single_p(OnigCaseFoldType flags, rb_encoding *enc, VALUE str) +{ + if ((flags & ONIGENC_CASE_ASCII_ONLY) && (enc==rb_utf8_encoding() || rb_enc_mbmaxlen(enc) == 1)) + return true; + return !(flags & ONIGENC_CASE_FOLD_TURKISH_AZERI) && ENC_CODERANGE(str) == ENC_CODERANGE_7BIT; +} + +/* 16 should be long enough to absorb any kind of single character length increase */ +#define CASE_MAPPING_ADDITIONAL_LENGTH 20 +#ifndef CASEMAP_DEBUG +# define CASEMAP_DEBUG 0 +#endif + +struct mapping_buffer; +typedef struct mapping_buffer { + size_t capa; + size_t used; + struct mapping_buffer *next; + OnigUChar space[FLEX_ARY_LEN]; +} mapping_buffer; + +static void +mapping_buffer_free(void *p) +{ + mapping_buffer *previous_buffer; + mapping_buffer *current_buffer = p; + while (current_buffer) { + previous_buffer = current_buffer; + current_buffer = current_buffer->next; + ruby_sized_xfree(previous_buffer, previous_buffer->capa); + } +} + +static const rb_data_type_t mapping_buffer_type = { + "mapping_buffer", + {0, mapping_buffer_free,}, + 0, 0, RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED +}; + +static VALUE +rb_str_casemap(VALUE source, OnigCaseFoldType *flags, rb_encoding *enc) +{ + VALUE target; + + const OnigUChar *source_current, *source_end; + int target_length = 0; + VALUE buffer_anchor; + mapping_buffer *current_buffer = 0; + mapping_buffer **pre_buffer; + size_t buffer_count = 0; + int buffer_length_or_invalid; + + if (RSTRING_LEN(source) == 0) return str_duplicate(rb_cString, source); + + source_current = (OnigUChar*)RSTRING_PTR(source); + source_end = (OnigUChar*)RSTRING_END(source); + + buffer_anchor = TypedData_Wrap_Struct(0, &mapping_buffer_type, 0); + pre_buffer = (mapping_buffer **)&DATA_PTR(buffer_anchor); + while (source_current < source_end) { + /* increase multiplier using buffer count to converge quickly */ + size_t capa = (size_t)(source_end-source_current)*++buffer_count + CASE_MAPPING_ADDITIONAL_LENGTH; + if (CASEMAP_DEBUG) { + fprintf(stderr, "Buffer allocation, capa is %"PRIuSIZE"\n", capa); /* for tuning */ + } + current_buffer = xmalloc(offsetof(mapping_buffer, space) + capa); + *pre_buffer = current_buffer; + pre_buffer = ¤t_buffer->next; + current_buffer->next = NULL; + current_buffer->capa = capa; + buffer_length_or_invalid = enc->case_map(flags, + &source_current, source_end, + current_buffer->space, + current_buffer->space+current_buffer->capa, + enc); + if (buffer_length_or_invalid < 0) { + current_buffer = DATA_PTR(buffer_anchor); + DATA_PTR(buffer_anchor) = 0; + mapping_buffer_free(current_buffer); + rb_raise(rb_eArgError, "input string invalid"); + } + target_length += current_buffer->used = buffer_length_or_invalid; + } + if (CASEMAP_DEBUG) { + fprintf(stderr, "Buffer count is %"PRIuSIZE"\n", buffer_count); /* for tuning */ + } + + if (buffer_count==1) { + target = rb_str_new((const char*)current_buffer->space, target_length); + } + else { + char *target_current; + + target = rb_str_new(0, target_length); + target_current = RSTRING_PTR(target); + current_buffer = DATA_PTR(buffer_anchor); + while (current_buffer) { + memcpy(target_current, current_buffer->space, current_buffer->used); + target_current += current_buffer->used; + current_buffer = current_buffer->next; + } + } + current_buffer = DATA_PTR(buffer_anchor); + DATA_PTR(buffer_anchor) = 0; + mapping_buffer_free(current_buffer); + + RB_GC_GUARD(buffer_anchor); + + /* TODO: check about string terminator character */ + str_enc_copy_direct(target, source); + /*ENC_CODERANGE_SET(mapped, cr);*/ + + return target; +} + +static VALUE +rb_str_ascii_casemap(VALUE source, VALUE target, OnigCaseFoldType *flags, rb_encoding *enc) +{ + const OnigUChar *source_current, *source_end; + OnigUChar *target_current, *target_end; + long old_length = RSTRING_LEN(source); + int length_or_invalid; + + if (old_length == 0) return Qnil; + + source_current = (OnigUChar*)RSTRING_PTR(source); + source_end = (OnigUChar*)RSTRING_END(source); + if (source == target) { + target_current = (OnigUChar*)source_current; + target_end = (OnigUChar*)source_end; + } + else { + target_current = (OnigUChar*)RSTRING_PTR(target); + target_end = (OnigUChar*)RSTRING_END(target); + } + + length_or_invalid = onigenc_ascii_only_case_map(flags, + &source_current, source_end, + target_current, target_end, enc); + if (length_or_invalid < 0) + rb_raise(rb_eArgError, "input string invalid"); + if (CASEMAP_DEBUG && length_or_invalid != old_length) { + fprintf(stderr, "problem with rb_str_ascii_casemap" + "; old_length=%ld, new_length=%d\n", old_length, length_or_invalid); + rb_raise(rb_eArgError, "internal problem with rb_str_ascii_casemap" + "; old_length=%ld, new_length=%d\n", old_length, length_or_invalid); + } + + str_enc_copy(target, source); + + return target; +} + +static bool +upcase_single(VALUE str) +{ + char *s = RSTRING_PTR(str), *send = RSTRING_END(str); + bool modified = false; - rb_str_modify(str); - s = RSTRING(str)->ptr; send = s + RSTRING(str)->len; while (s < send) { - if (ismbchar(*s)) { - s+=mbclen(*s) - 1; - } - else if (ISLOWER(*s)) { - *s = toupper(*s); - modify = 1; - } - s++; + unsigned int c = *(unsigned char*)s; + + if ('a' <= c && c <= 'z') { + *s = 'A' + (c - 'a'); + modified = true; + } + s++; } + return modified; +} - if (modify) return str; +/* + * call-seq: + * upcase!(mapping) -> self or nil + * + * Like String#upcase, except that: + * + * - Changes character casings in +self+ (not in a copy of +self+). + * - Returns +self+ if any changes are made, +nil+ otherwise. + * + * Related: See {Modifying}[rdoc-ref:String@Modifying]. + */ + +static VALUE +rb_str_upcase_bang(int argc, VALUE *argv, VALUE str) +{ + rb_encoding *enc; + OnigCaseFoldType flags = ONIGENC_CASE_UPCASE; + + flags = check_case_options(argc, argv, flags); + str_modify_keep_cr(str); + enc = str_true_enc(str); + if (case_option_single_p(flags, enc, str)) { + if (upcase_single(str)) + flags |= ONIGENC_CASE_MODIFIED; + } + else if (flags&ONIGENC_CASE_ASCII_ONLY) + rb_str_ascii_casemap(str, str, &flags, enc); + else + str_shared_replace(str, rb_str_casemap(str, &flags, enc)); + + if (ONIGENC_CASE_MODIFIED&flags) return str; return Qnil; } /* * call-seq: - * str.upcase => new_str - * - * Returns a copy of <i>str</i> with all lowercase letters replaced with their - * uppercase counterparts. The operation is locale insensitive---only - * characters ``a'' to ``z'' are affected. - * - * "hEllO".upcase #=> "HELLO" + * upcase(mapping = :ascii) -> new_string + * + * :include: doc/string/upcase.rdoc */ static VALUE -rb_str_upcase(str) - VALUE str; +rb_str_upcase(int argc, VALUE *argv, VALUE str) { - str = rb_str_dup(str); - rb_str_upcase_bang(str); - return str; + rb_encoding *enc; + OnigCaseFoldType flags = ONIGENC_CASE_UPCASE; + VALUE ret; + + flags = check_case_options(argc, argv, flags); + enc = str_true_enc(str); + if (case_option_single_p(flags, enc, str)) { + ret = rb_str_new(RSTRING_PTR(str), RSTRING_LEN(str)); + str_enc_copy_direct(ret, str); + upcase_single(ret); + } + else if (flags&ONIGENC_CASE_ASCII_ONLY) { + ret = rb_str_new(0, RSTRING_LEN(str)); + rb_str_ascii_casemap(str, ret, &flags, enc); + } + else { + ret = rb_str_casemap(str, &flags, enc); + } + + return ret; } +static bool +downcase_single(VALUE str) +{ + char *s = RSTRING_PTR(str), *send = RSTRING_END(str); + bool modified = false; + + while (s < send) { + unsigned int c = *(unsigned char*)s; + + if ('A' <= c && c <= 'Z') { + *s = 'a' + (c - 'A'); + modified = true; + } + s++; + } + + return modified; +} /* * call-seq: - * str.downcase! => str or nil - * - * Downcases the contents of <i>str</i>, returning <code>nil</code> if no - * changes were made. + * downcase!(mapping) -> self or nil + * + * Like String#downcase, except that: + * + * - Changes character casings in +self+ (not in a copy of +self+). + * - Returns +self+ if any changes are made, +nil+ otherwise. + * + * Related: See {Modifying}[rdoc-ref:String@Modifying]. */ static VALUE -rb_str_downcase_bang(str) - VALUE str; +rb_str_downcase_bang(int argc, VALUE *argv, VALUE str) { - char *s, *send; - int modify = 0; + rb_encoding *enc; + OnigCaseFoldType flags = ONIGENC_CASE_DOWNCASE; - rb_str_modify(str); - s = RSTRING(str)->ptr; send = s + RSTRING(str)->len; - while (s < send) { - if (ismbchar(*s)) { - s+=mbclen(*s) - 1; - } - else if (ISUPPER(*s)) { - *s = tolower(*s); - modify = 1; - } - s++; + flags = check_case_options(argc, argv, flags); + str_modify_keep_cr(str); + enc = str_true_enc(str); + if (case_option_single_p(flags, enc, str)) { + if (downcase_single(str)) + flags |= ONIGENC_CASE_MODIFIED; } + else if (flags&ONIGENC_CASE_ASCII_ONLY) + rb_str_ascii_casemap(str, str, &flags, enc); + else + str_shared_replace(str, rb_str_casemap(str, &flags, enc)); - if (modify) return str; + if (ONIGENC_CASE_MODIFIED&flags) return str; return Qnil; } /* * call-seq: - * str.downcase => new_str - * - * Returns a copy of <i>str</i> with all uppercase letters replaced with their - * lowercase counterparts. The operation is locale insensitive---only - * characters ``A'' to ``Z'' are affected. - * - * "hEllO".downcase #=> "hello" + * downcase(mapping = :ascii) -> new_string + * + * :include: doc/string/downcase.rdoc + * */ static VALUE -rb_str_downcase(str) - VALUE str; +rb_str_downcase(int argc, VALUE *argv, VALUE str) { - str = rb_str_dup(str); - rb_str_downcase_bang(str); - return str; + rb_encoding *enc; + OnigCaseFoldType flags = ONIGENC_CASE_DOWNCASE; + VALUE ret; + + flags = check_case_options(argc, argv, flags); + enc = str_true_enc(str); + if (case_option_single_p(flags, enc, str)) { + ret = rb_str_new(RSTRING_PTR(str), RSTRING_LEN(str)); + str_enc_copy_direct(ret, str); + downcase_single(ret); + } + else if (flags&ONIGENC_CASE_ASCII_ONLY) { + ret = rb_str_new(0, RSTRING_LEN(str)); + rb_str_ascii_casemap(str, ret, &flags, enc); + } + else { + ret = rb_str_casemap(str, &flags, enc); + } + + return ret; } /* * call-seq: - * str.capitalize! => str or nil - * - * Modifies <i>str</i> by converting the first character to uppercase and the - * remainder to lowercase. Returns <code>nil</code> if no changes are made. - * - * a = "hello" - * a.capitalize! #=> "Hello" - * a #=> "Hello" - * a.capitalize! #=> nil + * capitalize!(mapping = :ascii) -> self or nil + * + * Like String#capitalize, except that: + * + * - Changes character casings in +self+ (not in a copy of +self+). + * - Returns +self+ if any changes are made, +nil+ otherwise. + * + * Related: See {Modifying}[rdoc-ref:String@Modifying]. */ static VALUE -rb_str_capitalize_bang(str) - VALUE str; +rb_str_capitalize_bang(int argc, VALUE *argv, VALUE str) { - char *s, *send; - int modify = 0; + rb_encoding *enc; + OnigCaseFoldType flags = ONIGENC_CASE_UPCASE | ONIGENC_CASE_TITLECASE; - rb_str_modify(str); - if (RSTRING(str)->len == 0 || !RSTRING(str)->ptr) return Qnil; - s = RSTRING(str)->ptr; send = s + RSTRING(str)->len; - if (ISLOWER(*s)) { - *s = toupper(*s); - modify = 1; - } - while (++s < send) { - if (ismbchar(*s)) { - s+=mbclen(*s) - 1; - } - else if (ISUPPER(*s)) { - *s = tolower(*s); - modify = 1; - } - } - if (modify) return str; + flags = check_case_options(argc, argv, flags); + str_modify_keep_cr(str); + enc = str_true_enc(str); + if (RSTRING_LEN(str) == 0 || !RSTRING_PTR(str)) return Qnil; + if (flags&ONIGENC_CASE_ASCII_ONLY) + rb_str_ascii_casemap(str, str, &flags, enc); + else + str_shared_replace(str, rb_str_casemap(str, &flags, enc)); + + if (ONIGENC_CASE_MODIFIED&flags) return str; return Qnil; } /* * call-seq: - * str.capitalize => new_str - * - * Returns a copy of <i>str</i> with the first character converted to uppercase - * and the remainder to lowercase. - * - * "hello".capitalize #=> "Hello" - * "HELLO".capitalize #=> "Hello" - * "123ABC".capitalize #=> "123abc" + * capitalize(mapping = :ascii) -> new_string + * + * :include: doc/string/capitalize.rdoc + * */ static VALUE -rb_str_capitalize(str) - VALUE str; +rb_str_capitalize(int argc, VALUE *argv, VALUE str) { - str = rb_str_dup(str); - rb_str_capitalize_bang(str); - return str; + rb_encoding *enc; + OnigCaseFoldType flags = ONIGENC_CASE_UPCASE | ONIGENC_CASE_TITLECASE; + VALUE ret; + + flags = check_case_options(argc, argv, flags); + enc = str_true_enc(str); + if (RSTRING_LEN(str) == 0 || !RSTRING_PTR(str)) return str; + if (flags&ONIGENC_CASE_ASCII_ONLY) { + ret = rb_str_new(0, RSTRING_LEN(str)); + rb_str_ascii_casemap(str, ret, &flags, enc); + } + else { + ret = rb_str_casemap(str, &flags, enc); + } + return ret; } /* * call-seq: - * str.swapcase! => str or nil - * - * Equivalent to <code>String#swapcase</code>, but modifies the receiver in - * place, returning <i>str</i>, or <code>nil</code> if no changes were made. + * swapcase!(mapping) -> self or nil + * + * Like String#swapcase, except that: + * + * - Changes are made to +self+, not to copy of +self+. + * - Returns +self+ if any changes are made, +nil+ otherwise. + * + * Related: see {Modifying}[rdoc-ref:String@Modifying]. */ static VALUE -rb_str_swapcase_bang(str) - VALUE str; +rb_str_swapcase_bang(int argc, VALUE *argv, VALUE str) { - char *s, *send; - int modify = 0; + rb_encoding *enc; + OnigCaseFoldType flags = ONIGENC_CASE_UPCASE | ONIGENC_CASE_DOWNCASE; - rb_str_modify(str); - s = RSTRING(str)->ptr; send = s + RSTRING(str)->len; - while (s < send) { - if (ismbchar(*s)) { - s+=mbclen(*s) - 1; - } - else if (ISUPPER(*s)) { - *s = tolower(*s); - modify = 1; - } - else if (ISLOWER(*s)) { - *s = toupper(*s); - modify = 1; - } - s++; - } + flags = check_case_options(argc, argv, flags); + str_modify_keep_cr(str); + enc = str_true_enc(str); + if (flags&ONIGENC_CASE_ASCII_ONLY) + rb_str_ascii_casemap(str, str, &flags, enc); + else + str_shared_replace(str, rb_str_casemap(str, &flags, enc)); - if (modify) return str; + if (ONIGENC_CASE_MODIFIED&flags) return str; return Qnil; } /* * call-seq: - * str.swapcase => new_str - * - * Returns a copy of <i>str</i> with uppercase alphabetic characters converted - * to lowercase and lowercase characters converted to uppercase. - * - * "Hello".swapcase #=> "hELLO" - * "cYbEr_PuNk11".swapcase #=> "CyBeR_pUnK11" + * swapcase(mapping = :ascii) -> new_string + * + * :include: doc/string/swapcase.rdoc + * */ static VALUE -rb_str_swapcase(str) - VALUE str; +rb_str_swapcase(int argc, VALUE *argv, VALUE str) { - str = rb_str_dup(str); - rb_str_swapcase_bang(str); - return str; + rb_encoding *enc; + OnigCaseFoldType flags = ONIGENC_CASE_UPCASE | ONIGENC_CASE_DOWNCASE; + VALUE ret; + + flags = check_case_options(argc, argv, flags); + enc = str_true_enc(str); + if (RSTRING_LEN(str) == 0 || !RSTRING_PTR(str)) return str_duplicate(rb_cString, str); + if (flags&ONIGENC_CASE_ASCII_ONLY) { + ret = rb_str_new(0, RSTRING_LEN(str)); + rb_str_ascii_casemap(str, ret, &flags, enc); + } + else { + ret = rb_str_casemap(str, &flags, enc); + } + return ret; } typedef unsigned char *USTR; struct tr { - int gen, now, max; + int gen; + unsigned int now, max; char *p, *pend; }; -static int -trnext(t) - struct tr *t; +static unsigned int +trnext(struct tr *t, rb_encoding *enc) { + int n; + for (;;) { - if (!t->gen) { - if (t->p == t->pend) return -1; - if (t->p < t->pend - 1 && *t->p == '\\') { - t->p++; - } - t->now = *(USTR)t->p++; - if (t->p < t->pend - 1 && *t->p == '-') { - t->p++; - if (t->p < t->pend) { - if (t->now > *(USTR)t->p) { - t->p++; - continue; - } - t->gen = 1; - t->max = *(USTR)t->p++; - } - } - return t->now; - } - else if (++t->now < t->max) { - return t->now; - } - else { - t->gen = 0; - return t->max; - } - } -} - -static VALUE rb_str_delete_bang _((int,VALUE*,VALUE)); - -static VALUE -tr_trans(str, src, repl, sflag) - VALUE str, src, repl; - int sflag; + nextpart: + if (!t->gen) { + if (t->p == t->pend) return -1; + if (rb_enc_ascget(t->p, t->pend, &n, enc) == '\\' && t->p + n < t->pend) { + t->p += n; + } + t->now = rb_enc_codepoint_len(t->p, t->pend, &n, enc); + t->p += n; + if (rb_enc_ascget(t->p, t->pend, &n, enc) == '-' && t->p + n < t->pend) { + t->p += n; + if (t->p < t->pend) { + unsigned int c = rb_enc_codepoint_len(t->p, t->pend, &n, enc); + t->p += n; + if (t->now > c) { + if (t->now < 0x80 && c < 0x80) { + rb_raise(rb_eArgError, + "invalid range \"%c-%c\" in string transliteration", + t->now, c); + } + else { + rb_raise(rb_eArgError, "invalid range in string transliteration"); + } + continue; /* not reached */ + } + else if (t->now < c) { + t->gen = 1; + t->max = c; + } + } + } + return t->now; + } + else { + while (ONIGENC_CODE_TO_MBCLEN(enc, ++t->now) <= 0) { + if (t->now == t->max) { + t->gen = 0; + goto nextpart; + } + } + if (t->now < t->max) { + return t->now; + } + else { + t->gen = 0; + return t->max; + } + } + } +} + +static VALUE rb_str_delete_bang(int,VALUE*,VALUE); + +static VALUE +tr_trans(VALUE str, VALUE src, VALUE repl, int sflag) { + const unsigned int errc = -1; + unsigned int trans[256]; + rb_encoding *enc, *e1, *e2; struct tr trsrc, trrepl; int cflag = 0; - int trans[256]; - int i, c, modify = 0; - char *s, *send; + unsigned int c, c0, last = 0; + int modify = 0, i, l; + unsigned char *s, *send; + VALUE hash = 0; + int singlebyte = single_byte_optimizable(str); + int termlen; + int cr; + +#define CHECK_IF_ASCII(c) \ + (void)((cr == ENC_CODERANGE_7BIT && !rb_isascii(c)) ? \ + (cr = ENC_CODERANGE_VALID) : 0) StringValue(src); StringValue(repl); - if (RSTRING(str)->len == 0 || !RSTRING(str)->ptr) return Qnil; - trsrc.p = RSTRING(src)->ptr; trsrc.pend = trsrc.p + RSTRING(src)->len; - if (RSTRING(src)->len >= 2 && RSTRING(src)->ptr[0] == '^') { - cflag++; - trsrc.p++; + if (RSTRING_LEN(str) == 0 || !RSTRING_PTR(str)) return Qnil; + if (RSTRING_LEN(repl) == 0) { + return rb_str_delete_bang(1, &src, str); + } + + cr = ENC_CODERANGE(str); + e1 = rb_enc_check(str, src); + e2 = rb_enc_check(str, repl); + if (e1 == e2) { + enc = e1; } - if (RSTRING(repl)->len == 0) { - return rb_str_delete_bang(1, &src, str); + else { + enc = rb_enc_check(src, repl); + } + trsrc.p = RSTRING_PTR(src); trsrc.pend = trsrc.p + RSTRING_LEN(src); + if (RSTRING_LEN(src) > 1 && + rb_enc_ascget(trsrc.p, trsrc.pend, &l, enc) == '^' && + trsrc.p + l < trsrc.pend) { + cflag = 1; + trsrc.p += l; } - trrepl.p = RSTRING(repl)->ptr; - trrepl.pend = trrepl.p + RSTRING(repl)->len; + trrepl.p = RSTRING_PTR(repl); + trrepl.pend = trrepl.p + RSTRING_LEN(repl); trsrc.gen = trrepl.gen = 0; trsrc.now = trrepl.now = 0; trsrc.max = trrepl.max = 0; if (cflag) { - for (i=0; i<256; i++) { - trans[i] = 1; - } - while ((c = trnext(&trsrc)) >= 0) { - trans[c & 0xff] = -1; - } - while ((c = trnext(&trrepl)) >= 0) - /* retrieve last replacer */; - for (i=0; i<256; i++) { - if (trans[i] >= 0) { - trans[i] = trrepl.now; - } - } + for (i=0; i<256; i++) { + trans[i] = 1; + } + while ((c = trnext(&trsrc, enc)) != errc) { + if (c < 256) { + trans[c] = errc; + } + else { + if (!hash) hash = rb_hash_new(); + rb_hash_aset(hash, UINT2NUM(c), Qtrue); + } + } + while ((c = trnext(&trrepl, enc)) != errc) + /* retrieve last replacer */; + last = trrepl.now; + for (i=0; i<256; i++) { + if (trans[i] != errc) { + trans[i] = last; + } + } } else { - int r; + unsigned int r; - for (i=0; i<256; i++) { - trans[i] = -1; - } - while ((c = trnext(&trsrc)) >= 0) { - r = trnext(&trrepl); - if (r == -1) r = trrepl.now; - trans[c & 0xff] = r; - } + for (i=0; i<256; i++) { + trans[i] = errc; + } + while ((c = trnext(&trsrc, enc)) != errc) { + r = trnext(&trrepl, enc); + if (r == errc) r = trrepl.now; + if (c < 256) { + trans[c] = r; + if (rb_enc_codelen(r, enc) != 1) singlebyte = 0; + } + else { + if (!hash) hash = rb_hash_new(); + rb_hash_aset(hash, UINT2NUM(c), UINT2NUM(r)); + } + } } - rb_str_modify(str); - s = RSTRING(str)->ptr; send = s + RSTRING(str)->len; + if (cr == ENC_CODERANGE_VALID && rb_enc_asciicompat(e1)) + cr = ENC_CODERANGE_7BIT; + str_modify_keep_cr(str); + s = (unsigned char *)RSTRING_PTR(str); send = (unsigned char *)RSTRING_END(str); + termlen = rb_enc_mbminlen(enc); if (sflag) { - char *t = s; - int c0, last = -1; - - while (s < send) { - c0 = *s++; - if ((c = trans[c0 & 0xff]) >= 0) { - if (last == c) continue; - last = c; - *t++ = c & 0xff; - modify = 1; - } - else { - last = -1; - *t++ = c0; - } - } - if (RSTRING(str)->len > (t - RSTRING(str)->ptr)) { - RSTRING(str)->len = (t - RSTRING(str)->ptr); - modify = 1; - *t = '\0'; - } + int clen, tlen; + long offset, max = RSTRING_LEN(str); + unsigned int save = -1; + unsigned char *buf = ALLOC_N(unsigned char, max + termlen), *t = buf; + + while (s < send) { + int may_modify = 0; + + int r = rb_enc_precise_mbclen((char *)s, (char *)send, e1); + if (!MBCLEN_CHARFOUND_P(r)) { + xfree(buf); + rb_raise(rb_eArgError, "invalid byte sequence in %s", rb_enc_name(e1)); + } + clen = MBCLEN_CHARFOUND_LEN(r); + c0 = c = rb_enc_mbc_to_codepoint((char *)s, (char *)send, e1); + + tlen = enc == e1 ? clen : rb_enc_codelen(c, enc); + + s += clen; + if (c < 256) { + c = trans[c]; + } + else if (hash) { + VALUE tmp = rb_hash_lookup(hash, UINT2NUM(c)); + if (NIL_P(tmp)) { + if (cflag) c = last; + else c = errc; + } + else if (cflag) c = errc; + else c = NUM2INT(tmp); + } + else { + c = errc; + } + if (c != (unsigned int)-1) { + if (save == c) { + CHECK_IF_ASCII(c); + continue; + } + save = c; + tlen = rb_enc_codelen(c, enc); + modify = 1; + } + else { + save = -1; + c = c0; + if (enc != e1) may_modify = 1; + } + if ((offset = t - buf) + tlen > max) { + size_t MAYBE_UNUSED(old) = max + termlen; + max = offset + tlen + (send - s); + SIZED_REALLOC_N(buf, unsigned char, max + termlen, old); + t = buf + offset; + } + rb_enc_mbcput(c, t, enc); + if (may_modify && memcmp(s, t, tlen) != 0) { + modify = 1; + } + CHECK_IF_ASCII(c); + t += tlen; + } + if (!STR_EMBED_P(str)) { + ruby_sized_xfree(STR_HEAP_PTR(str), STR_HEAP_SIZE(str)); + } + TERM_FILL((char *)t, termlen); + RSTRING(str)->as.heap.ptr = (char *)buf; + STR_SET_LEN(str, t - buf); + STR_SET_NOEMBED(str); + RSTRING(str)->as.heap.aux.capa = max; + } + else if (rb_enc_mbmaxlen(enc) == 1 || (singlebyte && !hash)) { + while (s < send) { + c = (unsigned char)*s; + if (trans[c] != errc) { + if (!cflag) { + c = trans[c]; + *s = c; + modify = 1; + } + else { + *s = last; + modify = 1; + } + } + CHECK_IF_ASCII(c); + s++; + } } else { - while (s < send) { - if ((c = trans[*s & 0xff]) >= 0) { - *s = c & 0xff; - modify = 1; - } - s++; - } + int clen, tlen; + long offset, max = (long)((send - s) * 1.2); + unsigned char *buf = ALLOC_N(unsigned char, max + termlen), *t = buf; + + while (s < send) { + int may_modify = 0; + + int r = rb_enc_precise_mbclen((char *)s, (char *)send, e1); + if (!MBCLEN_CHARFOUND_P(r)) { + xfree(buf); + rb_raise(rb_eArgError, "invalid byte sequence in %s", rb_enc_name(e1)); + } + clen = MBCLEN_CHARFOUND_LEN(r); + c0 = c = rb_enc_mbc_to_codepoint((char *)s, (char *)send, e1); + + tlen = enc == e1 ? clen : rb_enc_codelen(c, enc); + + if (c < 256) { + c = trans[c]; + } + else if (hash) { + VALUE tmp = rb_hash_lookup(hash, UINT2NUM(c)); + if (NIL_P(tmp)) { + if (cflag) c = last; + else c = errc; + } + else if (cflag) c = errc; + else c = NUM2INT(tmp); + } + else { + c = cflag ? last : errc; + } + if (c != errc) { + tlen = rb_enc_codelen(c, enc); + modify = 1; + } + else { + c = c0; + if (enc != e1) may_modify = 1; + } + if ((offset = t - buf) + tlen > max) { + size_t MAYBE_UNUSED(old) = max + termlen; + max = offset + tlen + (long)((send - s) * 1.2); + SIZED_REALLOC_N(buf, unsigned char, max + termlen, old); + t = buf + offset; + } + if (s != t) { + rb_enc_mbcput(c, t, enc); + if (may_modify && memcmp(s, t, tlen) != 0) { + modify = 1; + } + } + CHECK_IF_ASCII(c); + s += clen; + t += tlen; + } + if (!STR_EMBED_P(str)) { + ruby_sized_xfree(STR_HEAP_PTR(str), STR_HEAP_SIZE(str)); + } + TERM_FILL((char *)t, termlen); + RSTRING(str)->as.heap.ptr = (char *)buf; + STR_SET_LEN(str, t - buf); + STR_SET_NOEMBED(str); + RSTRING(str)->as.heap.aux.capa = max; } - if (modify) return str; + if (modify) { + if (cr != ENC_CODERANGE_BROKEN) + ENC_CODERANGE_SET(str, cr); + rb_enc_associate(str, enc); + return str; + } return Qnil; } /* * call-seq: - * str.tr!(from_str, to_str) => str or nil - * - * Translates <i>str</i> in place, using the same rules as - * <code>String#tr</code>. Returns <i>str</i>, or <code>nil</code> if no - * changes were made. + * tr!(selector, replacements) -> self or nil + * + * Like String#tr, except: + * + * - Performs substitutions in +self+ (not in a copy of +self+). + * - Returns +self+ if any modifications were made, +nil+ otherwise. + * + * Related: {Modifying}[rdoc-ref:String@Modifying]. */ static VALUE -rb_str_tr_bang(str, src, repl) - VALUE str, src, repl; +rb_str_tr_bang(VALUE str, VALUE src, VALUE repl) { return tr_trans(str, src, repl, 0); } @@ -3065,109 +8580,203 @@ rb_str_tr_bang(str, src, repl) /* * call-seq: - * str.tr(from_str, to_str) => new_str - * - * Returns a copy of <i>str</i> with the characters in <i>from_str</i> replaced - * by the corresponding characters in <i>to_str</i>. If <i>to_str</i> is - * shorter than <i>from_str</i>, it is padded with its last character. Both - * strings may use the c1--c2 notation to denote ranges of characters, and - * <i>from_str</i> may start with a <code>^</code>, which denotes all - * characters except those listed. - * - * "hello".tr('aeiou', '*') #=> "h*ll*" - * "hello".tr('^aeiou', '*') #=> "*e**o" - * "hello".tr('el', 'ip') #=> "hippo" - * "hello".tr('a-y', 'b-z') #=> "ifmmp" + * tr(selector, replacements) -> new_string + * + * Returns a copy of +self+ with each character specified by string +selector+ + * translated to the corresponding character in string +replacements+. + * The correspondence is _positional_: + * + * - Each occurrence of the first character specified by +selector+ + * is translated to the first character in +replacements+. + * - Each occurrence of the second character specified by +selector+ + * is translated to the second character in +replacements+. + * - And so on. + * + * Example: + * + * 'hello'.tr('el', 'ip') #=> "hippo" + * + * If +replacements+ is shorter than +selector+, + * it is implicitly padded with its own last character: + * + * 'hello'.tr('aeiou', '-') # => "h-ll-" + * 'hello'.tr('aeiou', 'AA-') # => "hAll-" + * + * Arguments +selector+ and +replacements+ must be valid character selectors + * (see {Character Selectors}[rdoc-ref:character_selectors.rdoc]), + * and may use any of its valid forms, including negation, ranges, and escapes: + * + * 'hello'.tr('^aeiou', '-') # => "-e--o" # Negation. + * 'ibm'.tr('b-z', 'a-z') # => "hal" # Range. + * 'hel^lo'.tr('\^aeiou', '-') # => "h-l-l-" # Escaped leading caret. + * 'i-b-m'.tr('b\-z', 'a-z') # => "ibabm" # Escaped embedded hyphen. + * 'foo\\bar'.tr('ab\\', 'XYZ') # => "fooZYXr" # Escaped backslash. + * + * Related: see {Converting to New String}[rdoc-ref:String@Converting+to+New+String]. */ static VALUE -rb_str_tr(str, src, repl) - VALUE str, src, repl; +rb_str_tr(VALUE str, VALUE src, VALUE repl) { - str = rb_str_dup(str); + str = str_duplicate(rb_cString, str); tr_trans(str, src, repl, 0); return str; } +#define TR_TABLE_MAX (UCHAR_MAX+1) +#define TR_TABLE_SIZE (TR_TABLE_MAX+1) static void -tr_setup_table(str, table, init) - VALUE str; - char table[256]; - int init; +tr_setup_table(VALUE str, char stable[TR_TABLE_SIZE], int first, + VALUE *tablep, VALUE *ctablep, rb_encoding *enc) { - char buf[256]; + const unsigned int errc = -1; + char buf[TR_TABLE_MAX]; struct tr tr; - int i, c; - int cflag = 0; + unsigned int c; + VALUE table = 0, ptable = 0; + int i, l, cflag = 0; - tr.p = RSTRING(str)->ptr; tr.pend = tr.p + RSTRING(str)->len; + tr.p = RSTRING_PTR(str); tr.pend = tr.p + RSTRING_LEN(str); tr.gen = tr.now = tr.max = 0; - if (RSTRING(str)->len > 1 && RSTRING(str)->ptr[0] == '^') { - cflag = 1; - tr.p++; - } - if (init) { - for (i=0; i<256; i++) { - table[i] = 1; - } + if (RSTRING_LEN(str) > 1 && rb_enc_ascget(tr.p, tr.pend, &l, enc) == '^') { + cflag = 1; + tr.p += l; + } + if (first) { + for (i=0; i<TR_TABLE_MAX; i++) { + stable[i] = 1; + } + stable[TR_TABLE_MAX] = cflag; } - for (i=0; i<256; i++) { - buf[i] = cflag; + else if (stable[TR_TABLE_MAX] && !cflag) { + stable[TR_TABLE_MAX] = 0; } - while ((c = trnext(&tr)) >= 0) { - buf[c & 0xff] = !cflag; + for (i=0; i<TR_TABLE_MAX; i++) { + buf[i] = cflag; } - for (i=0; i<256; i++) { - table[i] = table[i]&&buf[i]; + + while ((c = trnext(&tr, enc)) != errc) { + if (c < TR_TABLE_MAX) { + buf[(unsigned char)c] = !cflag; + } + else { + VALUE key = UINT2NUM(c); + + if (!table && (first || *tablep || stable[TR_TABLE_MAX])) { + if (cflag) { + ptable = *ctablep; + table = ptable ? ptable : rb_hash_new(); + *ctablep = table; + } + else { + table = rb_hash_new(); + ptable = *tablep; + *tablep = table; + } + } + if (table && (!ptable || (cflag ^ !NIL_P(rb_hash_aref(ptable, key))))) { + rb_hash_aset(table, key, Qtrue); + } + } + } + for (i=0; i<TR_TABLE_MAX; i++) { + stable[i] = stable[i] && buf[i]; + } + if (!table && !cflag) { + *tablep = 0; } } +static int +tr_find(unsigned int c, const char table[TR_TABLE_SIZE], VALUE del, VALUE nodel) +{ + if (c < TR_TABLE_MAX) { + return table[c] != 0; + } + else { + VALUE v = UINT2NUM(c); + + if (del) { + if (!NIL_P(rb_hash_lookup(del, v)) && + (!nodel || NIL_P(rb_hash_lookup(nodel, v)))) { + return TRUE; + } + } + else if (nodel && !NIL_P(rb_hash_lookup(nodel, v))) { + return FALSE; + } + return table[TR_TABLE_MAX] ? TRUE : FALSE; + } +} + /* * call-seq: - * str.delete!([other_str]+>) => str or nil - * - * Performs a <code>delete</code> operation in place, returning <i>str</i>, or - * <code>nil</code> if <i>str</i> was not modified. + * delete!(*selectors) -> self or nil + * + * Like String#delete, but modifies +self+ in place; + * returns +self+ if any characters were deleted, +nil+ otherwise. + * + * Related: see {Modifying}[rdoc-ref:String@Modifying]. */ static VALUE -rb_str_delete_bang(argc, argv, str) - int argc; - VALUE *argv; - VALUE str; +rb_str_delete_bang(int argc, VALUE *argv, VALUE str) { + char squeez[TR_TABLE_SIZE]; + rb_encoding *enc = 0; char *s, *send, *t; - char squeez[256]; + VALUE del = 0, nodel = 0; int modify = 0; - int init = 1; - int i; + int i, ascompat, cr; - if (argc < 1) { - rb_raise(rb_eArgError, "wrong number of arguments"); - } + if (RSTRING_LEN(str) == 0 || !RSTRING_PTR(str)) return Qnil; + rb_check_arity(argc, 1, UNLIMITED_ARGUMENTS); for (i=0; i<argc; i++) { - VALUE s = argv[i]; + VALUE s = argv[i]; - StringValue(s); - tr_setup_table(s, squeez, init); - init = 0; + StringValue(s); + enc = rb_enc_check(str, s); + tr_setup_table(s, squeez, i==0, &del, &nodel, enc); } - rb_str_modify(str); - s = t = RSTRING(str)->ptr; - if (!s || RSTRING(str)->len == 0) return Qnil; - send = s + RSTRING(str)->len; + str_modify_keep_cr(str); + ascompat = rb_enc_asciicompat(enc); + s = t = RSTRING_PTR(str); + send = RSTRING_END(str); + cr = ascompat ? ENC_CODERANGE_7BIT : ENC_CODERANGE_VALID; while (s < send) { - if (squeez[*s & 0xff]) - modify = 1; - else - *t++ = *s; - s++; + unsigned int c; + int clen; + + if (ascompat && (c = *(unsigned char*)s) < 0x80) { + if (squeez[c]) { + modify = 1; + } + else { + if (t != s) *t = c; + t++; + } + s++; + } + else { + c = rb_enc_codepoint_len(s, send, &clen, enc); + + if (tr_find(c, squeez, del, nodel)) { + modify = 1; + } + else { + if (t != s) rb_enc_mbcput(c, t, enc); + t += clen; + if (cr == ENC_CODERANGE_7BIT) cr = ENC_CODERANGE_VALID; + } + s += clen; + } } - *t = '\0'; - RSTRING(str)->len = t - RSTRING(str)->ptr; + TERM_FILL(t, TERM_LEN(str)); + STR_SET_LEN(str, t - RSTRING_PTR(str)); + ENC_CODERANGE_SET(str, cr); if (modify) return str; return Qnil; @@ -3176,25 +8785,16 @@ rb_str_delete_bang(argc, argv, str) /* * call-seq: - * str.delete([other_str]+) => new_str - * - * Returns a copy of <i>str</i> with all characters in the intersection of its - * arguments deleted. Uses the same rules for building the set of characters as - * <code>String#count</code>. - * - * "hello".delete "l","lo" #=> "heo" - * "hello".delete "lo" #=> "he" - * "hello".delete "aeiou", "^e" #=> "hell" - * "hello".delete "ej-m" #=> "ho" + * delete(*selectors) -> new_string + * + * :include: doc/string/delete.rdoc + * */ static VALUE -rb_str_delete(argc, argv, str) - int argc; - VALUE *argv; - VALUE str; +rb_str_delete(int argc, VALUE *argv, VALUE str) { - str = rb_str_dup(str); + str = str_duplicate(rb_cString, str); rb_str_delete_bang(argc, argv, str); return str; } @@ -3202,54 +8802,85 @@ rb_str_delete(argc, argv, str) /* * call-seq: - * str.squeeze!([other_str]*) => str or nil - * - * Squeezes <i>str</i> in place, returning either <i>str</i>, or - * <code>nil</code> if no changes were made. + * squeeze!(*selectors) -> self or nil + * + * Like String#squeeze, except that: + * + * - Characters are squeezed in +self+ (not in a copy of +self+). + * - Returns +self+ if any changes are made, +nil+ otherwise. + * + * Related: See {Modifying}[rdoc-ref:String@Modifying]. */ static VALUE -rb_str_squeeze_bang(argc, argv, str) - int argc; - VALUE *argv; - VALUE str; +rb_str_squeeze_bang(int argc, VALUE *argv, VALUE str) { - char squeez[256]; - char *s, *send, *t; - int c, save, modify = 0; - int init = 1; - int i; + char squeez[TR_TABLE_SIZE]; + rb_encoding *enc = 0; + VALUE del = 0, nodel = 0; + unsigned char *s, *send, *t; + int i, modify = 0; + int ascompat, singlebyte = single_byte_optimizable(str); + unsigned int save; if (argc == 0) { - for (i=0; i<256; i++) { - squeez[i] = 1; - } + enc = STR_ENC_GET(str); } else { - for (i=0; i<argc; i++) { - VALUE s = argv[i]; - - StringValue(s); - tr_setup_table(s, squeez, init); - init = 0; - } + for (i=0; i<argc; i++) { + VALUE s = argv[i]; + + StringValue(s); + enc = rb_enc_check(str, s); + if (singlebyte && !single_byte_optimizable(s)) + singlebyte = 0; + tr_setup_table(s, squeez, i==0, &del, &nodel, enc); + } } - rb_str_modify(str); - s = t = RSTRING(str)->ptr; - if (!s || RSTRING(str)->len == 0) return Qnil; - send = s + RSTRING(str)->len; + str_modify_keep_cr(str); + s = t = (unsigned char *)RSTRING_PTR(str); + if (!s || RSTRING_LEN(str) == 0) return Qnil; + send = (unsigned char *)RSTRING_END(str); save = -1; - while (s < send) { - c = *s++ & 0xff; - if (c != save || !squeez[c]) { - *t++ = save = c; - } + ascompat = rb_enc_asciicompat(enc); + + if (singlebyte) { + while (s < send) { + unsigned int c = *s++; + if (c != save || (argc > 0 && !squeez[c])) { + *t++ = save = c; + } + } } - *t = '\0'; - if (t - RSTRING(str)->ptr != RSTRING(str)->len) { - RSTRING(str)->len = t - RSTRING(str)->ptr; - modify = 1; + else { + while (s < send) { + unsigned int c; + int clen; + + if (ascompat && (c = *s) < 0x80) { + if (c != save || (argc > 0 && !squeez[c])) { + *t++ = save = c; + } + s++; + } + else { + c = rb_enc_codepoint_len((char *)s, (char *)send, &clen, enc); + + if (c != save || (argc > 0 && !tr_find(c, squeez, del, nodel))) { + if (t != s) rb_enc_mbcput(c, t, enc); + save = c; + t += clen; + } + s += clen; + } + } + } + + TERM_FILL((char *)t, TERM_LEN(str)); + if ((char *)t - RSTRING_PTR(str) != RSTRING_LEN(str)) { + STR_SET_LEN(str, (char *)t - RSTRING_PTR(str)); + modify = 1; } if (modify) return str; @@ -3259,26 +8890,16 @@ rb_str_squeeze_bang(argc, argv, str) /* * call-seq: - * str.squeeze([other_str]*) => new_str - * - * Builds a set of characters from the <i>other_str</i> parameter(s) using the - * procedure described for <code>String#count</code>. Returns a new string - * where runs of the same character that occur in this set are replaced by a - * single character. If no arguments are given, all runs of identical - * characters are replaced by a single character. - * - * "yellow moon".squeeze #=> "yelow mon" - * " now is the".squeeze(" ") #=> " now is the" - * "putters shoot balls".squeeze("m-z") #=> "puters shot balls" + * squeeze(*selectors) -> new_string + * + * :include: doc/string/squeeze.rdoc + * */ static VALUE -rb_str_squeeze(argc, argv, str) - int argc; - VALUE *argv; - VALUE str; +rb_str_squeeze(int argc, VALUE *argv, VALUE str) { - str = rb_str_dup(str); + str = str_duplicate(rb_cString, str); rb_str_squeeze_bang(argc, argv, str); return str; } @@ -3286,15 +8907,18 @@ rb_str_squeeze(argc, argv, str) /* * call-seq: - * str.tr_s!(from_str, to_str) => str or nil - * - * Performs <code>String#tr_s</code> processing on <i>str</i> in place, - * returning <i>str</i>, or <code>nil</code> if no changes were made. + * tr_s!(selector, replacements) -> self or nil + * + * Like String#tr_s, except: + * + * - Modifies +self+ in place (not a copy of +self+). + * - Returns +self+ if any changes were made, +nil+ otherwise. + * + * Related: {Modifying}[rdoc-ref:String@Modifying]. */ static VALUE -rb_str_tr_s_bang(str, src, repl) - VALUE str, src, repl; +rb_str_tr_s_bang(VALUE str, VALUE src, VALUE repl) { return tr_trans(str, src, repl, 1); } @@ -3302,22 +8926,28 @@ rb_str_tr_s_bang(str, src, repl) /* * call-seq: - * str.tr_s(from_str, to_str) => new_str - * - * Processes a copy of <i>str</i> as described under <code>String#tr</code>, - * then removes duplicate characters in regions that were affected by the - * translation. - * - * "hello".tr_s('l', 'r') #=> "hero" - * "hello".tr_s('el', '*') #=> "h*o" - * "hello".tr_s('el', 'hx') #=> "hhxo" + * tr_s(selector, replacements) -> new_string + * + * Like String#tr, except: + * + * - Also squeezes the modified portions of the translated string; + * see String#squeeze. + * - Returns the translated and squeezed string. + * + * Examples: + * + * 'hello'.tr_s('l', 'r') #=> "hero" + * 'hello'.tr_s('el', '-') #=> "h-o" + * 'hello'.tr_s('el', 'hx') #=> "hhxo" + * + * Related: see {Converting to New String}[rdoc-ref:String@Converting+to+New+String]. + * */ static VALUE -rb_str_tr_s(str, src, repl) - VALUE str, src, repl; +rb_str_tr_s(VALUE str, VALUE src, VALUE repl) { - str = rb_str_dup(str); + str = str_duplicate(rb_cString, str); tr_trans(str, src, repl, 1); return str; } @@ -3325,562 +8955,1065 @@ rb_str_tr_s(str, src, repl) /* * call-seq: - * str.count([other_str]+) => fixnum - * - * Each <i>other_str</i> parameter defines a set of characters to count. The - * intersection of these sets defines the characters to count in - * <i>str</i>. Any <i>other_str</i> that starts with a caret (^) is - * negated. The sequence c1--c2 means all characters between c1 and c2. - * - * a = "hello world" - * a.count "lo" #=> 5 - * a.count "lo", "o" #=> 2 - * a.count "hello", "^l" #=> 4 - * a.count "ej-m" #=> 4 + * count(*selectors) -> integer + * + * :include: doc/string/count.rdoc */ static VALUE -rb_str_count(argc, argv, str) - int argc; - VALUE *argv; - VALUE str; +rb_str_count(int argc, VALUE *argv, VALUE str) { - char table[256]; + char table[TR_TABLE_SIZE]; + rb_encoding *enc = 0; + VALUE del = 0, nodel = 0, tstr; char *s, *send; - int init = 1; int i; - - if (argc < 1) { - rb_raise(rb_eArgError, "wrong number of arguments"); + int ascompat; + size_t n = 0; + + rb_check_arity(argc, 1, UNLIMITED_ARGUMENTS); + + tstr = argv[0]; + StringValue(tstr); + enc = rb_enc_check(str, tstr); + if (argc == 1) { + const char *ptstr; + if (RSTRING_LEN(tstr) == 1 && rb_enc_asciicompat(enc) && + (ptstr = RSTRING_PTR(tstr), + ONIGENC_IS_ALLOWED_REVERSE_MATCH(enc, (const unsigned char *)ptstr, (const unsigned char *)ptstr+1)) && + !is_broken_string(str)) { + int clen; + unsigned char c = rb_enc_codepoint_len(ptstr, ptstr+1, &clen, enc); + + s = RSTRING_PTR(str); + if (!s || RSTRING_LEN(str) == 0) return INT2FIX(0); + send = RSTRING_END(str); + while (s < send) { + if (*(unsigned char*)s++ == c) n++; + } + return SIZET2NUM(n); + } } - for (i=0; i<argc; i++) { - VALUE s = argv[i]; - StringValue(s); - tr_setup_table(s, table, init); - init = 0; + tr_setup_table(tstr, table, TRUE, &del, &nodel, enc); + for (i=1; i<argc; i++) { + tstr = argv[i]; + StringValue(tstr); + enc = rb_enc_check(str, tstr); + tr_setup_table(tstr, table, FALSE, &del, &nodel, enc); } - s = RSTRING(str)->ptr; - if (!s || RSTRING(str)->len == 0) return INT2FIX(0); - send = s + RSTRING(str)->len; - i = 0; + s = RSTRING_PTR(str); + if (!s || RSTRING_LEN(str) == 0) return INT2FIX(0); + send = RSTRING_END(str); + ascompat = rb_enc_asciicompat(enc); while (s < send) { - if (table[*s++ & 0xff]) { - i++; - } - } - return INT2NUM(i); -} - - -/* - * call-seq: - * str.split(pattern=$;, [limit]) => anArray - * - * Divides <i>str</i> into substrings based on a delimiter, returning an array - * of these substrings. - * - * If <i>pattern</i> is a <code>String</code>, then its contents are used as - * the delimiter when splitting <i>str</i>. If <i>pattern</i> is a single - * space, <i>str</i> is split on whitespace, with leading whitespace and runs - * of contiguous whitespace characters ignored. - * - * If <i>pattern</i> is a <code>Regexp</code>, <i>str</i> is divided where the - * pattern matches. Whenever the pattern matches a zero-length string, - * <i>str</i> is split into individual characters. - * - * If <i>pattern</i> is omitted, the value of <code>$;</code> is used. If - * <code>$;</code> is <code>nil</code> (which is the default), <i>str</i> is - * split on whitespace as if ` ' were specified. - * - * If the <i>limit</i> parameter is omitted, trailing null fields are - * suppressed. If <i>limit</i> is a positive number, at most that number of - * fields will be returned (if <i>limit</i> is <code>1</code>, the entire - * string is returned as the only entry in an array). If negative, there is no - * limit to the number of fields returned, and trailing null fields are not - * suppressed. - * - * " now's the time".split #=> ["now's", "the", "time"] - * " now's the time".split(' ') #=> ["now's", "the", "time"] - * " now's the time".split(/ /) #=> ["", "now's", "", "the", "time"] - * "1, 2.34,56, 7".split(%r{,\s*}) #=> ["1", "2.34", "56", "7"] - * "hello".split(//) #=> ["h", "e", "l", "l", "o"] - * "hello".split(//, 3) #=> ["h", "e", "llo"] - * "hi mom".split(%r{\s*}) #=> ["h", "i", "m", "o", "m"] - * - * "mellow yellow".split("ello") #=> ["m", "w y", "w"] - * "1,2,,3,4,,".split(',') #=> ["1", "2", "", "3", "4"] - * "1,2,,3,4,,".split(',', 4) #=> ["1", "2", "", "3,4,,"] - * "1,2,,3,4,,".split(',', -4) #=> ["1", "2", "", "3", "4", "", ""] - */ - -static VALUE -rb_str_split_m(argc, argv, str) - int argc; - VALUE *argv; - VALUE str; + unsigned int c; + + if (ascompat && (c = *(unsigned char*)s) < 0x80) { + if (table[c]) { + n++; + } + s++; + } + else { + int clen; + c = rb_enc_codepoint_len(s, send, &clen, enc); + if (tr_find(c, table, del, nodel)) { + n++; + } + s += clen; + } + } + + return SIZET2NUM(n); +} + +static VALUE +rb_fs_check(VALUE val) +{ + if (!NIL_P(val) && !RB_TYPE_P(val, T_STRING) && !RB_TYPE_P(val, T_REGEXP)) { + val = rb_check_string_type(val); + if (NIL_P(val)) return 0; + } + return val; +} + +static const char isspacetable[256] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +}; + +#define ascii_isspace(c) isspacetable[(unsigned char)(c)] + +static long +split_string(VALUE result, VALUE str, long beg, long len, long empty_count) +{ + if (empty_count >= 0 && len == 0) { + return empty_count + 1; + } + if (empty_count > 0) { + /* make different substrings */ + if (result) { + do { + rb_ary_push(result, str_new_empty_String(str)); + } while (--empty_count > 0); + } + else { + do { + rb_yield(str_new_empty_String(str)); + } while (--empty_count > 0); + } + } + str = rb_str_subseq(str, beg, len); + if (result) { + rb_ary_push(result, str); + } + else { + rb_yield(str); + } + return empty_count; +} + +typedef enum { + SPLIT_TYPE_AWK, SPLIT_TYPE_STRING, SPLIT_TYPE_REGEXP, SPLIT_TYPE_CHARS +} split_type_t; + +static split_type_t +literal_split_pattern(VALUE spat, split_type_t default_type) +{ + rb_encoding *enc = STR_ENC_GET(spat); + const char *ptr; + long len; + RSTRING_GETMEM(spat, ptr, len); + if (len == 0) { + /* Special case - split into chars */ + return SPLIT_TYPE_CHARS; + } + else if (rb_enc_asciicompat(enc)) { + if (len == 1 && ptr[0] == ' ') { + return SPLIT_TYPE_AWK; + } + } + else { + int l; + if (rb_enc_ascget(ptr, ptr + len, &l, enc) == ' ' && len == l) { + return SPLIT_TYPE_AWK; + } + } + return default_type; +} + +/* + * call-seq: + * split(field_sep = $;, limit = 0) -> array_of_substrings + * split(field_sep = $;, limit = 0) {|substring| ... } -> self + * + * :include: doc/string/split.rdoc + * + */ + +static VALUE +rb_str_split_m(int argc, VALUE *argv, VALUE str) { + rb_encoding *enc; VALUE spat; VALUE limit; - int awk_split = Qfalse; - long beg, end, i = 0; + split_type_t split_type; + long beg, end, i = 0, empty_count = -1; int lim = 0; VALUE result, tmp; + result = rb_block_given_p() ? Qfalse : Qnil; if (rb_scan_args(argc, argv, "02", &spat, &limit) == 2) { - lim = NUM2INT(limit); - if (lim <= 0) limit = Qnil; - else if (lim == 1) { - if (RSTRING(str)->len == 0) - return rb_ary_new2(0); - return rb_ary_new3(1, str); - } - i = 1; - } - - if (NIL_P(spat)) { - if (!NIL_P(rb_fs)) { - spat = rb_fs; - goto fs_set; - } - awk_split = Qtrue; + lim = NUM2INT(limit); + if (lim <= 0) limit = Qnil; + else if (lim == 1) { + if (RSTRING_LEN(str) == 0) + return result ? rb_ary_new2(0) : str; + tmp = str_duplicate(rb_cString, str); + if (!result) { + rb_yield(tmp); + return str; + } + return rb_ary_new3(1, tmp); + } + i = 1; + } + if (NIL_P(limit) && !lim) empty_count = 0; + + enc = STR_ENC_GET(str); + split_type = SPLIT_TYPE_REGEXP; + if (!NIL_P(spat)) { + spat = get_pat_quoted(spat, 0); + } + else if (NIL_P(spat = rb_fs)) { + split_type = SPLIT_TYPE_AWK; + } + else if (!(spat = rb_fs_check(spat))) { + rb_raise(rb_eTypeError, "value of $; must be String or Regexp"); } else { - fs_set: - if (TYPE(spat) == T_STRING && RSTRING(spat)->len == 1) { - if (RSTRING(spat)->ptr[0] == ' ') { - awk_split = Qtrue; - } - else { - spat = rb_reg_regcomp(rb_reg_quote(spat)); - } - } - else { - spat = get_pat(spat, 1); - } - } - - result = rb_ary_new(); + rb_category_warn(RB_WARN_CATEGORY_DEPRECATED, "$; is set to non-nil value"); + } + if (split_type != SPLIT_TYPE_AWK) { + switch (BUILTIN_TYPE(spat)) { + case T_REGEXP: + rb_reg_options(spat); /* check if uninitialized */ + tmp = RREGEXP_SRC(spat); + split_type = literal_split_pattern(tmp, SPLIT_TYPE_REGEXP); + if (split_type == SPLIT_TYPE_AWK) { + spat = tmp; + split_type = SPLIT_TYPE_STRING; + } + break; + + case T_STRING: + mustnot_broken(spat); + split_type = literal_split_pattern(spat, SPLIT_TYPE_STRING); + break; + + default: + UNREACHABLE_RETURN(Qnil); + } + } + +#define SPLIT_STR(beg, len) ( \ + empty_count = split_string(result, str, beg, len, empty_count), \ + str_mod_check(str, str_start, str_len)) + beg = 0; - if (awk_split) { - char *ptr = RSTRING(str)->ptr; - long len = RSTRING(str)->len; - char *eptr = ptr + len; - int skip = 1; - - for (end = beg = 0; ptr<eptr; ptr++) { - if (skip) { - if (ISSPACE(*ptr)) { - beg++; - } - else { - end = beg+1; - skip = 0; - if (!NIL_P(limit) && lim <= i) break; - } - } - else { - if (ISSPACE(*ptr)) { - rb_ary_push(result, rb_str_substr(str, beg, end-beg)); - skip = 1; - beg = end + 1; - if (!NIL_P(limit)) ++i; - } - else { - end++; - } - } - } + char *ptr = RSTRING_PTR(str); + char *const str_start = ptr; + const long str_len = RSTRING_LEN(str); + char *const eptr = str_start + str_len; + if (split_type == SPLIT_TYPE_AWK) { + char *bptr = ptr; + int skip = 1; + unsigned int c; + + if (result) result = rb_ary_new(); + end = beg; + if (is_ascii_string(str)) { + while (ptr < eptr) { + c = (unsigned char)*ptr++; + if (skip) { + if (ascii_isspace(c)) { + beg = ptr - bptr; + } + else { + end = ptr - bptr; + skip = 0; + if (!NIL_P(limit) && lim <= i) break; + } + } + else if (ascii_isspace(c)) { + SPLIT_STR(beg, end-beg); + skip = 1; + beg = ptr - bptr; + if (!NIL_P(limit)) ++i; + } + else { + end = ptr - bptr; + } + } + } + else { + while (ptr < eptr) { + int n; + + c = rb_enc_codepoint_len(ptr, eptr, &n, enc); + ptr += n; + if (skip) { + if (rb_isspace(c)) { + beg = ptr - bptr; + } + else { + end = ptr - bptr; + skip = 0; + if (!NIL_P(limit) && lim <= i) break; + } + } + else if (rb_isspace(c)) { + SPLIT_STR(beg, end-beg); + skip = 1; + beg = ptr - bptr; + if (!NIL_P(limit)) ++i; + } + else { + end = ptr - bptr; + } + } + } + } + else if (split_type == SPLIT_TYPE_STRING) { + char *substr_start = ptr; + char *sptr = RSTRING_PTR(spat); + long slen = RSTRING_LEN(spat); + + if (result) result = rb_ary_new(); + mustnot_broken(str); + enc = rb_enc_check(str, spat); + while (ptr < eptr && + (end = rb_memsearch(sptr, slen, ptr, eptr - ptr, enc)) >= 0) { + /* Check we are at the start of a char */ + char *t = rb_enc_right_char_head(ptr, ptr + end, eptr, enc); + if (t != ptr + end) { + ptr = t; + continue; + } + SPLIT_STR(substr_start - str_start, (ptr+end) - substr_start); + str_mod_check(spat, sptr, slen); + ptr += end + slen; + substr_start = ptr; + if (!NIL_P(limit) && lim <= ++i) break; + } + beg = ptr - str_start; + } + else if (split_type == SPLIT_TYPE_CHARS) { + int n; + + if (result) result = rb_ary_new_capa(RSTRING_LEN(str)); + mustnot_broken(str); + enc = rb_enc_get(str); + while (ptr < eptr && + (n = rb_enc_precise_mbclen(ptr, eptr, enc)) > 0) { + SPLIT_STR(ptr - str_start, n); + ptr += n; + if (!NIL_P(limit) && lim <= ++i) break; + } + beg = ptr - str_start; } else { - long start = beg; - long idx; - int last_null = 0; - struct re_registers *regs; - - while ((end = rb_reg_search(spat, str, start, 0)) >= 0) { - regs = RMATCH(rb_backref_get())->regs; - if (start == end && BEG(0) == END(0)) { - if (!RSTRING(str)->ptr) { - rb_ary_push(result, rb_str_new("", 0)); - break; + if (result) result = rb_ary_new(); + long len = RSTRING_LEN(str); + long start = beg; + long idx; + int last_null = 0; + struct re_registers *regs; + VALUE match = 0; + + for (; rb_reg_search(spat, str, start, 0) >= 0; + (match ? (rb_match_unbusy(match), rb_backref_set(match)) : (void)0)) { + match = rb_backref_get(); + if (!result) rb_match_busy(match); + regs = RMATCH_REGS(match); + end = BEG(0); + if (start == end && BEG(0) == END(0)) { + if (!ptr) { + SPLIT_STR(0, 0); + break; } - else if (last_null == 1) { - rb_ary_push(result, rb_str_substr(str, beg, mbclen2(RSTRING(str)->ptr[beg],spat))); - beg = start; - } - else { - start += mbclen2(RSTRING(str)->ptr[start],spat); - last_null = 1; - continue; - } - } - else { - rb_ary_push(result, rb_str_substr(str, beg, end-beg)); - beg = start = END(0); - } - last_null = 0; - - for (idx=1; idx < regs->num_regs; idx++) { - if (BEG(idx) == -1) continue; - if (BEG(idx) == END(idx)) - tmp = rb_str_new5(str, 0, 0); - else - tmp = rb_str_substr(str, BEG(idx), END(idx)-BEG(idx)); - rb_ary_push(result, tmp); - } - if (!NIL_P(limit) && lim <= ++i) break; - } - } - if (RSTRING(str)->len > 0 && (!NIL_P(limit) || RSTRING(str)->len > beg || lim < 0)) { - if (RSTRING(str)->len == beg) - tmp = rb_str_new5(str, 0, 0); - else - tmp = rb_str_substr(str, beg, RSTRING(str)->len-beg); - rb_ary_push(result, tmp); - } - if (NIL_P(limit) && lim == 0) { - while (RARRAY(result)->len > 0 && - RSTRING(RARRAY(result)->ptr[RARRAY(result)->len-1])->len == 0) - rb_ary_pop(result); + else if (last_null == 1) { + SPLIT_STR(beg, rb_enc_fast_mbclen(ptr+beg, eptr, enc)); + beg = start; + } + else { + if (start == len) + start++; + else + start += rb_enc_fast_mbclen(ptr+start,eptr,enc); + last_null = 1; + continue; + } + } + else { + SPLIT_STR(beg, end-beg); + beg = start = END(0); + } + last_null = 0; + + for (idx=1; idx < regs->num_regs; idx++) { + if (BEG(idx) == -1) continue; + SPLIT_STR(BEG(idx), END(idx)-BEG(idx)); + } + if (!NIL_P(limit) && lim <= ++i) break; + } + if (match) rb_match_unbusy(match); + } + if (RSTRING_LEN(str) > 0 && (!NIL_P(limit) || RSTRING_LEN(str) > beg || lim < 0)) { + SPLIT_STR(beg, RSTRING_LEN(str)-beg); } - return result; + return result ? result : str; } VALUE -rb_str_split(str, sep0) - VALUE str; - const char *sep0; +rb_str_split(VALUE str, const char *sep0) { VALUE sep; StringValue(str); - sep = rb_str_new2(sep0); + sep = rb_str_new_cstr(sep0); return rb_str_split_m(1, &sep, str); } -/* - * call-seq: - * split([pattern [, limit]]) => array - * - * Equivalent to <code>$_.split(<i>pattern</i>, <i>limit</i>)</code>. - * See <code>String#split</code>. - */ +#define WANTARRAY(m, size) (!rb_block_given_p() ? rb_ary_new_capa(size) : 0) -static VALUE -rb_f_split(argc, argv) - int argc; - VALUE *argv; +static inline int +enumerator_element(VALUE ary, VALUE e) { - return rb_str_split_m(argc, argv, uscore_get()); + if (ary) { + rb_ary_push(ary, e); + return 0; + } + else { + rb_yield(e); + return 1; + } } +#define ENUM_ELEM(ary, e) enumerator_element(ary, e) -/* - * call-seq: - * str.each(separator=$/) {|substr| block } => str - * str.each_line(separator=$/) {|substr| block } => str - * - * Splits <i>str</i> using the supplied parameter as the record separator - * (<code>$/</code> by default), passing each substring in turn to the supplied - * block. If a zero-length record separator is supplied, the string is split on - * <code>\n</code> characters, except that multiple successive newlines are - * appended together. - * - * print "Example one\n" - * "hello\nworld".each {|s| p s} - * print "Example two\n" - * "hello\nworld".each('l') {|s| p s} - * print "Example three\n" - * "hello\n\n\nworld".each('') {|s| p s} - * - * <em>produces:</em> - * - * Example one - * "hello\n" - * "world" - * Example two - * "hel" - * "l" - * "o\nworl" - * "d" - * Example three - * "hello\n\n\n" - * "world" - */ +static const char * +chomp_newline(const char *p, const char *e, rb_encoding *enc) +{ + const char *prev = rb_enc_prev_char(p, e, e, enc); + if (rb_enc_is_newline(prev, e, enc)) { + e = prev; + prev = rb_enc_prev_char(p, e, e, enc); + if (prev && rb_enc_ascget(prev, e, NULL, enc) == '\r') + e = prev; + } + return e; +} static VALUE -rb_str_each_line(argc, argv, str) - int argc; - VALUE *argv; - VALUE str; +get_rs(void) { - VALUE rs; - int newline; - char *p = RSTRING(str)->ptr, *pend = p + RSTRING(str)->len, *s; - char *ptr = p; - long len = RSTRING(str)->len, rslen; - VALUE line; + VALUE rs = rb_rs; + if (!NIL_P(rs) && + (!RB_TYPE_P(rs, T_STRING) || + RSTRING_LEN(rs) != 1 || + RSTRING_PTR(rs)[0] != '\n')) { + rb_category_warn(RB_WARN_CATEGORY_DEPRECATED, "$/ is set to non-default value"); + } + return rs; +} + +#define rb_rs get_rs() - if (rb_scan_args(argc, argv, "01", &rs) == 0) { - rs = rb_rs; +static VALUE +rb_str_enumerate_lines(int argc, VALUE *argv, VALUE str, VALUE ary) +{ + rb_encoding *enc; + VALUE line, rs, orig = str, opts = Qnil, chomp = Qfalse; + const char *ptr, *pend, *subptr, *subend, *rsptr, *hit, *adjusted; + long pos, len, rslen; + int rsnewline = 0; + + if (rb_scan_args(argc, argv, "01:", &rs, &opts) == 0) + rs = rb_rs; + if (!NIL_P(opts)) { + static ID keywords[1]; + if (!keywords[0]) { + keywords[0] = rb_intern_const("chomp"); + } + rb_get_kwargs(opts, keywords, 0, 1, &chomp); + chomp = (!UNDEF_P(chomp) && RTEST(chomp)); } if (NIL_P(rs)) { - rb_yield(str); - return str; + if (!ENUM_ELEM(ary, str)) { + return ary; + } + else { + return orig; + } } + + if (!RSTRING_LEN(str)) goto end; + str = rb_str_new_frozen(str); + ptr = subptr = RSTRING_PTR(str); + pend = RSTRING_END(str); + len = RSTRING_LEN(str); StringValue(rs); - rslen = RSTRING(rs)->len; + rslen = RSTRING_LEN(rs); + + if (rs == rb_default_rs) + enc = rb_enc_get(str); + else + enc = rb_enc_check(str, rs); + if (rslen == 0) { - newline = '\n'; + /* paragraph mode */ + int n; + const char *eol = NULL; + subend = subptr; + while (subend < pend) { + long chomp_rslen = 0; + do { + if (rb_enc_ascget(subend, pend, &n, enc) != '\r') + n = 0; + rslen = n + rb_enc_mbclen(subend + n, pend, enc); + if (rb_enc_is_newline(subend + n, pend, enc)) { + if (eol == subend) break; + subend += rslen; + if (subptr) { + eol = subend; + chomp_rslen = -rslen; + } + } + else { + if (!subptr) subptr = subend; + subend += rslen; + } + rslen = 0; + } while (subend < pend); + if (!subptr) break; + if (rslen == 0) chomp_rslen = 0; + line = rb_str_subseq(str, subptr - ptr, + subend - subptr + (chomp ? chomp_rslen : rslen)); + if (ENUM_ELEM(ary, line)) { + str_mod_check(str, ptr, len); + } + subptr = eol = NULL; + } + goto end; } else { - newline = RSTRING(rs)->ptr[rslen-1]; + rsptr = RSTRING_PTR(rs); + if (RSTRING_LEN(rs) == rb_enc_mbminlen(enc) && + rb_enc_is_newline(rsptr, rsptr + RSTRING_LEN(rs), enc)) { + rsnewline = 1; + } + } + + if ((rs == rb_default_rs) && !rb_enc_asciicompat(enc)) { + rs = rb_str_new(rsptr, rslen); + rs = rb_str_encode(rs, rb_enc_from_encoding(enc), 0, Qnil); + rsptr = RSTRING_PTR(rs); + rslen = RSTRING_LEN(rs); } - for (s = p, p += rslen; p < pend; p++) { - if (rslen == 0 && *p == '\n') { - if (*++p != '\n') continue; - while (*p == '\n') p++; - } - if (RSTRING(str)->ptr < p && p[-1] == newline && - (rslen <= 1 || - rb_memcmp(RSTRING(rs)->ptr, p-rslen, rslen) == 0)) { - line = rb_str_new5(str, s, p - s); - OBJ_INFECT(line, str); - rb_yield(line); - if (RSTRING(str)->ptr != ptr || RSTRING(str)->len != len) - rb_raise(rb_eArgError, "string modified"); - s = p; - } + while (subptr < pend) { + pos = rb_memsearch(rsptr, rslen, subptr, pend - subptr, enc); + if (pos < 0) break; + hit = subptr + pos; + adjusted = rb_enc_right_char_head(subptr, hit, pend, enc); + if (hit != adjusted) { + subptr = adjusted; + continue; + } + subend = hit += rslen; + if (chomp) { + if (rsnewline) { + subend = chomp_newline(subptr, subend, enc); + } + else { + subend -= rslen; + } + } + line = rb_str_subseq(str, subptr - ptr, subend - subptr); + if (ENUM_ELEM(ary, line)) { + str_mod_check(str, ptr, len); + } + subptr = hit; } - if (s != pend) { - if (p > pend) p = pend; - line = rb_str_new5(str, s, p - s); - OBJ_INFECT(line, str); - rb_yield(line); + if (subptr != pend) { + if (chomp) { + if (rsnewline) { + pend = chomp_newline(subptr, pend, enc); + } + else if (pend - subptr >= rslen && + memcmp(pend - rslen, rsptr, rslen) == 0) { + pend -= rslen; + } + } + line = rb_str_subseq(str, subptr - ptr, pend - subptr); + ENUM_ELEM(ary, line); + RB_GC_GUARD(str); } - return str; + end: + if (ary) + return ary; + else + return orig; } +/* + * call-seq: + * each_line(record_separator = $/, chomp: false) {|substring| ... } -> self + * each_line(record_separator = $/, chomp: false) -> enumerator + * + * :include: doc/string/each_line.rdoc + * + */ + +static VALUE +rb_str_each_line(int argc, VALUE *argv, VALUE str) +{ + RETURN_SIZED_ENUMERATOR(str, argc, argv, 0); + return rb_str_enumerate_lines(argc, argv, str, 0); +} /* * call-seq: - * str.each_byte {|fixnum| block } => str - * - * Passes each byte in <i>str</i> to the given block. - * - * "hello".each_byte {|c| print c, ' ' } - * - * <em>produces:</em> - * - * 104 101 108 108 111 + * lines(record_separator = $/, chomp: false) -> array_of_strings + * + * Returns substrings ("lines") of +self+ + * according to the given arguments: + * + * s = <<~EOT + * This is the first line. + * This is line two. + * + * This is line four. + * This is line five. + * EOT + * + * With the default argument values: + * + * $/ # => "\n" + * s.lines + * # => + * ["This is the first line.\n", + * "This is line two.\n", + * "\n", + * "This is line four.\n", + * "This is line five.\n"] + * + * With a different +record_separator+: + * + * record_separator = ' is ' + * s.lines(record_separator) + * # => + * ["This is ", + * "the first line.\nThis is ", + * "line two.\n\nThis is ", + * "line four.\nThis is ", + * "line five.\n"] + * + * With keyword argument +chomp+ as +true+, + * removes the trailing newline from each line: + * + * s.lines(chomp: true) + * # => + * ["This is the first line.", + * "This is line two.", + * "", + * "This is line four.", + * "This is line five."] + * + * Related: see {Converting to Non-String}[rdoc-ref:String@Converting+to+Non--5CString]. */ static VALUE -rb_str_each_byte(str) - VALUE str; +rb_str_lines(int argc, VALUE *argv, VALUE str) +{ + VALUE ary = WANTARRAY("lines", 0); + return rb_str_enumerate_lines(argc, argv, str, ary); +} + +static VALUE +rb_str_each_byte_size(VALUE str, VALUE args, VALUE eobj) +{ + return LONG2FIX(RSTRING_LEN(str)); +} + +static VALUE +rb_str_enumerate_bytes(VALUE str, VALUE ary) { long i; - for (i=0; i<RSTRING(str)->len; i++) { - rb_yield(INT2FIX(RSTRING(str)->ptr[i] & 0xff)); + for (i=0; i<RSTRING_LEN(str); i++) { + ENUM_ELEM(ary, INT2FIX((unsigned char)RSTRING_PTR(str)[i])); } - return str; + if (ary) + return ary; + else + return str; } +/* + * call-seq: + * each_byte {|byte| ... } -> self + * each_byte -> enumerator + * + * :include: doc/string/each_byte.rdoc + * + */ + +static VALUE +rb_str_each_byte(VALUE str) +{ + RETURN_SIZED_ENUMERATOR(str, 0, 0, rb_str_each_byte_size); + return rb_str_enumerate_bytes(str, 0); +} /* * call-seq: - * str.chop! => str or nil - * - * Processes <i>str</i> as for <code>String#chop</code>, returning <i>str</i>, - * or <code>nil</code> if <i>str</i> is the empty string. See also - * <code>String#chomp!</code>. + * bytes -> array_of_bytes + * + * :include: doc/string/bytes.rdoc + * */ static VALUE -rb_str_chop_bang(str) - VALUE str; +rb_str_bytes(VALUE str) +{ + VALUE ary = WANTARRAY("bytes", RSTRING_LEN(str)); + return rb_str_enumerate_bytes(str, ary); +} + +static VALUE +rb_str_each_char_size(VALUE str, VALUE args, VALUE eobj) +{ + return rb_str_length(str); +} + +static VALUE +rb_str_enumerate_chars(VALUE str, VALUE ary) { - if (RSTRING(str)->len > 0) { - rb_str_modify(str); - RSTRING(str)->len--; - if (RSTRING(str)->ptr[RSTRING(str)->len] == '\n') { - if (RSTRING(str)->len > 0 && - RSTRING(str)->ptr[RSTRING(str)->len-1] == '\r') { - RSTRING(str)->len--; - } - } - RSTRING(str)->ptr[RSTRING(str)->len] = '\0'; - return str; + VALUE orig = str; + long i, len, n; + const char *ptr; + rb_encoding *enc; + + str = rb_str_new_frozen(str); + ptr = RSTRING_PTR(str); + len = RSTRING_LEN(str); + enc = rb_enc_get(str); + + if (ENC_CODERANGE_CLEAN_P(ENC_CODERANGE(str))) { + for (i = 0; i < len; i += n) { + n = rb_enc_fast_mbclen(ptr + i, ptr + len, enc); + ENUM_ELEM(ary, rb_str_subseq(str, i, n)); + } } - return Qnil; + else { + for (i = 0; i < len; i += n) { + n = rb_enc_mbclen(ptr + i, ptr + len, enc); + ENUM_ELEM(ary, rb_str_subseq(str, i, n)); + } + } + RB_GC_GUARD(str); + if (ary) + return ary; + else + return orig; } +/* + * call-seq: + * each_char {|char| ... } -> self + * each_char -> enumerator + * + * :include: doc/string/each_char.rdoc + * + */ + +static VALUE +rb_str_each_char(VALUE str) +{ + RETURN_SIZED_ENUMERATOR(str, 0, 0, rb_str_each_char_size); + return rb_str_enumerate_chars(str, 0); +} /* * call-seq: - * str.chop => new_str - * - * Returns a new <code>String</code> with the last character removed. If the - * string ends with <code>\r\n</code>, both characters are removed. Applying - * <code>chop</code> to an empty string returns an empty - * string. <code>String#chomp</code> is often a safer alternative, as it leaves - * the string unchanged if it doesn't end in a record separator. - * - * "string\r\n".chop #=> "string" - * "string\n\r".chop #=> "string\n" - * "string\n".chop #=> "string" - * "string".chop #=> "strin" - * "x".chop.chop #=> "" + * chars -> array_of_characters + * + * :include: doc/string/chars.rdoc + * */ static VALUE -rb_str_chop(str) - VALUE str; +rb_str_chars(VALUE str) { - str = rb_str_dup(str); - rb_str_chop_bang(str); - return str; + VALUE ary = WANTARRAY("chars", rb_str_strlen(str)); + return rb_str_enumerate_chars(str, ary); } +static VALUE +rb_str_enumerate_codepoints(VALUE str, VALUE ary) +{ + VALUE orig = str; + int n; + unsigned int c; + const char *ptr, *end; + rb_encoding *enc; + + if (single_byte_optimizable(str)) + return rb_str_enumerate_bytes(str, ary); + + str = rb_str_new_frozen(str); + ptr = RSTRING_PTR(str); + end = RSTRING_END(str); + enc = STR_ENC_GET(str); + + while (ptr < end) { + c = rb_enc_codepoint_len(ptr, end, &n, enc); + ENUM_ELEM(ary, UINT2NUM(c)); + ptr += n; + } + RB_GC_GUARD(str); + if (ary) + return ary; + else + return orig; +} /* * call-seq: - * chop! => $_ or nil - * - * Equivalent to <code>$_.chop!</code>. - * - * a = "now\r\n" - * $_ = a - * chop! #=> "now" - * chop! #=> "no" - * chop! #=> "n" - * chop! #=> "" - * chop! #=> nil - * $_ #=> "" - * a #=> "" + * each_codepoint {|codepoint| ... } -> self + * each_codepoint -> enumerator + * + * :include: doc/string/each_codepoint.rdoc + * */ static VALUE -rb_f_chop_bang(str) - VALUE str; +rb_str_each_codepoint(VALUE str) { - return rb_str_chop_bang(uscore_get()); + RETURN_SIZED_ENUMERATOR(str, 0, 0, rb_str_each_char_size); + return rb_str_enumerate_codepoints(str, 0); } /* * call-seq: - * chop => string - * - * Equivalent to <code>($_.dup).chop!</code>, except <code>nil</code> - * is never returned. See <code>String#chop!</code>. - * - * a = "now\r\n" - * $_ = a - * chop #=> "now" - * $_ #=> "now" - * chop #=> "no" - * chop #=> "n" - * chop #=> "" - * chop #=> "" - * a #=> "now\r\n" + * codepoints -> array_of_integers + * + * :include: doc/string/codepoints.rdoc + * */ static VALUE -rb_f_chop() +rb_str_codepoints(VALUE str) +{ + VALUE ary = WANTARRAY("codepoints", rb_str_strlen(str)); + return rb_str_enumerate_codepoints(str, ary); +} + +static regex_t * +get_reg_grapheme_cluster(rb_encoding *enc) { - VALUE str = uscore_get(); + int encidx = rb_enc_to_index(enc); - if (RSTRING(str)->len > 0) { - str = rb_str_dup(str); - rb_str_chop_bang(str); - rb_lastline_set(str); + const OnigUChar source_ascii[] = "\\X"; + const OnigUChar *source = source_ascii; + size_t source_len = sizeof(source_ascii) - 1; + + switch (encidx) { +#define CHARS_16BE(x) (OnigUChar)((x)>>8), (OnigUChar)(x) +#define CHARS_16LE(x) (OnigUChar)(x), (OnigUChar)((x)>>8) +#define CHARS_32BE(x) CHARS_16BE((x)>>16), CHARS_16BE(x) +#define CHARS_32LE(x) CHARS_16LE(x), CHARS_16LE((x)>>16) +#define CASE_UTF(e) \ + case ENCINDEX_UTF_##e: { \ + static const OnigUChar source_UTF_##e[] = {CHARS_##e('\\'), CHARS_##e('X')}; \ + source = source_UTF_##e; \ + source_len = sizeof(source_UTF_##e); \ + break; \ + } + CASE_UTF(16BE); CASE_UTF(16LE); CASE_UTF(32BE); CASE_UTF(32LE); +#undef CASE_UTF +#undef CHARS_16BE +#undef CHARS_16LE +#undef CHARS_32BE +#undef CHARS_32LE } - return str; + + regex_t *reg_grapheme_cluster; + OnigErrorInfo einfo; + int r = onig_new(®_grapheme_cluster, source, source + source_len, + ONIG_OPTION_DEFAULT, enc, OnigDefaultSyntax, &einfo); + if (r) { + UChar message[ONIG_MAX_ERROR_MESSAGE_LEN]; + onig_error_code_to_str(message, r, &einfo); + rb_fatal("cannot compile grapheme cluster regexp: %s", (char *)message); + } + + return reg_grapheme_cluster; } +static regex_t * +get_cached_reg_grapheme_cluster(rb_encoding *enc) +{ + int encidx = rb_enc_to_index(enc); + static regex_t *reg_grapheme_cluster_utf8 = NULL; + + if (encidx == rb_utf8_encindex()) { + if (!reg_grapheme_cluster_utf8) { + reg_grapheme_cluster_utf8 = get_reg_grapheme_cluster(enc); + } + + return reg_grapheme_cluster_utf8; + } + + return NULL; +} + +static VALUE +rb_str_each_grapheme_cluster_size(VALUE str, VALUE args, VALUE eobj) +{ + size_t grapheme_cluster_count = 0; + rb_encoding *enc = get_encoding(str); + const char *ptr, *end; + + if (!rb_enc_unicode_p(enc)) { + return rb_str_length(str); + } + + bool cached_reg_grapheme_cluster = true; + regex_t *reg_grapheme_cluster = get_cached_reg_grapheme_cluster(enc); + if (!reg_grapheme_cluster) { + reg_grapheme_cluster = get_reg_grapheme_cluster(enc); + cached_reg_grapheme_cluster = false; + } + + ptr = RSTRING_PTR(str); + end = RSTRING_END(str); + + while (ptr < end) { + OnigPosition len = onig_match(reg_grapheme_cluster, + (const OnigUChar *)ptr, (const OnigUChar *)end, + (const OnigUChar *)ptr, NULL, 0); + if (len <= 0) break; + grapheme_cluster_count++; + ptr += len; + } + + if (!cached_reg_grapheme_cluster) { + onig_free(reg_grapheme_cluster); + } + + return SIZET2NUM(grapheme_cluster_count); +} + +static VALUE +rb_str_enumerate_grapheme_clusters(VALUE str, VALUE ary) +{ + VALUE orig = str; + rb_encoding *enc = get_encoding(str); + const char *ptr0, *ptr, *end; + + if (!rb_enc_unicode_p(enc)) { + return rb_str_enumerate_chars(str, ary); + } + + if (!ary) str = rb_str_new_frozen(str); + + bool cached_reg_grapheme_cluster = true; + regex_t *reg_grapheme_cluster = get_cached_reg_grapheme_cluster(enc); + if (!reg_grapheme_cluster) { + reg_grapheme_cluster = get_reg_grapheme_cluster(enc); + cached_reg_grapheme_cluster = false; + } + + ptr0 = ptr = RSTRING_PTR(str); + end = RSTRING_END(str); + + while (ptr < end) { + OnigPosition len = onig_match(reg_grapheme_cluster, + (const OnigUChar *)ptr, (const OnigUChar *)end, + (const OnigUChar *)ptr, NULL, 0); + if (len <= 0) break; + ENUM_ELEM(ary, rb_str_subseq(str, ptr-ptr0, len)); + ptr += len; + } + + if (!cached_reg_grapheme_cluster) { + onig_free(reg_grapheme_cluster); + } + + RB_GC_GUARD(str); + if (ary) + return ary; + else + return orig; +} /* * call-seq: - * str.chomp!(separator=$/) => str or nil - * - * Modifies <i>str</i> in place as described for <code>String#chomp</code>, - * returning <i>str</i>, or <code>nil</code> if no modifications were made. + * each_grapheme_cluster {|grapheme_cluster| ... } -> self + * each_grapheme_cluster -> enumerator + * + * :include: doc/string/each_grapheme_cluster.rdoc + * */ static VALUE -rb_str_chomp_bang(argc, argv, str) - int argc; - VALUE *argv; - VALUE str; +rb_str_each_grapheme_cluster(VALUE str) { - VALUE rs; - int newline; - char *p = RSTRING(str)->ptr; - long len = RSTRING(str)->len, rslen; - - if (rb_scan_args(argc, argv, "01", &rs) == 0) { - if (len == 0) return Qnil; - rs = rb_rs; - if (rs == rb_default_rs) { - smart_chomp: - if (RSTRING(str)->ptr[len-1] == '\n') { - rb_str_modify(str); - RSTRING(str)->len--; - if (RSTRING(str)->len > 0 && - RSTRING(str)->ptr[RSTRING(str)->len-1] == '\r') { - RSTRING(str)->len--; - } - } - else if (RSTRING(str)->ptr[len-1] == '\r') { - rb_str_modify(str); - RSTRING(str)->len--; - } - else { - return Qnil; - } - RSTRING(str)->ptr[RSTRING(str)->len] = '\0'; - return str; - } + RETURN_SIZED_ENUMERATOR(str, 0, 0, rb_str_each_grapheme_cluster_size); + return rb_str_enumerate_grapheme_clusters(str, 0); +} + +/* + * call-seq: + * grapheme_clusters -> array_of_grapheme_clusters + * + * :include: doc/string/grapheme_clusters.rdoc + * + */ + +static VALUE +rb_str_grapheme_clusters(VALUE str) +{ + VALUE ary = WANTARRAY("grapheme_clusters", rb_str_strlen(str)); + return rb_str_enumerate_grapheme_clusters(str, ary); +} + +static long +chopped_length(VALUE str) +{ + rb_encoding *enc = STR_ENC_GET(str); + const char *p, *p2, *beg, *end; + + beg = RSTRING_PTR(str); + end = beg + RSTRING_LEN(str); + if (beg >= end) return 0; + p = rb_enc_prev_char(beg, end, end, enc); + if (!p) return 0; + if (p > beg && rb_enc_ascget(p, end, 0, enc) == '\n') { + p2 = rb_enc_prev_char(beg, p, end, enc); + if (p2 && rb_enc_ascget(p2, end, 0, enc) == '\r') p = p2; } - if (NIL_P(rs)) return Qnil; - if (len == 0) return Qnil; + return p - beg; +} - StringValue(rs); - rslen = RSTRING(rs)->len; - if (rslen == 0) { - while (len>0 && p[len-1] == '\n') { - len--; - if (len>0 && p[len-1] == '\r') - len--; - } - if (len < RSTRING(str)->len) { - rb_str_modify(str); - RSTRING(str)->len = len; - RSTRING(str)->ptr[len] = '\0'; - return str; - } - return Qnil; - } - if (rslen > len) return Qnil; - newline = RSTRING(rs)->ptr[rslen-1]; - if (rslen == 1 && newline == '\n') - goto smart_chomp; +/* + * call-seq: + * chop! -> self or nil + * + * Like String#chop, except that: + * + * - Removes trailing characters from +self+ (not from a copy of +self+). + * - Returns +self+ if any characters are removed, +nil+ otherwise. + * + * Related: see {Modifying}[rdoc-ref:String@Modifying]. + */ - if (p[len-1] == newline && - (rslen <= 1 || - rb_memcmp(RSTRING(rs)->ptr, p+len-rslen, rslen) == 0)) { - rb_str_modify(str); - RSTRING(str)->len -= rslen; - RSTRING(str)->ptr[RSTRING(str)->len] = '\0'; - return str; +static VALUE +rb_str_chop_bang(VALUE str) +{ + str_modify_keep_cr(str); + if (RSTRING_LEN(str) > 0) { + long len; + len = chopped_length(str); + STR_SET_LEN(str, len); + TERM_FILL(&RSTRING_PTR(str)[len], TERM_LEN(str)); + if (ENC_CODERANGE(str) != ENC_CODERANGE_7BIT) { + ENC_CODERANGE_CLEAR(str); + } + return str; } return Qnil; } @@ -3888,119 +10021,294 @@ rb_str_chomp_bang(argc, argv, str) /* * call-seq: - * str.chomp(separator=$/) => new_str - * - * Returns a new <code>String</code> with the given record separator removed - * from the end of <i>str</i> (if present). If <code>$/</code> has not been - * changed from the default Ruby record separator, then <code>chomp</code> also - * removes carriage return characters (that is it will remove <code>\n</code>, - * <code>\r</code>, and <code>\r\n</code>). - * - * "hello".chomp #=> "hello" - * "hello\n".chomp #=> "hello" - * "hello\r\n".chomp #=> "hello" - * "hello\n\r".chomp #=> "hello\n" - * "hello\r".chomp #=> "hello" - * "hello \n there".chomp #=> "hello \n there" - * "hello".chomp("llo") #=> "he" + * chop -> new_string + * + * :include: doc/string/chop.rdoc + * */ static VALUE -rb_str_chomp(argc, argv, str) - int argc; - VALUE *argv; - VALUE str; +rb_str_chop(VALUE str) +{ + return rb_str_subseq(str, 0, chopped_length(str)); +} + +static long +smart_chomp(VALUE str, const char *e, const char *p) +{ + rb_encoding *enc = rb_enc_get(str); + if (rb_enc_mbminlen(enc) > 1) { + const char *pp = rb_enc_left_char_head(p, e-rb_enc_mbminlen(enc), e, enc); + if (rb_enc_is_newline(pp, e, enc)) { + e = pp; + } + pp = e - rb_enc_mbminlen(enc); + if (pp >= p) { + pp = rb_enc_left_char_head(p, pp, e, enc); + if (rb_enc_ascget(pp, e, 0, enc) == '\r') { + e = pp; + } + } + } + else { + switch (*(e-1)) { /* not e[-1] to get rid of VC bug */ + case '\n': + if (--e > p && *(e-1) == '\r') { + --e; + } + break; + case '\r': + --e; + break; + } + } + return e - p; +} + +static long +chompped_length(VALUE str, VALUE rs) +{ + rb_encoding *enc; + int newline; + char *pp, *e, *rsptr; + long rslen; + char *const p = RSTRING_PTR(str); + long len = RSTRING_LEN(str); + + if (len == 0) return 0; + e = p + len; + if (rs == rb_default_rs) { + return smart_chomp(str, e, p); + } + + enc = rb_enc_get(str); + RSTRING_GETMEM(rs, rsptr, rslen); + if (rslen == 0) { + if (rb_enc_mbminlen(enc) > 1) { + while (e > p) { + pp = rb_enc_left_char_head(p, e-rb_enc_mbminlen(enc), e, enc); + if (!rb_enc_is_newline(pp, e, enc)) break; + e = pp; + pp -= rb_enc_mbminlen(enc); + if (pp >= p) { + pp = rb_enc_left_char_head(p, pp, e, enc); + if (rb_enc_ascget(pp, e, 0, enc) == '\r') { + e = pp; + } + } + } + } + else { + while (e > p && *(e-1) == '\n') { + --e; + if (e > p && *(e-1) == '\r') + --e; + } + } + return e - p; + } + if (rslen > len) return len; + + enc = rb_enc_get(rs); + newline = rsptr[rslen-1]; + if (rslen == rb_enc_mbminlen(enc)) { + if (rslen == 1) { + if (newline == '\n') + return smart_chomp(str, e, p); + } + else { + if (rb_enc_is_newline(rsptr, rsptr+rslen, enc)) + return smart_chomp(str, e, p); + } + } + + enc = rb_enc_check(str, rs); + if (is_broken_string(rs)) { + return len; + } + pp = e - rslen; + if (p[len-1] == newline && + (rslen <= 1 || + memcmp(rsptr, pp, rslen) == 0)) { + if (at_char_boundary(p, pp, e, enc)) + return len - rslen; + RB_GC_GUARD(rs); + } + return len; +} + +/*! + * Returns the separator for arguments of rb_str_chomp. + * + * @return returns rb_rs ($/) as default, the default value of rb_rs ($/) is "\n". + */ +static VALUE +chomp_rs(int argc, const VALUE *argv) { - str = rb_str_dup(str); - rb_str_chomp_bang(argc, argv, str); + rb_check_arity(argc, 0, 1); + if (argc > 0) { + VALUE rs = argv[0]; + if (!NIL_P(rs)) StringValue(rs); + return rs; + } + else { + return rb_rs; + } +} + +VALUE +rb_str_chomp_string(VALUE str, VALUE rs) +{ + long olen = RSTRING_LEN(str); + long len = chompped_length(str, rs); + if (len >= olen) return Qnil; + str_modify_keep_cr(str); + STR_SET_LEN(str, len); + TERM_FILL(&RSTRING_PTR(str)[len], TERM_LEN(str)); + if (ENC_CODERANGE(str) != ENC_CODERANGE_7BIT) { + ENC_CODERANGE_CLEAR(str); + } return str; } /* * call-seq: - * chomp! => $_ or nil - * chomp!(string) => $_ or nil - * - * Equivalent to <code>$_.chomp!(<em>string</em>)</code>. See - * <code>String#chomp!</code> - * - * $_ = "now\n" - * chomp! #=> "now" - * $_ #=> "now" - * chomp! "x" #=> nil - * $_ #=> "now" + * chomp!(line_sep = $/) -> self or nil + * + * Like String#chomp, except that: + * + * - Removes trailing characters from +self+ (not from a copy of +self+). + * - Returns +self+ if any characters are removed, +nil+ otherwise. + * + * Related: see {Modifying}[rdoc-ref:String@Modifying]. */ static VALUE -rb_f_chomp_bang(argc, argv) - int argc; - VALUE *argv; +rb_str_chomp_bang(int argc, VALUE *argv, VALUE str) { - return rb_str_chomp_bang(argc, argv, uscore_get()); + VALUE rs; + str_modifiable(str); + if (RSTRING_LEN(str) == 0 && argc < 2) return Qnil; + rs = chomp_rs(argc, argv); + if (NIL_P(rs)) return Qnil; + return rb_str_chomp_string(str, rs); } + /* * call-seq: - * chomp => $_ - * chomp(string) => $_ - * - * Equivalent to <code>$_ = $_.chomp(<em>string</em>)</code>. See - * <code>String#chomp</code>. - * - * $_ = "now\n" - * chomp #=> "now" - * $_ #=> "now" - * chomp "ow" #=> "n" - * $_ #=> "n" - * chomp "xxx" #=> "n" - * $_ #=> "n" + * chomp(line_sep = $/) -> new_string + * + * :include: doc/string/chomp.rdoc + * */ static VALUE -rb_f_chomp(argc, argv) - int argc; - VALUE *argv; +rb_str_chomp(int argc, VALUE *argv, VALUE str) { - VALUE str = uscore_get(); - VALUE dup = rb_str_dup(str); + VALUE rs = chomp_rs(argc, argv); + if (NIL_P(rs)) return str_duplicate(rb_cString, str); + return rb_str_subseq(str, 0, chompped_length(str, rs)); +} - if (NIL_P(rb_str_chomp_bang(argc, argv, dup))) - return str; - rb_lastline_set(dup); - return dup; +static void +tr_setup_table_multi(char table[TR_TABLE_SIZE], VALUE *tablep, VALUE *ctablep, + VALUE str, int num_selectors, VALUE *selectors) +{ + int i; + + for (i=0; i<num_selectors; i++) { + VALUE selector = selectors[i]; + rb_encoding *enc; + + StringValue(selector); + enc = rb_enc_check(str, selector); + tr_setup_table(selector, table, i==0, tablep, ctablep, enc); + } +} + +static long +lstrip_offset(VALUE str, const char *s, const char *e, rb_encoding *enc) +{ + const char *const start = s; + + if (!s || s >= e) return 0; + + /* remove spaces at head */ + if (single_byte_optimizable(str)) { + while (s < e && (*s == '\0' || ascii_isspace(*s))) s++; + } + else { + while (s < e) { + int n; + unsigned int cc = rb_enc_codepoint_len(s, e, &n, enc); + + if (cc && !rb_isspace(cc)) break; + s += n; + } + } + return s - start; } +static long +lstrip_offset_table(VALUE str, const char *s, const char *e, rb_encoding *enc, + char table[TR_TABLE_SIZE], VALUE del, VALUE nodel) +{ + const char *const start = s; + + if (!s || s >= e) return 0; + + /* remove leading characters in the table */ + while (s < e) { + int n; + unsigned int cc = rb_enc_codepoint_len(s, e, &n, enc); + + if (!tr_find(cc, table, del, nodel)) break; + s += n; + } + return s - start; +} /* * call-seq: - * str.lstrip! => self or nil - * - * Removes leading whitespace from <i>str</i>, returning <code>nil</code> if no - * change was made. See also <code>String#rstrip!</code> and - * <code>String#strip!</code>. - * - * " hello ".lstrip #=> "hello " - * "hello".lstrip! #=> nil + * lstrip!(*selectors) -> self or nil + * + * Like String#lstrip, except that: + * + * - Performs stripping in +self+ (not in a copy of +self+). + * - Returns +self+ if any characters are stripped, +nil+ otherwise. + * + * Related: see {Modifying}[rdoc-ref:String@Modifying]. */ static VALUE -rb_str_lstrip_bang(str) - VALUE str; +rb_str_lstrip_bang(int argc, VALUE *argv, VALUE str) { - char *s, *t, *e; + rb_encoding *enc; + char *start, *s; + long olen, loffset; - s = RSTRING(str)->ptr; - if (!s || RSTRING(str)->len == 0) return Qnil; - e = t = s + RSTRING(str)->len; - /* remove spaces at head */ - while (s < t && ISSPACE(*s)) s++; + str_modify_keep_cr(str); + enc = STR_ENC_GET(str); + RSTRING_GETMEM(str, start, olen); + if (argc > 0) { + char table[TR_TABLE_SIZE]; + VALUE del = 0, nodel = 0; + + tr_setup_table_multi(table, &del, &nodel, str, argc, argv); + loffset = lstrip_offset_table(str, start, start+olen, enc, table, del, nodel); + } + else { + loffset = lstrip_offset(str, start, start+olen, enc); + } - if (s > RSTRING(str)->ptr) { - rb_str_modify(str); - RSTRING(str)->len = t-s; - memmove(RSTRING(str)->ptr, s, RSTRING(str)->len); - RSTRING(str)->ptr[RSTRING(str)->len] = '\0'; - return str; + if (loffset > 0) { + long len = olen-loffset; + s = start + loffset; + memmove(start, s, len); + STR_SET_LEN(str, len); + TERM_FILL(start+len, rb_enc_mbminlen(enc)); + return str; } return Qnil; } @@ -4008,58 +10316,143 @@ rb_str_lstrip_bang(str) /* * call-seq: - * str.lstrip => new_str - * - * Returns a copy of <i>str</i> with leading whitespace removed. See also - * <code>String#rstrip</code> and <code>String#strip</code>. - * - * " hello ".lstrip #=> "hello " - * "hello".lstrip #=> "hello" + * lstrip(*selectors) -> new_string + * + * Returns a copy of +self+ with leading whitespace removed; + * see {Whitespace in Strings}[rdoc-ref:String@Whitespace+in+Strings]: + * + * whitespace = "\x00\t\n\v\f\r " + * s = whitespace + 'abc' + whitespace + * # => "\u0000\t\n\v\f\r abc\u0000\t\n\v\f\r " + * s.lstrip + * # => "abc\u0000\t\n\v\f\r " + * + * If +selectors+ are given, removes characters of +selectors+ from the beginning of +self+: + * + * s = "---abc+++" + * s.lstrip("-") # => "abc+++" + * + * +selectors+ must be valid character selectors (see {Character Selectors}[rdoc-ref:character_selectors.rdoc]), + * and may use any of its valid forms, including negation, ranges, and escapes: + * + * "01234abc56789".lstrip("0-9") # "abc56789" + * "01234abc56789".lstrip("0-9", "^4-6") # "4abc56789" + * + * Related: see {Converting to New String}[rdoc-ref:String@Converting+to+New+String]. */ static VALUE -rb_str_lstrip(str) - VALUE str; +rb_str_lstrip(int argc, VALUE *argv, VALUE str) { - str = rb_str_dup(str); - rb_str_lstrip_bang(str); - return str; + char *start; + long len, loffset; + + RSTRING_GETMEM(str, start, len); + if (argc > 0) { + char table[TR_TABLE_SIZE]; + VALUE del = 0, nodel = 0; + + tr_setup_table_multi(table, &del, &nodel, str, argc, argv); + loffset = lstrip_offset_table(str, start, start+len, STR_ENC_GET(str), table, del, nodel); + } + else { + loffset = lstrip_offset(str, start, start+len, STR_ENC_GET(str)); + } + if (loffset <= 0) return str_duplicate(rb_cString, str); + return rb_str_subseq(str, loffset, len - loffset); } +static long +rstrip_offset(VALUE str, const char *s, const char *e, rb_encoding *enc) +{ + const char *t; + + rb_str_check_dummy_enc(enc); + if (rb_enc_str_coderange(str) == ENC_CODERANGE_BROKEN) { + rb_raise(rb_eEncCompatError, "invalid byte sequence in %s", rb_enc_name(enc)); + } + if (!s || s >= e) return 0; + t = e; + + /* remove trailing spaces or '\0's */ + if (single_byte_optimizable(str)) { + unsigned char c; + while (s < t && ((c = *(t-1)) == '\0' || ascii_isspace(c))) t--; + } + else { + char *tp; + + while ((tp = rb_enc_prev_char(s, t, e, enc)) != NULL) { + unsigned int c = rb_enc_codepoint(tp, e, enc); + if (c && !rb_isspace(c)) break; + t = tp; + } + } + return e - t; +} + +static long +rstrip_offset_table(VALUE str, const char *s, const char *e, rb_encoding *enc, + char table[TR_TABLE_SIZE], VALUE del, VALUE nodel) +{ + const char *t; + char *tp; + + rb_str_check_dummy_enc(enc); + if (rb_enc_str_coderange(str) == ENC_CODERANGE_BROKEN) { + rb_raise(rb_eEncCompatError, "invalid byte sequence in %s", rb_enc_name(enc)); + } + if (!s || s >= e) return 0; + t = e; + + /* remove trailing characters in the table */ + while ((tp = rb_enc_prev_char(s, t, e, enc)) != NULL) { + unsigned int c = rb_enc_codepoint(tp, e, enc); + if (!tr_find(c, table, del, nodel)) break; + t = tp; + } + + return e - t; +} /* * call-seq: - * str.rstrip! => self or nil - * - * Removes trailing whitespace from <i>str</i>, returning <code>nil</code> if - * no change was made. See also <code>String#lstrip!</code> and - * <code>String#strip!</code>. - * - * " hello ".rstrip #=> " hello" - * "hello".rstrip! #=> nil + * rstrip!(*selectors) -> self or nil + * + * Like String#rstrip, except that: + * + * - Performs stripping in +self+ (not in a copy of +self+). + * - Returns +self+ if any characters are stripped, +nil+ otherwise. + * + * Related: see {Modifying}[rdoc-ref:String@Modifying]. */ static VALUE -rb_str_rstrip_bang(str) - VALUE str; +rb_str_rstrip_bang(int argc, VALUE *argv, VALUE str) { - char *s, *t, *e; - - s = RSTRING(str)->ptr; - if (!s || RSTRING(str)->len == 0) return Qnil; - e = t = s + RSTRING(str)->len; + rb_encoding *enc; + char *start; + long olen, roffset; - /* remove trailing '\0's */ - while (s < t && t[-1] == '\0') t--; + str_modify_keep_cr(str); + enc = STR_ENC_GET(str); + RSTRING_GETMEM(str, start, olen); + if (argc > 0) { + char table[TR_TABLE_SIZE]; + VALUE del = 0, nodel = 0; - /* remove trailing spaces */ - while (s < t && ISSPACE(*(t-1))) t--; + tr_setup_table_multi(table, &del, &nodel, str, argc, argv); + roffset = rstrip_offset_table(str, start, start+olen, enc, table, del, nodel); + } + else { + roffset = rstrip_offset(str, start, start+olen, enc); + } + if (roffset > 0) { + long len = olen - roffset; - if (t < e) { - rb_str_modify(str); - RSTRING(str)->len = t-s; - RSTRING(str)->ptr[RSTRING(str)->len] = '\0'; - return str; + STR_SET_LEN(str, len); + TERM_FILL(start+len, rb_enc_mbminlen(enc)); + return str; } return Qnil; } @@ -4067,543 +10460,2396 @@ rb_str_rstrip_bang(str) /* * call-seq: - * str.rstrip => new_str - * - * Returns a copy of <i>str</i> with trailing whitespace removed. See also - * <code>String#lstrip</code> and <code>String#strip</code>. - * - * " hello ".rstrip #=> " hello" - * "hello".rstrip #=> "hello" + * rstrip(*selectors) -> new_string + * + * Returns a copy of +self+ with trailing whitespace removed; + * see {Whitespace in Strings}[rdoc-ref:String@Whitespace+in+Strings]: + * + * whitespace = "\x00\t\n\v\f\r " + * s = whitespace + 'abc' + whitespace + * s # => "\u0000\t\n\v\f\r abc\u0000\t\n\v\f\r " + * s.rstrip # => "\u0000\t\n\v\f\r abc" + * + * If +selectors+ are given, removes characters of +selectors+ from the end of +self+: + * + * s = "---abc+++" + * s.rstrip("+") # => "---abc" + * + * +selectors+ must be valid character selectors (see {Character Selectors}[rdoc-ref:character_selectors.rdoc]), + * and may use any of its valid forms, including negation, ranges, and escapes: + * + * "01234abc56789".rstrip("0-9") # "01234abc" + * "01234abc56789".rstrip("0-9", "^4-6") # "01234abc56" + * + * Related: see {Converting to New String}[rdoc-ref:String@Converting+to+New+String]. */ static VALUE -rb_str_rstrip(str) - VALUE str; +rb_str_rstrip(int argc, VALUE *argv, VALUE str) { - str = rb_str_dup(str); - rb_str_rstrip_bang(str); - return str; + rb_encoding *enc; + char *start; + long olen, roffset; + + enc = STR_ENC_GET(str); + RSTRING_GETMEM(str, start, olen); + if (argc > 0) { + char table[TR_TABLE_SIZE]; + VALUE del = 0, nodel = 0; + + tr_setup_table_multi(table, &del, &nodel, str, argc, argv); + roffset = rstrip_offset_table(str, start, start+olen, enc, table, del, nodel); + } + else { + roffset = rstrip_offset(str, start, start+olen, enc); + } + if (roffset <= 0) return str_duplicate(rb_cString, str); + return rb_str_subseq(str, 0, olen-roffset); } /* * call-seq: - * str.strip! => str or nil - * - * Removes leading and trailing whitespace from <i>str</i>. Returns - * <code>nil</code> if <i>str</i> was not altered. + * strip!(*selectors) -> self or nil + * + * Like String#strip, except that: + * + * - Any modifications are made to +self+. + * - Returns +self+ if any modification are made, +nil+ otherwise. + * + * Related: see {Modifying}[rdoc-ref:String@Modifying]. */ static VALUE -rb_str_strip_bang(str) - VALUE str; +rb_str_strip_bang(int argc, VALUE *argv, VALUE str) { - VALUE l = rb_str_lstrip_bang(str); - VALUE r = rb_str_rstrip_bang(str); + char *start; + long olen, loffset, roffset; + rb_encoding *enc; - if (NIL_P(l) && NIL_P(r)) return Qnil; - return str; + str_modify_keep_cr(str); + enc = STR_ENC_GET(str); + RSTRING_GETMEM(str, start, olen); + + if (argc > 0) { + char table[TR_TABLE_SIZE]; + VALUE del = 0, nodel = 0; + + tr_setup_table_multi(table, &del, &nodel, str, argc, argv); + loffset = lstrip_offset_table(str, start, start+olen, enc, table, del, nodel); + roffset = rstrip_offset_table(str, start+loffset, start+olen, enc, table, del, nodel); + } + else { + loffset = lstrip_offset(str, start, start+olen, enc); + roffset = rstrip_offset(str, start+loffset, start+olen, enc); + } + + if (loffset > 0 || roffset > 0) { + long len = olen-roffset; + if (loffset > 0) { + len -= loffset; + memmove(start, start + loffset, len); + } + STR_SET_LEN(str, len); + TERM_FILL(start+len, rb_enc_mbminlen(enc)); + return str; + } + return Qnil; } /* * call-seq: - * str.strip => new_str - * - * Returns a copy of <i>str</i> with leading and trailing whitespace removed. - * - * " hello ".strip #=> "hello" - * "\tgoodbye\r\n".strip #=> "goodbye" + * strip(*selectors) -> new_string + * + * Returns a copy of +self+ with leading and trailing whitespace removed; + * see {Whitespace in Strings}[rdoc-ref:String@Whitespace+in+Strings]: + * + * whitespace = "\x00\t\n\v\f\r " + * s = whitespace + 'abc' + whitespace + * # => "\u0000\t\n\v\f\r abc\u0000\t\n\v\f\r " + * s.strip # => "abc" + * + * If +selectors+ are given, removes characters of +selectors+ from both ends of +self+: + * + * s = "---abc+++" + * s.strip("-+") # => "abc" + * s.strip("+-") # => "abc" + * + * +selectors+ must be valid character selectors (see {Character Selectors}[rdoc-ref:character_selectors.rdoc]), + * and may use any of its valid forms, including negation, ranges, and escapes: + * + * "01234abc56789".strip("0-9") # "abc" + * "01234abc56789".strip("0-9", "^4-6") # "4abc56" + * + * Related: see {Converting to New String}[rdoc-ref:String@Converting+to+New+String]. */ static VALUE -rb_str_strip(str) - VALUE str; +rb_str_strip(int argc, VALUE *argv, VALUE str) { - str = rb_str_dup(str); - rb_str_strip_bang(str); - return str; + char *start; + long olen, loffset, roffset; + rb_encoding *enc = STR_ENC_GET(str); + + RSTRING_GETMEM(str, start, olen); + + if (argc > 0) { + char table[TR_TABLE_SIZE]; + VALUE del = 0, nodel = 0; + + tr_setup_table_multi(table, &del, &nodel, str, argc, argv); + loffset = lstrip_offset_table(str, start, start+olen, enc, table, del, nodel); + roffset = rstrip_offset_table(str, start+loffset, start+olen, enc, table, del, nodel); + } + else { + loffset = lstrip_offset(str, start, start+olen, enc); + roffset = rstrip_offset(str, start+loffset, start+olen, enc); + } + + if (loffset <= 0 && roffset <= 0) return str_duplicate(rb_cString, str); + return rb_str_subseq(str, loffset, olen-loffset-roffset); } static VALUE -scan_once(str, pat, start) - VALUE str, pat; - long *start; -{ - VALUE result, match; - struct re_registers *regs; - long i; +scan_once(VALUE str, VALUE pat, long *start, int set_backref_str) +{ + VALUE result = Qnil; + long end, pos = rb_pat_search(pat, str, *start, set_backref_str); + if (pos >= 0) { + VALUE match; + struct re_registers *regs; + if (BUILTIN_TYPE(pat) == T_STRING) { + regs = NULL; + end = pos + RSTRING_LEN(pat); + } + else { + match = rb_backref_get(); + regs = RMATCH_REGS(match); + pos = BEG(0); + end = END(0); + } + + if (pos == end) { + rb_encoding *enc = STR_ENC_GET(str); + /* + * Always consume at least one character of the input string + */ + if (RSTRING_LEN(str) > end) + *start = end + rb_enc_fast_mbclen(RSTRING_PTR(str) + end, + RSTRING_END(str), enc); + else + *start = end + 1; + } + else { + *start = end; + } + + if (!regs || regs->num_regs == 1) { + result = rb_str_subseq(str, pos, end - pos); + return result; + } + else { + result = rb_ary_new2(regs->num_regs); + for (int i = 1; i < regs->num_regs; i++) { + VALUE s = Qnil; + if (BEG(i) >= 0) { + s = rb_str_subseq(str, BEG(i), END(i)-BEG(i)); + } - if (rb_reg_search(pat, str, *start, 0) >= 0) { - match = rb_backref_get(); - regs = RMATCH(match)->regs; - if (BEG(0) == END(0)) { - /* - * Always consume at least one character of the input string - */ - if (RSTRING(str)->len < END(0)) - *start = END(0)+mbclen2(RSTRING(str)->ptr[END(0)],pat); - else - *start = END(0)+1; - } - else { - *start = END(0); - } - if (regs->num_regs == 1) { - return rb_reg_nth_match(0, match); - } - result = rb_ary_new2(regs->num_regs); - for (i=1; i < regs->num_regs; i++) { - rb_ary_push(result, rb_reg_nth_match(i, match)); - } - - return result; + rb_ary_push(result, s); + } + } + + RB_GC_GUARD(match); } - return Qnil; + + return result; } /* * call-seq: - * str.scan(pattern) => array - * str.scan(pattern) {|match, ...| block } => str - * - * Both forms iterate through <i>str</i>, matching the pattern (which may be a - * <code>Regexp</code> or a <code>String</code>). For each match, a result is - * generated and either added to the result array or passed to the block. If - * the pattern contains no groups, each individual result consists of the - * matched string, <code>$&</code>. If the pattern contains groups, each - * individual result is itself an array containing one entry per group. - * - * a = "cruel world" - * a.scan(/\w+/) #=> ["cruel", "world"] - * a.scan(/.../) #=> ["cru", "el ", "wor"] - * a.scan(/(...)/) #=> [["cru"], ["el "], ["wor"]] - * a.scan(/(..)(..)/) #=> [["cr", "ue"], ["l ", "wo"]] - * - * And the block form: - * - * a.scan(/\w+/) {|w| print "<<#{w}>> " } - * print "\n" - * a.scan(/(.)(.)/) {|a,b| print b, a } - * print "\n" - * - * <em>produces:</em> - * - * <<cruel>> <<world>> - * rceu lowlr + * scan(pattern) -> array_of_results + * scan(pattern) {|result| ... } -> self + * + * :include: doc/string/scan.rdoc + * */ static VALUE -rb_str_scan(str, pat) - VALUE str, pat; +rb_str_scan(VALUE str, VALUE pat) { VALUE result; long start = 0; - VALUE match = Qnil; + long last = -1, prev = 0; + char *p = RSTRING_PTR(str); long len = RSTRING_LEN(str); - pat = get_pat(pat, 1); + pat = get_pat_quoted(pat, 1); + mustnot_broken(str); if (!rb_block_given_p()) { - VALUE ary = rb_ary_new(); - - while (!NIL_P(result = scan_once(str, pat, &start))) { - match = rb_backref_get(); - rb_ary_push(ary, result); - } - rb_backref_set(match); - return ary; - } - - while (!NIL_P(result = scan_once(str, pat, &start))) { - match = rb_backref_get(); - rb_match_busy(match); - rb_yield(result); - rb_backref_set(match); /* restore $~ value */ - } - rb_backref_set(match); + VALUE ary = rb_ary_new(); + + while (!NIL_P(result = scan_once(str, pat, &start, 0))) { + last = prev; + prev = start; + rb_ary_push(ary, result); + } + if (last >= 0) rb_pat_search(pat, str, last, 1); + else rb_backref_set(Qnil); + return ary; + } + + while (!NIL_P(result = scan_once(str, pat, &start, 1))) { + last = prev; + prev = start; + rb_yield(result); + str_mod_check(str, p, len); + } + if (last >= 0) rb_pat_search(pat, str, last, 1); return str; } + /* * call-seq: - * scan(pattern) => array - * scan(pattern) {|///| block } => $_ - * - * Equivalent to calling <code>$_.scan</code>. See - * <code>String#scan</code>. + * hex -> integer + * + * Interprets the leading substring of +self+ as hexadecimal, possibly signed; + * returns its value as an integer. + * + * The leading substring is interpreted as hexadecimal when it begins with: + * + * - One or more character representing hexadecimal digits + * (each in one of the ranges <tt>'0'..'9'</tt>, <tt>'a'..'f'</tt>, or <tt>'A'..'F'</tt>); + * the string to be interpreted ends at the first character that does not represent a hexadecimal digit: + * + * 'f'.hex # => 15 + * '11'.hex # => 17 + * 'FFF'.hex # => 4095 + * 'fffg'.hex # => 4095 + * 'foo'.hex # => 15 # 'f' hexadecimal, 'oo' not. + * 'bar'.hex # => 186 # 'ba' hexadecimal, 'r' not. + * 'deadbeef'.hex # => 3735928559 + * + * - <tt>'0x'</tt> or <tt>'0X'</tt>, followed by one or more hexadecimal digits: + * + * '0xfff'.hex # => 4095 + * '0xfffg'.hex # => 4095 + * + * Any of the above may prefixed with <tt>'-'</tt>, which negates the interpreted value: + * + * '-fff'.hex # => -4095 + * '-0xFFF'.hex # => -4095 + * + * For any substring not described above, returns zero: + * + * 'xxx'.hex # => 0 + * ''.hex # => 0 + * + * Note that, unlike #oct, this method interprets only hexadecimal, + * and not binary, octal, or decimal notations: + * + * '0b111'.hex # => 45329 + * '0o777'.hex # => 0 + * '0d999'.hex # => 55705 + * + * Related: See {Converting to Non-String}[rdoc-ref:String@Converting+to+Non--5CString]. */ static VALUE -rb_f_scan(self, pat) - VALUE self, pat; +rb_str_hex(VALUE str) { - return rb_str_scan(uscore_get(), pat); + return rb_str_to_inum(str, 16, FALSE); } /* * call-seq: - * str.hex => integer - * - * Treats leading characters from <i>str</i> as a string of hexadecimal digits - * (with an optional sign and an optional <code>0x</code>) and returns the - * corresponding number. Zero is returned on error. - * - * "0x0a".hex #=> 10 - * "-1234".hex #=> -4660 - * "0".hex #=> 0 - * "wombat".hex #=> 0 + * oct -> integer + * + * Interprets the leading substring of +self+ as octal, binary, decimal, or hexadecimal, possibly signed; + * returns their value as an integer. + * + * In brief: + * + * # Interpreted as octal. + * '777'.oct # => 511 + * '777x'.oct # => 511 + * '0777'.oct # => 511 + * '0o777'.oct # => 511 + * '-777'.oct # => -511 + * # Not interpreted as octal. + * '0b111'.oct # => 7 # Interpreted as binary. + * '0d999'.oct # => 999 # Interpreted as decimal. + * '0xfff'.oct # => 4095 # Interpreted as hexadecimal. + * + * The leading substring is interpreted as octal when it begins with: + * + * - One or more character representing octal digits + * (each in the range <tt>'0'..'7'</tt>); + * the string to be interpreted ends at the first character that does not represent an octal digit: + * + * '7'.oct @ => 7 + * '11'.oct # => 9 + * '777'.oct # => 511 + * '0777'.oct # => 511 + * '7778'.oct # => 511 + * '777x'.oct # => 511 + * + * - <tt>'0o'</tt>, followed by one or more octal digits: + * + * '0o777'.oct # => 511 + * '0o7778'.oct # => 511 + * + * The leading substring is _not_ interpreted as octal when it begins with: + * + * - <tt>'0b'</tt>, followed by one or more characters representing binary digits + * (each in the range <tt>'0'..'1'</tt>); + * the string to be interpreted ends at the first character that does not represent a binary digit. + * the string is interpreted as binary digits (base 2): + * + * '0b111'.oct # => 7 + * '0b1112'.oct # => 7 + * + * - <tt>'0d'</tt>, followed by one or more characters representing decimal digits + * (each in the range <tt>'0'..'9'</tt>); + * the string to be interpreted ends at the first character that does not represent a decimal digit. + * the string is interpreted as decimal digits (base 10): + * + * '0d999'.oct # => 999 + * '0d999x'.oct # => 999 + * + * - <tt>'0x'</tt>, followed by one or more characters representing hexadecimal digits + * (each in one of the ranges <tt>'0'..'9'</tt>, <tt>'a'..'f'</tt>, or <tt>'A'..'F'</tt>); + * the string to be interpreted ends at the first character that does not represent a hexadecimal digit. + * the string is interpreted as hexadecimal digits (base 16): + * + * '0xfff'.oct # => 4095 + * '0xfffg'.oct # => 4095 + * + * Any of the above may prefixed with <tt>'-'</tt>, which negates the interpreted value: + * + * '-777'.oct # => -511 + * '-0777'.oct # => -511 + * '-0b111'.oct # => -7 + * '-0xfff'.oct # => -4095 + * + * For any substring not described above, returns zero: + * + * 'foo'.oct # => 0 + * ''.oct # => 0 + * + * Related: see {Converting to Non-String}[rdoc-ref:String@Converting+to+Non--5CString]. */ static VALUE -rb_str_hex(str) - VALUE str; +rb_str_oct(VALUE str) { - return rb_str_to_inum(str, 16, Qfalse); + return rb_str_to_inum(str, -8, FALSE); } +#ifndef HAVE_CRYPT_R +# include "ruby/thread_native.h" +# include "ruby/atomic.h" + +static struct { + rb_nativethread_lock_t lock; +} crypt_mutex = {PTHREAD_MUTEX_INITIALIZER}; +#endif /* * call-seq: - * str.oct => integer - * - * Treats leading characters of <i>str</i> as a string of octal digits (with an - * optional sign) and returns the corresponding number. Returns 0 if the - * conversion fails. - * - * "123".oct #=> 83 - * "-377".oct #=> -255 - * "bad".oct #=> 0 - * "0377bad".oct #=> 255 + * crypt(salt_str) -> new_string + * + * Returns the string generated by calling <code>crypt(3)</code> + * standard library function with <code>str</code> and + * <code>salt_str</code>, in this order, as its arguments. Please do + * not use this method any longer. It is legacy; provided only for + * backward compatibility with ruby scripts in earlier days. It is + * bad to use in contemporary programs for several reasons: + * + * * Behaviour of C's <code>crypt(3)</code> depends on the OS it is + * run. The generated string lacks data portability. + * + * * On some OSes such as Mac OS, <code>crypt(3)</code> never fails + * (i.e. silently ends up in unexpected results). + * + * * On some OSes such as Mac OS, <code>crypt(3)</code> is not + * thread safe. + * + * * So-called "traditional" usage of <code>crypt(3)</code> is very + * very very weak. According to its manpage, Linux's traditional + * <code>crypt(3)</code> output has only 2**56 variations; too + * easy to brute force today. And this is the default behaviour. + * + * * In order to make things robust some OSes implement so-called + * "modular" usage. To go through, you have to do a complex + * build-up of the <code>salt_str</code> parameter, by hand. + * Failure in generation of a proper salt string tends not to + * yield any errors; typos in parameters are normally not + * detectable. + * + * * For instance, in the following example, the second invocation + * of String#crypt is wrong; it has a typo in "round=" (lacks + * "s"). However the call does not fail and something unexpected + * is generated. + * + * "foo".crypt("$5$rounds=1000$salt$") # OK, proper usage + * "foo".crypt("$5$round=1000$salt$") # Typo not detected + * + * * Even in the "modular" mode, some hash functions are considered + * archaic and no longer recommended at all; for instance module + * <code>$1$</code> is officially abandoned by its author: see + * http://phk.freebsd.dk/sagas/md5crypt_eol/ . For another + * instance module <code>$3$</code> is considered completely + * broken: see the manpage of FreeBSD. + * + * * On some OS such as Mac OS, there is no modular mode. Yet, as + * written above, <code>crypt(3)</code> on Mac OS never fails. + * This means even if you build up a proper salt string it + * generates a traditional DES hash anyways, and there is no way + * for you to be aware of. + * + * "foo".crypt("$5$rounds=1000$salt$") # => "$5fNPQMxC5j6." + * + * If for some reason you cannot migrate to other secure contemporary + * password hashing algorithms, install the string-crypt gem and + * <code>require 'string/crypt'</code> to continue using it. */ static VALUE -rb_str_oct(str) - VALUE str; +rb_str_crypt(VALUE str, VALUE salt) { - return rb_str_to_inum(str, -8, Qfalse); +#ifdef HAVE_CRYPT_R + VALUE databuf; + struct crypt_data *data; +# define CRYPT_END() ALLOCV_END(databuf) +#else + char *tmp_buf; + extern char *crypt(const char *, const char *); +# define CRYPT_END() rb_nativethread_lock_unlock(&crypt_mutex.lock) +#endif + VALUE result; + const char *s, *saltp; + char *res; +#ifdef BROKEN_CRYPT + char salt_8bit_clean[3]; +#endif + + StringValue(salt); + mustnot_wchar(str); + mustnot_wchar(salt); + s = StringValueCStr(str); + saltp = RSTRING_PTR(salt); + if (RSTRING_LEN(salt) < 2 || !saltp[0] || !saltp[1]) { + rb_raise(rb_eArgError, "salt too short (need >=2 bytes)"); + } + +#ifdef BROKEN_CRYPT + if (!ISASCII((unsigned char)saltp[0]) || !ISASCII((unsigned char)saltp[1])) { + salt_8bit_clean[0] = saltp[0] & 0x7f; + salt_8bit_clean[1] = saltp[1] & 0x7f; + salt_8bit_clean[2] = '\0'; + saltp = salt_8bit_clean; + } +#endif +#ifdef HAVE_CRYPT_R + data = ALLOCV(databuf, sizeof(struct crypt_data)); +# ifdef HAVE_STRUCT_CRYPT_DATA_INITIALIZED + data->initialized = 0; +# endif + res = crypt_r(s, saltp, data); +#else + rb_nativethread_lock_lock(&crypt_mutex.lock); + res = crypt(s, saltp); +#endif + if (!res) { + int err = errno; + CRYPT_END(); + rb_syserr_fail(err, "crypt"); + } +#ifdef HAVE_CRYPT_R + result = rb_str_new_cstr(res); + CRYPT_END(); +#else + // We need to copy this buffer because it's static and we need to unlock the mutex + // before allocating a new object (the string to be returned). If we allocate while + // holding the lock, we could run GC which fires the VM barrier and causes a deadlock + // if other ractors are waiting on this lock. + size_t res_size = strlen(res)+1; + tmp_buf = ALLOCA_N(char, res_size); // should be small enough to alloca + memcpy(tmp_buf, res, res_size); + res = tmp_buf; + CRYPT_END(); + result = rb_str_new_cstr(res); +#endif + return result; } /* * call-seq: - * str.crypt(other_str) => new_str - * - * Applies a one-way cryptographic hash to <i>str</i> by invoking the standard - * library function <code>crypt</code>. The argument is the salt string, which - * should be two characters long, each character drawn from - * <code>[a-zA-Z0-9./]</code>. + * ord -> integer + * + * :include: doc/string/ord.rdoc + * */ static VALUE -rb_str_crypt(str, salt) - VALUE str, salt; +rb_str_ord(VALUE s) { - extern char *crypt(); - VALUE result; - char *s; + unsigned int c; - StringValue(salt); - if (RSTRING(salt)->len < 2) - rb_raise(rb_eArgError, "salt too short(need >=2 bytes)"); - - if (RSTRING(str)->ptr) s = RSTRING(str)->ptr; - else s = ""; - result = rb_str_new2(crypt(s, RSTRING(salt)->ptr)); - OBJ_INFECT(result, str); - OBJ_INFECT(result, salt); - return result; + c = rb_enc_codepoint(RSTRING_PTR(s), RSTRING_END(s), STR_ENC_GET(s)); + return UINT2NUM(c); +} +/* + * call-seq: + * sum(n = 16) -> integer + * + * :include: doc/string/sum.rdoc + * + */ + +static VALUE +rb_str_sum(int argc, VALUE *argv, VALUE str) +{ + int bits = 16; + char *ptr, *p, *pend; + long len; + VALUE sum = INT2FIX(0); + unsigned long sum0 = 0; + + if (rb_check_arity(argc, 0, 1) && (bits = NUM2INT(argv[0])) < 0) { + bits = 0; + } + ptr = p = RSTRING_PTR(str); + len = RSTRING_LEN(str); + pend = p + len; + + while (p < pend) { + if (FIXNUM_MAX - UCHAR_MAX < sum0) { + sum = rb_funcall(sum, '+', 1, LONG2FIX(sum0)); + str_mod_check(str, ptr, len); + sum0 = 0; + } + sum0 += (unsigned char)*p; + p++; + } + + if (bits == 0) { + if (sum0) { + sum = rb_funcall(sum, '+', 1, LONG2FIX(sum0)); + } + } + else { + if (sum == INT2FIX(0)) { + if (bits < (int)sizeof(long)*CHAR_BIT) { + sum0 &= (((unsigned long)1)<<bits)-1; + } + sum = LONG2FIX(sum0); + } + else { + VALUE mod; + + if (sum0) { + sum = rb_funcall(sum, '+', 1, LONG2FIX(sum0)); + } + + mod = rb_funcall(INT2FIX(1), idLTLT, 1, INT2FIX(bits)); + mod = rb_funcall(mod, '-', 1, INT2FIX(1)); + sum = rb_funcall(sum, '&', 1, mod); + } + } + return sum; +} + +static VALUE +rb_str_justify(int argc, VALUE *argv, VALUE str, char jflag) +{ + rb_encoding *enc; + VALUE w; + long width, len, flen = 1, fclen = 1; + VALUE res; + char *p; + const char *f = " "; + long n, size, llen, rlen, llen2 = 0, rlen2 = 0; + VALUE pad; + int singlebyte = 1, cr; + int termlen; + + rb_scan_args(argc, argv, "11", &w, &pad); + enc = STR_ENC_GET(str); + termlen = rb_enc_mbminlen(enc); + width = NUM2LONG(w); + if (argc == 2) { + StringValue(pad); + enc = rb_enc_check(str, pad); + f = RSTRING_PTR(pad); + flen = RSTRING_LEN(pad); + fclen = str_strlen(pad, enc); /* rb_enc_check */ + singlebyte = single_byte_optimizable(pad); + if (flen == 0 || fclen == 0) { + rb_raise(rb_eArgError, "zero width padding"); + } + } + len = str_strlen(str, enc); /* rb_enc_check */ + if (width < 0 || len >= width) return str_duplicate(rb_cString, str); + n = width - len; + llen = (jflag == 'l') ? 0 : ((jflag == 'r') ? n : n/2); + rlen = n - llen; + cr = ENC_CODERANGE(str); + if (flen > 1) { + llen2 = str_offset(f, f + flen, llen % fclen, enc, singlebyte); + rlen2 = str_offset(f, f + flen, rlen % fclen, enc, singlebyte); + } + size = RSTRING_LEN(str); + if ((len = llen / fclen + rlen / fclen) >= LONG_MAX / flen || + (len *= flen) >= LONG_MAX - llen2 - rlen2 || + (len += llen2 + rlen2) >= LONG_MAX - size) { + rb_raise(rb_eArgError, "argument too big"); + } + len += size; + res = str_enc_new(rb_cString, 0, len, enc); + p = RSTRING_PTR(res); + if (flen <= 1) { + memset(p, *f, llen); + p += llen; + } + else { + while (llen >= fclen) { + memcpy(p,f,flen); + p += flen; + llen -= fclen; + } + if (llen > 0) { + memcpy(p, f, llen2); + p += llen2; + } + } + memcpy(p, RSTRING_PTR(str), size); + p += size; + if (flen <= 1) { + memset(p, *f, rlen); + p += rlen; + } + else { + while (rlen >= fclen) { + memcpy(p,f,flen); + p += flen; + rlen -= fclen; + } + if (rlen > 0) { + memcpy(p, f, rlen2); + p += rlen2; + } + } + TERM_FILL(p, termlen); + STR_SET_LEN(res, p-RSTRING_PTR(res)); + + if (argc == 2) + cr = ENC_CODERANGE_AND(cr, ENC_CODERANGE(pad)); + if (cr != ENC_CODERANGE_BROKEN) + ENC_CODERANGE_SET(res, cr); + + RB_GC_GUARD(pad); + return res; } /* * call-seq: - * str.intern => symbol - * str.to_sym => symbol - * - * Returns the <code>Symbol</code> corresponding to <i>str</i>, creating the - * symbol if it did not previously exist. See <code>Symbol#id2name</code>. - * - * "Koala".intern #=> :Koala - * s = 'cat'.to_sym #=> :cat - * s == :cat #=> true - * s = '@cat'.to_sym #=> :@cat - * s == :@cat #=> true + * ljust(width, pad_string = ' ') -> new_string + * + * :include: doc/string/ljust.rdoc * - * This can also be used to create symbols that cannot be represented using the - * <code>:xxx</code> notation. - * - * 'cat and dog'.to_sym #=> :"cat and dog" */ -VALUE -rb_str_intern(str) - VALUE str; +static VALUE +rb_str_ljust(int argc, VALUE *argv, VALUE str) { - ID id; + return rb_str_justify(argc, argv, str, 'l'); +} + +/* + * call-seq: + * rjust(width, pad_string = ' ') -> new_string + * + * :include: doc/string/rjust.rdoc + * + */ + +static VALUE +rb_str_rjust(int argc, VALUE *argv, VALUE str) +{ + return rb_str_justify(argc, argv, str, 'r'); +} + + +/* + * call-seq: + * center(size, pad_string = ' ') -> new_string + * + * :include: doc/string/center.rdoc + * + */ + +static VALUE +rb_str_center(int argc, VALUE *argv, VALUE str) +{ + return rb_str_justify(argc, argv, str, 'c'); +} + +/* + * call-seq: + * partition(pattern) -> [pre_match, first_match, post_match] + * + * :include: doc/string/partition.rdoc + * + */ - if (!RSTRING(str)->ptr || RSTRING(str)->len == 0) { - rb_raise(rb_eArgError, "interning empty string"); +static VALUE +rb_str_partition(VALUE str, VALUE sep) +{ + long pos; + + sep = get_pat_quoted(sep, 0); + if (RB_TYPE_P(sep, T_REGEXP)) { + if (rb_reg_search(sep, str, 0, 0) < 0) { + goto failed; + } + VALUE match = rb_backref_get(); + struct re_registers *regs = RMATCH_REGS(match); + + pos = BEG(0); + sep = rb_str_subseq(str, pos, END(0) - pos); + } + else { + pos = rb_str_index(str, sep, 0); + if (pos < 0) goto failed; } - if (strlen(RSTRING(str)->ptr) != RSTRING(str)->len) - rb_raise(rb_eArgError, "symbol string may not contain `\\0'"); - id = rb_intern(RSTRING(str)->ptr); - return ID2SYM(id); + return rb_ary_new3(3, rb_str_subseq(str, 0, pos), + sep, + rb_str_subseq(str, pos+RSTRING_LEN(sep), + RSTRING_LEN(str)-pos-RSTRING_LEN(sep))); + + failed: + return rb_ary_new3(3, str_duplicate(rb_cString, str), str_new_empty_String(str), str_new_empty_String(str)); } +/* + * call-seq: + * rpartition(pattern) -> [pre_match, last_match, post_match] + * + * :include: doc/string/rpartition.rdoc + * + */ + +static VALUE +rb_str_rpartition(VALUE str, VALUE sep) +{ + long pos = RSTRING_LEN(str); + + sep = get_pat_quoted(sep, 0); + if (RB_TYPE_P(sep, T_REGEXP)) { + if (rb_reg_search(sep, str, pos, 1) < 0) { + goto failed; + } + VALUE match = rb_backref_get(); + struct re_registers *regs = RMATCH_REGS(match); + + pos = BEG(0); + sep = rb_str_subseq(str, pos, END(0) - pos); + } + else { + pos = rb_str_sublen(str, pos); + pos = rb_str_rindex(str, sep, pos); + if (pos < 0) { + goto failed; + } + } + + return rb_ary_new3(3, rb_str_subseq(str, 0, pos), + sep, + rb_str_subseq(str, pos+RSTRING_LEN(sep), + RSTRING_LEN(str)-pos-RSTRING_LEN(sep))); + failed: + return rb_ary_new3(3, str_new_empty_String(str), str_new_empty_String(str), str_duplicate(rb_cString, str)); +} /* * call-seq: - * str.sum(n=16) => integer - * - * Returns a basic <em>n</em>-bit checksum of the characters in <i>str</i>, - * where <em>n</em> is the optional <code>Fixnum</code> parameter, defaulting - * to 16. The result is simply the sum of the binary value of each character in - * <i>str</i> modulo <code>2n - 1</code>. This is not a particularly good - * checksum. + * start_with?(*patterns) -> true or false + * + * :include: doc/string/start_with_p.rdoc + * */ static VALUE -rb_str_sum(argc, argv, str) - int argc; - VALUE *argv; - VALUE str; +rb_str_start_with(int argc, VALUE *argv, VALUE str) { - VALUE vbits; - int bits; - char *p, *pend; + int i; - if (rb_scan_args(argc, argv, "01", &vbits) == 0) { - bits = 16; + for (i=0; i<argc; i++) { + VALUE tmp = argv[i]; + if (RB_TYPE_P(tmp, T_REGEXP)) { + if (rb_reg_start_with_p(tmp, str)) + return Qtrue; + } + else { + const char *p, *s, *e; + long slen, tlen; + rb_encoding *enc; + + StringValue(tmp); + enc = rb_enc_check(str, tmp); + if ((tlen = RSTRING_LEN(tmp)) == 0) return Qtrue; + if ((slen = RSTRING_LEN(str)) < tlen) continue; + p = RSTRING_PTR(str); + e = p + slen; + s = p + tlen; + if (!at_char_right_boundary(p, s, e, enc)) + continue; + if (memcmp(p, RSTRING_PTR(tmp), tlen) == 0) + return Qtrue; + } } - else bits = NUM2INT(vbits); + return Qfalse; +} - p = RSTRING(str)->ptr; pend = p + RSTRING(str)->len; - if (bits > sizeof(long)*CHAR_BIT) { - VALUE res = INT2FIX(0); - VALUE mod; +/* + * call-seq: + * end_with?(*strings) -> true or false + * + * :include: doc/string/end_with_p.rdoc + * + */ - mod = rb_funcall(INT2FIX(1), rb_intern("<<"), 1, INT2FIX(bits)); - mod = rb_funcall(mod, '-', 1, INT2FIX(1)); +static VALUE +rb_str_end_with(int argc, VALUE *argv, VALUE str) +{ + int i; - while (p < pend) { - res = rb_funcall(res, '+', 1, INT2FIX((unsigned int)*p)); - p++; - } - res = rb_funcall(res, '&', 1, mod); - return res; + for (i=0; i<argc; i++) { + VALUE tmp = argv[i]; + const char *p, *s, *e; + long slen, tlen; + rb_encoding *enc; + + StringValue(tmp); + enc = rb_enc_check(str, tmp); + if ((tlen = RSTRING_LEN(tmp)) == 0) return Qtrue; + if ((slen = RSTRING_LEN(str)) < tlen) continue; + p = RSTRING_PTR(str); + e = p + slen; + s = e - tlen; + if (!at_char_boundary(p, s, e, enc)) + continue; + if (memcmp(s, RSTRING_PTR(tmp), tlen) == 0) + return Qtrue; + } + return Qfalse; +} + +/*! + * Returns the length of the <i>prefix</i> to be deleted in the given <i>str</i>, + * returning 0 if <i>str</i> does not start with the <i>prefix</i>. + * + * @param str the target + * @param prefix the prefix + * @retval 0 if the given <i>str</i> does not start with the given <i>prefix</i> + * @retval Positive-Integer otherwise + */ +static long +deleted_prefix_length(VALUE str, VALUE prefix) +{ + const char *strptr, *prefixptr; + long olen, prefixlen; + rb_encoding *enc = rb_enc_get(str); + + StringValue(prefix); + + if (!is_broken_string(prefix) || + !rb_enc_asciicompat(enc) || + !rb_enc_asciicompat(rb_enc_get(prefix))) { + enc = rb_enc_check(str, prefix); + } + + /* return 0 if not start with prefix */ + prefixlen = RSTRING_LEN(prefix); + if (prefixlen <= 0) return 0; + olen = RSTRING_LEN(str); + if (olen < prefixlen) return 0; + strptr = RSTRING_PTR(str); + prefixptr = RSTRING_PTR(prefix); + if (memcmp(strptr, prefixptr, prefixlen) != 0) return 0; + if (is_broken_string(prefix)) { + if (!is_broken_string(str)) { + /* prefix in a valid string cannot be broken */ + return 0; + } + const char *strend = strptr + olen; + const char *after_prefix = strptr + prefixlen; + if (!at_char_right_boundary(strptr, after_prefix, strend, enc)) { + /* prefix does not end at char-boundary */ + return 0; + } + } + /* prefix part in `str` also should be valid. */ + + return prefixlen; +} + +/* + * call-seq: + * delete_prefix!(prefix) -> self or nil + * + * Like String#delete_prefix, except that +self+ is modified in place; + * returns +self+ if the prefix is removed, +nil+ otherwise. + * + * Related: see {Modifying}[rdoc-ref:String@Modifying]. + */ + +static VALUE +rb_str_delete_prefix_bang(VALUE str, VALUE prefix) +{ + long prefixlen; + str_modify_keep_cr(str); + + prefixlen = deleted_prefix_length(str, prefix); + if (prefixlen <= 0) return Qnil; + + return rb_str_drop_bytes(str, prefixlen); +} + +/* + * call-seq: + * delete_prefix(prefix) -> new_string + * + * :include: doc/string/delete_prefix.rdoc + * + */ + +static VALUE +rb_str_delete_prefix(VALUE str, VALUE prefix) +{ + long prefixlen; + + prefixlen = deleted_prefix_length(str, prefix); + if (prefixlen <= 0) return str_duplicate(rb_cString, str); + + return rb_str_subseq(str, prefixlen, RSTRING_LEN(str) - prefixlen); +} + +/*! + * Returns the length of the <i>suffix</i> to be deleted in the given <i>str</i>, + * returning 0 if <i>str</i> does not end with the <i>suffix</i>. + * + * @param str the target + * @param suffix the suffix + * @retval 0 if the given <i>str</i> does not end with the given <i>suffix</i> + * @retval Positive-Integer otherwise + */ +static long +deleted_suffix_length(VALUE str, VALUE suffix) +{ + const char *strptr, *suffixptr; + long olen, suffixlen; + rb_encoding *enc; + + StringValue(suffix); + if (is_broken_string(suffix)) return 0; + enc = rb_enc_check(str, suffix); + + /* return 0 if not start with suffix */ + suffixlen = RSTRING_LEN(suffix); + if (suffixlen <= 0) return 0; + olen = RSTRING_LEN(str); + if (olen < suffixlen) return 0; + strptr = RSTRING_PTR(str); + suffixptr = RSTRING_PTR(suffix); + const char *strend = strptr + olen; + const char *before_suffix = strend - suffixlen; + if (memcmp(before_suffix, suffixptr, suffixlen) != 0) return 0; + if (!at_char_boundary(strptr, before_suffix, strend, enc)) return 0; + + return suffixlen; +} + +/* + * call-seq: + * delete_suffix!(suffix) -> self or nil + * + * Like String#delete_suffix, except that +self+ is modified in place; + * returns +self+ if the suffix is removed, +nil+ otherwise. + * + * Related: see {Modifying}[rdoc-ref:String@Modifying]. + */ + +static VALUE +rb_str_delete_suffix_bang(VALUE str, VALUE suffix) +{ + long olen, suffixlen, len; + str_modifiable(str); + + suffixlen = deleted_suffix_length(str, suffix); + if (suffixlen <= 0) return Qnil; + + olen = RSTRING_LEN(str); + str_modify_keep_cr(str); + len = olen - suffixlen; + STR_SET_LEN(str, len); + TERM_FILL(&RSTRING_PTR(str)[len], TERM_LEN(str)); + if (ENC_CODERANGE(str) != ENC_CODERANGE_7BIT) { + ENC_CODERANGE_CLEAR(str); + } + return str; +} + +/* + * call-seq: + * delete_suffix(suffix) -> new_string + * + * :include: doc/string/delete_suffix.rdoc + * + */ + +static VALUE +rb_str_delete_suffix(VALUE str, VALUE suffix) +{ + long suffixlen; + + suffixlen = deleted_suffix_length(str, suffix); + if (suffixlen <= 0) return str_duplicate(rb_cString, str); + + return rb_str_subseq(str, 0, RSTRING_LEN(str) - suffixlen); +} + +void +rb_str_setter(VALUE val, ID id, VALUE *var) +{ + if (!NIL_P(val) && !RB_TYPE_P(val, T_STRING)) { + rb_raise(rb_eTypeError, "value of %"PRIsVALUE" must be String", rb_id2str(id)); + } + *var = val; +} + +static void +nil_setter_warning(ID id) +{ + rb_warn_deprecated("non-nil '%"PRIsVALUE"'", NULL, rb_id2str(id)); +} + +void +rb_deprecated_str_setter(VALUE val, ID id, VALUE *var) +{ + rb_str_setter(val, id, var); + if (!NIL_P(*var)) { + nil_setter_warning(id); + } +} + +static void +rb_fs_setter(VALUE val, ID id, VALUE *var) +{ + val = rb_fs_check(val); + if (!val) { + rb_raise(rb_eTypeError, + "value of %"PRIsVALUE" must be String or Regexp", + rb_id2str(id)); + } + if (!NIL_P(val)) { + nil_setter_warning(id); + } + *var = val; +} + + +/* + * call-seq: + * force_encoding(encoding) -> self + * + * :include: doc/string/force_encoding.rdoc + * + */ + +static VALUE +rb_str_force_encoding(VALUE str, VALUE enc) +{ + str_modifiable(str); + + rb_encoding *encoding = rb_to_encoding(enc); + int idx = rb_enc_to_index(encoding); + + // If the encoding is unchanged, we do nothing. + if (ENCODING_GET(str) == idx) { + return str; + } + + rb_enc_associate_index(str, idx); + + // If the coderange was 7bit and the new encoding is ASCII-compatible + // we can keep the coderange. + if (ENC_CODERANGE(str) == ENC_CODERANGE_7BIT && encoding && rb_enc_asciicompat(encoding)) { + return str; + } + + ENC_CODERANGE_CLEAR(str); + return str; +} + +/* + * call-seq: + * b -> new_string + * + * :include: doc/string/b.rdoc + * + */ + +static VALUE +rb_str_b(VALUE str) +{ + VALUE str2; + if (STR_EMBED_P(str)) { + str2 = str_alloc_embed(rb_cString, RSTRING_LEN(str) + TERM_LEN(str)); } else { - unsigned int res = 0; - unsigned int mod = (1<<bits)-1; + str2 = str_alloc_heap(rb_cString); + } + str_replace_shared_without_enc(str2, str); + + if (rb_enc_asciicompat(STR_ENC_GET(str))) { + // BINARY strings can never be broken; they're either 7-bit ASCII or VALID. + // If we know the receiver's code range then we know the result's code range. + int cr = ENC_CODERANGE(str); + switch (cr) { + case ENC_CODERANGE_7BIT: + ENC_CODERANGE_SET(str2, ENC_CODERANGE_7BIT); + break; + case ENC_CODERANGE_BROKEN: + case ENC_CODERANGE_VALID: + ENC_CODERANGE_SET(str2, ENC_CODERANGE_VALID); + break; + default: + ENC_CODERANGE_CLEAR(str2); + break; + } + } - if (mod == 0) { - mod = -1; - } - while (p < pend) { - res += (unsigned int)*p; - p++; - } - res &= mod; - return rb_int2inum(res); + return str2; +} + +/* + * call-seq: + * valid_encoding? -> true or false + * + * :include: doc/string/valid_encoding_p.rdoc + * + */ + +static VALUE +rb_str_valid_encoding_p(VALUE str) +{ + int cr = rb_enc_str_coderange(str); + + return RBOOL(cr != ENC_CODERANGE_BROKEN); +} + +/* + * call-seq: + * ascii_only? -> true or false + * + * Returns whether +self+ contains only ASCII characters: + * + * 'abc'.ascii_only? # => true + * "abc\u{6666}".ascii_only? # => false + * + * Related: see {Querying}[rdoc-ref:String@Querying]. + */ + +static VALUE +rb_str_is_ascii_only_p(VALUE str) +{ + int cr = rb_enc_str_coderange(str); + + return RBOOL(cr == ENC_CODERANGE_7BIT); +} + +VALUE +rb_str_ellipsize(VALUE str, long len) +{ + static const char ellipsis[] = "..."; + const long ellipsislen = sizeof(ellipsis) - 1; + rb_encoding *const enc = rb_enc_get(str); + const long blen = RSTRING_LEN(str); + const char *const p = RSTRING_PTR(str), *e = p + blen; + VALUE estr, ret = 0; + + if (len < 0) rb_raise(rb_eIndexError, "negative length %ld", len); + if (len * rb_enc_mbminlen(enc) >= blen || + (e = rb_enc_nth(p, e, len, enc)) - p == blen) { + ret = str; + } + else if (len <= ellipsislen || + !(e = rb_enc_step_back(p, e, e, len = ellipsislen, enc))) { + if (rb_enc_asciicompat(enc)) { + ret = rb_str_new(ellipsis, len); + rb_enc_associate(ret, enc); + } + else { + estr = rb_usascii_str_new(ellipsis, len); + ret = rb_str_encode(estr, rb_enc_from_encoding(enc), 0, Qnil); + } + } + else if (ret = rb_str_subseq(str, 0, e - p), rb_enc_asciicompat(enc)) { + rb_str_cat(ret, ellipsis, ellipsislen); + } + else { + estr = rb_str_encode(rb_usascii_str_new(ellipsis, ellipsislen), + rb_enc_from_encoding(enc), 0, Qnil); + rb_str_append(ret, estr); } + return ret; } static VALUE -rb_str_justify(argc, argv, str, jflag) - int argc; - VALUE *argv; - VALUE str; - char jflag; +str_compat_and_valid(VALUE str, rb_encoding *enc) { - VALUE w; - long width, flen = 0; - VALUE res; - char *p, *pend, *f = " "; - long n; - VALUE pad; + int cr; + str = StringValue(str); + cr = rb_enc_str_coderange(str); + if (cr == ENC_CODERANGE_BROKEN) { + rb_raise(rb_eArgError, "replacement must be valid byte sequence '%+"PRIsVALUE"'", str); + } + else { + rb_encoding *e = STR_ENC_GET(str); + if (cr == ENC_CODERANGE_7BIT ? rb_enc_mbminlen(enc) != 1 : enc != e) { + rb_raise(rb_eEncCompatError, "incompatible character encodings: %s and %s", + rb_enc_inspect_name(enc), rb_enc_inspect_name(e)); + } + } + return str; +} + +static VALUE enc_str_scrub(rb_encoding *enc, VALUE str, VALUE repl, int cr); - if (rb_scan_args(argc, argv, "11", &w, &pad) == 2) { - if (!NIL_P(pad)) { - StringValue(pad); - if (RSTRING(pad)->len > 0) { - f = RSTRING(pad)->ptr; - flen = RSTRING(pad)->len; +VALUE +rb_str_scrub(VALUE str, VALUE repl) +{ + rb_encoding *enc = STR_ENC_GET(str); + return enc_str_scrub(enc, str, repl, ENC_CODERANGE(str)); +} + +VALUE +rb_enc_str_scrub(rb_encoding *enc, VALUE str, VALUE repl) +{ + int cr = ENC_CODERANGE_UNKNOWN; + if (enc == STR_ENC_GET(str)) { + /* cached coderange makes sense only when enc equals the + * actual encoding of str */ + cr = ENC_CODERANGE(str); + } + return enc_str_scrub(enc, str, repl, cr); +} + +static VALUE +enc_str_scrub(rb_encoding *enc, VALUE str, VALUE repl, int cr) +{ + int encidx; + VALUE buf = Qnil; + const char *rep, *p, *e, *p1, *sp; + long replen = -1; + long slen; + + if (rb_block_given_p()) { + if (!NIL_P(repl)) + rb_raise(rb_eArgError, "both of block and replacement given"); + replen = 0; + } + + if (ENC_CODERANGE_CLEAN_P(cr)) + return Qnil; + + if (!NIL_P(repl)) { + repl = str_compat_and_valid(repl, enc); + } + + if (rb_enc_dummy_p(enc)) { + return Qnil; + } + encidx = rb_enc_to_index(enc); + +#define DEFAULT_REPLACE_CHAR(str) do { \ + RBIMPL_ATTR_NONSTRING() static const char replace[sizeof(str)-1] = str; \ + rep = replace; replen = (int)sizeof(replace); \ + } while (0) + + slen = RSTRING_LEN(str); + p = RSTRING_PTR(str); + e = RSTRING_END(str); + p1 = p; + sp = p; + + if (rb_enc_asciicompat(enc)) { + int rep7bit_p; + if (!replen) { + rep = NULL; + rep7bit_p = FALSE; + } + else if (!NIL_P(repl)) { + rep = RSTRING_PTR(repl); + replen = RSTRING_LEN(repl); + rep7bit_p = (ENC_CODERANGE(repl) == ENC_CODERANGE_7BIT); + } + else if (encidx == rb_utf8_encindex()) { + DEFAULT_REPLACE_CHAR("\xEF\xBF\xBD"); + rep7bit_p = FALSE; + } + else { + DEFAULT_REPLACE_CHAR("?"); + rep7bit_p = TRUE; + } + cr = ENC_CODERANGE_7BIT; + + p = search_nonascii(p, e); + if (!p) { + p = e; + } + while (p < e) { + int ret = rb_enc_precise_mbclen(p, e, enc); + if (MBCLEN_NEEDMORE_P(ret)) { + break; + } + else if (MBCLEN_CHARFOUND_P(ret)) { + cr = ENC_CODERANGE_VALID; + p += MBCLEN_CHARFOUND_LEN(ret); + } + else if (MBCLEN_INVALID_P(ret)) { + /* + * p1~p: valid ascii/multibyte chars + * p ~e: invalid bytes + unknown bytes + */ + long clen = rb_enc_mbmaxlen(enc); + if (NIL_P(buf)) buf = rb_str_buf_new(RSTRING_LEN(str)); + if (p > p1) { + rb_str_buf_cat(buf, p1, p - p1); + } + + if (e - p < clen) clen = e - p; + if (clen <= 2) { + clen = 1; + } + else { + const char *q = p; + clen--; + for (; clen > 1; clen--) { + ret = rb_enc_precise_mbclen(q, q + clen, enc); + if (MBCLEN_NEEDMORE_P(ret)) break; + if (MBCLEN_INVALID_P(ret)) continue; + UNREACHABLE; + } + } + if (rep) { + rb_str_buf_cat(buf, rep, replen); + if (!rep7bit_p) cr = ENC_CODERANGE_VALID; + } + else { + repl = rb_yield(rb_enc_str_new(p, clen, enc)); + str_mod_check(str, sp, slen); + repl = str_compat_and_valid(repl, enc); + rb_str_buf_cat(buf, RSTRING_PTR(repl), RSTRING_LEN(repl)); + if (ENC_CODERANGE(repl) == ENC_CODERANGE_VALID) + cr = ENC_CODERANGE_VALID; + } + p += clen; + p1 = p; + p = search_nonascii(p, e); + if (!p) { + p = e; + break; + } + } + else { + UNREACHABLE; } } - } - width = NUM2LONG(w); - if (width < 0 || RSTRING(str)->len >= width) return rb_str_dup(str); - res = rb_str_new5(str, 0, width); - p = RSTRING(res)->ptr; - if (jflag != 'l') { - n = width - RSTRING(str)->len; - pend = p + ((jflag == 'r') ? n : n/2); - if (flen <= 1) { - while (p < pend) { - *p++ = *f; + if (NIL_P(buf)) { + if (p == e) { + ENC_CODERANGE_SET(str, cr); + return Qnil; } + buf = rb_str_buf_new(RSTRING_LEN(str)); } - else { - char *q = f; - while (p + flen <= pend) { - memcpy(p,f,flen); - p += flen; + if (p1 < p) { + rb_str_buf_cat(buf, p1, p - p1); + } + if (p < e) { + if (rep) { + rb_str_buf_cat(buf, rep, replen); + if (!rep7bit_p) cr = ENC_CODERANGE_VALID; } - while (p < pend) { - *p++ = *q++; + else { + repl = rb_yield(rb_enc_str_new(p, e-p, enc)); + str_mod_check(str, sp, slen); + repl = str_compat_and_valid(repl, enc); + rb_str_buf_cat(buf, RSTRING_PTR(repl), RSTRING_LEN(repl)); + if (ENC_CODERANGE(repl) == ENC_CODERANGE_VALID) + cr = ENC_CODERANGE_VALID; } } } - memcpy(p, RSTRING(str)->ptr, RSTRING(str)->len); - if (jflag != 'r') { - p += RSTRING(str)->len; pend = RSTRING(res)->ptr + width; - if (flen <= 1) { - while (p < pend) { - *p++ = *f; - } + else { + /* ASCII incompatible */ + long mbminlen = rb_enc_mbminlen(enc); + if (!replen) { + rep = NULL; + } + else if (!NIL_P(repl)) { + rep = RSTRING_PTR(repl); + replen = RSTRING_LEN(repl); + } + else if (encidx == ENCINDEX_UTF_16BE) { + DEFAULT_REPLACE_CHAR("\xFF\xFD"); + } + else if (encidx == ENCINDEX_UTF_16LE) { + DEFAULT_REPLACE_CHAR("\xFD\xFF"); + } + else if (encidx == ENCINDEX_UTF_32BE) { + DEFAULT_REPLACE_CHAR("\x00\x00\xFF\xFD"); + } + else if (encidx == ENCINDEX_UTF_32LE) { + DEFAULT_REPLACE_CHAR("\xFD\xFF\x00\x00"); } else { - while (p + flen <= pend) { - memcpy(p,f,flen); - p += flen; + DEFAULT_REPLACE_CHAR("?"); + } + + while (p < e) { + int ret = rb_enc_precise_mbclen(p, e, enc); + if (MBCLEN_NEEDMORE_P(ret)) { + break; + } + else if (MBCLEN_CHARFOUND_P(ret)) { + p += MBCLEN_CHARFOUND_LEN(ret); + } + else if (MBCLEN_INVALID_P(ret)) { + const char *q = p; + long clen = rb_enc_mbmaxlen(enc); + if (NIL_P(buf)) buf = rb_str_buf_new(RSTRING_LEN(str)); + if (p > p1) rb_str_buf_cat(buf, p1, p - p1); + + if (e - p < clen) clen = e - p; + if (clen <= mbminlen * 2) { + clen = mbminlen; + } + else { + clen -= mbminlen; + for (; clen > mbminlen; clen-=mbminlen) { + ret = rb_enc_precise_mbclen(q, q + clen, enc); + if (MBCLEN_NEEDMORE_P(ret)) break; + if (MBCLEN_INVALID_P(ret)) continue; + UNREACHABLE; + } + } + if (rep) { + rb_str_buf_cat(buf, rep, replen); + } + else { + repl = rb_yield(rb_enc_str_new(p, clen, enc)); + str_mod_check(str, sp, slen); + repl = str_compat_and_valid(repl, enc); + rb_str_buf_cat(buf, RSTRING_PTR(repl), RSTRING_LEN(repl)); + } + p += clen; + p1 = p; + } + else { + UNREACHABLE; + } + } + if (NIL_P(buf)) { + if (p == e) { + ENC_CODERANGE_SET(str, ENC_CODERANGE_VALID); + return Qnil; + } + buf = rb_str_buf_new(RSTRING_LEN(str)); + } + if (p1 < p) { + rb_str_buf_cat(buf, p1, p - p1); + } + if (p < e) { + if (rep) { + rb_str_buf_cat(buf, rep, replen); } - while (p < pend) { - *p++ = *f++; + else { + repl = rb_yield(rb_enc_str_new(p, e-p, enc)); + str_mod_check(str, sp, slen); + repl = str_compat_and_valid(repl, enc); + rb_str_buf_cat(buf, RSTRING_PTR(repl), RSTRING_LEN(repl)); } } + cr = ENC_CODERANGE_VALID; } - OBJ_INFECT(res, str); - if (flen > 0) OBJ_INFECT(res, pad); - return res; + ENCODING_CODERANGE_SET(buf, rb_enc_to_index(enc), cr); + return buf; } +/* + * call-seq: + * scrub(replacement_string = default_replacement_string) -> new_string + * scrub{|sequence| ... } -> new_string + * + * :include: doc/string/scrub.rdoc + * + */ +static VALUE +str_scrub(int argc, VALUE *argv, VALUE str) +{ + VALUE repl = argc ? (rb_check_arity(argc, 0, 1), argv[0]) : Qnil; + VALUE new = rb_str_scrub(str, repl); + return NIL_P(new) ? str_duplicate(rb_cString, str): new; +} /* * call-seq: - * str.ljust(integer) => new_str - * - * If <i>integer</i> is greater than the length of <i>str</i>, returns a new - * <code>String</code> of length <i>integer</i> with <i>str</i> left justified - * and space padded; otherwise, returns <i>str</i>. - * - * "hello".ljust(4) #=> "hello" - * "hello".ljust(20) #=> "hello " + * scrub!(replacement_string = default_replacement_string) -> self + * scrub!{|sequence| ... } -> self + * + * Like String#scrub, except that: + * + * - Any replacements are made in +self+. + * - Returns +self+. + * + * Related: see {Modifying}[rdoc-ref:String@Modifying]. + * */ +static VALUE +str_scrub_bang(int argc, VALUE *argv, VALUE str) +{ + VALUE repl = argc ? (rb_check_arity(argc, 0, 1), argv[0]) : Qnil; + VALUE new = rb_str_scrub(str, repl); + if (!NIL_P(new)) rb_str_replace(str, new); + return str; +} + +static ID id_normalize; +static ID id_normalized_p; +static VALUE mUnicodeNormalize; static VALUE -rb_str_ljust(argc, argv, str) - int argc; - VALUE *argv; - VALUE str; +unicode_normalize_common(int argc, VALUE *argv, VALUE str, ID id) { - return rb_str_justify(argc, argv, str, 'l'); + static int UnicodeNormalizeRequired = 0; + VALUE argv2[2]; + + if (!UnicodeNormalizeRequired) { + rb_require("unicode_normalize/normalize.rb"); + UnicodeNormalizeRequired = 1; + } + argv2[0] = str; + if (rb_check_arity(argc, 0, 1)) argv2[1] = argv[0]; + return rb_funcallv(mUnicodeNormalize, id, argc+1, argv2); } +/* + * call-seq: + * unicode_normalize(form = :nfc) -> string + * + * :include: doc/string/unicode_normalize.rdoc + * + */ +static VALUE +rb_str_unicode_normalize(int argc, VALUE *argv, VALUE str) +{ + return unicode_normalize_common(argc, argv, str, id_normalize); +} /* * call-seq: - * str.rjust(integer) => new_str - * - * If <i>integer</i> is greater than the length of <i>str</i>, returns a new - * <code>String</code> of length <i>integer</i> with <i>str</i> right justified - * and space padded; otherwise, returns <i>str</i>. - * - * "hello".rjust(4) #=> "hello" - * "hello".rjust(20) #=> " hello" + * unicode_normalize!(form = :nfc) -> self + * + * Like String#unicode_normalize, except that the normalization + * is performed on +self+ (not on a copy of +self+). + * + * Related: see {Modifying}[rdoc-ref:String@Modifying]. + * */ +static VALUE +rb_str_unicode_normalize_bang(int argc, VALUE *argv, VALUE str) +{ + return rb_str_replace(str, unicode_normalize_common(argc, argv, str, id_normalize)); +} +/* call-seq: + * unicode_normalized?(form = :nfc) -> true or false + * + * Returns whether +self+ is in the given +form+ of Unicode normalization; + * see String#unicode_normalize. + * + * The +form+ must be one of +:nfc+, +:nfd+, +:nfkc+, or +:nfkd+. + * + * Examples: + * + * "a\u0300".unicode_normalized? # => false + * "a\u0300".unicode_normalized?(:nfd) # => true + * "\u00E0".unicode_normalized? # => true + * "\u00E0".unicode_normalized?(:nfd) # => false + * + * + * Raises an exception if +self+ is not in a Unicode encoding: + * + * s = "\xE0".force_encoding(Encoding::ISO_8859_1) + * s.unicode_normalized? # Raises Encoding::CompatibilityError + * + * Related: see {Querying}[rdoc-ref:String@Querying]. + */ static VALUE -rb_str_rjust(argc, argv, str) - int argc; - VALUE *argv; - VALUE str; +rb_str_unicode_normalized_p(int argc, VALUE *argv, VALUE str) { - return rb_str_justify(argc, argv, str, 'r'); + return unicode_normalize_common(argc, argv, str, id_normalized_p); +} + +/********************************************************************** + * Document-class: Symbol + * + * A +Symbol+ object represents a named identifier inside the Ruby interpreter. + * + * You can create a +Symbol+ object explicitly with: + * + * - A {symbol literal}[rdoc-ref:syntax/literals.rdoc@Symbol+Literals]. + * + * The same +Symbol+ object will be + * created for a given name or string for the duration of a program's + * execution, regardless of the context or meaning of that name. Thus + * if <code>Fred</code> is a constant in one context, a method in + * another, and a class in a third, the +Symbol+ <code>:Fred</code> + * will be the same object in all three contexts. + * + * module One + * class Fred + * end + * $f1 = :Fred + * end + * module Two + * Fred = 1 + * $f2 = :Fred + * end + * def Fred() + * end + * $f3 = :Fred + * $f1.object_id #=> 2514190 + * $f2.object_id #=> 2514190 + * $f3.object_id #=> 2514190 + * + * Constant, method, and variable names are returned as symbols: + * + * module One + * Two = 2 + * def three; 3 end + * @four = 4 + * @@five = 5 + * $six = 6 + * end + * seven = 7 + * + * One.constants + * # => [:Two] + * One.instance_methods(true) + * # => [:three] + * One.instance_variables + * # => [:@four] + * One.class_variables + * # => [:@@five] + * global_variables.grep(/six/) + * # => [:$six] + * local_variables + * # => [:seven] + * + * A +Symbol+ object differs from a String object in that + * a +Symbol+ object represents an identifier, while a String object + * represents text or data. + * + * == What's Here + * + * First, what's elsewhere. Class +Symbol+: + * + * - Inherits from {class Object}[rdoc-ref:Object@What-27s+Here]. + * - Includes {module Comparable}[rdoc-ref:Comparable@What-27s+Here]. + * + * Here, class +Symbol+ provides methods that are useful for: + * + * - {Querying}[rdoc-ref:Symbol@Methods+for+Querying] + * - {Comparing}[rdoc-ref:Symbol@Methods+for+Comparing] + * - {Converting}[rdoc-ref:Symbol@Methods+for+Converting] + * + * === Methods for Querying + * + * - ::all_symbols: Returns an array of the symbols currently in Ruby's symbol table. + * - #=~: Returns the index of the first substring in symbol that matches a + * given Regexp or other object; returns +nil+ if no match is found. + * - #[], #slice : Returns a substring of symbol + * determined by a given index, start/length, or range, or string. + * - #empty?: Returns +true+ if +self.length+ is zero; +false+ otherwise. + * - #encoding: Returns the Encoding object that represents the encoding + * of symbol. + * - #end_with?: Returns +true+ if symbol ends with + * any of the given strings. + * - #match: Returns a MatchData object if symbol + * matches a given Regexp; +nil+ otherwise. + * - #match?: Returns +true+ if symbol + * matches a given Regexp; +false+ otherwise. + * - #length, #size: Returns the number of characters in symbol. + * - #start_with?: Returns +true+ if symbol starts with + * any of the given strings. + * + * === Methods for Comparing + * + * - #<=>: Returns -1, 0, or 1 as a given symbol is smaller than, equal to, + * or larger than symbol. + * - #==, #===: Returns +true+ if a given symbol has the same content and + * encoding. + * - #casecmp: Ignoring case, returns -1, 0, or 1 as a given + * symbol is smaller than, equal to, or larger than symbol. + * - #casecmp?: Returns +true+ if symbol is equal to a given symbol + * after Unicode case folding; +false+ otherwise. + * + * === Methods for Converting + * + * - #capitalize: Returns symbol with the first character upcased + * and all other characters downcased. + * - #downcase: Returns symbol with all characters downcased. + * - #inspect: Returns the string representation of +self+ as a symbol literal. + * - #name: Returns the frozen string corresponding to symbol. + * - #succ, #next: Returns the symbol that is the successor to symbol. + * - #swapcase: Returns symbol with all upcase characters downcased + * and all downcase characters upcased. + * - #to_proc: Returns a Proc object which responds to the method named by symbol. + * - #to_s, #id2name: Returns the string corresponding to +self+. + * - #to_sym, #intern: Returns +self+. + * - #upcase: Returns symbol with all characters upcased. + * + */ + + +/* + * call-seq: + * self == other -> true or false + * + * Returns whether +other+ is the same object as +self+. + */ + +#define sym_equal rb_obj_equal + +static int +sym_printable(const char *s, const char *send, rb_encoding *enc) +{ + while (s < send) { + int n; + int c = rb_enc_precise_mbclen(s, send, enc); + + if (!MBCLEN_CHARFOUND_P(c)) return FALSE; + n = MBCLEN_CHARFOUND_LEN(c); + c = rb_enc_mbc_to_codepoint(s, send, enc); + if (!rb_enc_isprint(c, enc)) return FALSE; + s += n; + } + return TRUE; +} + +int +rb_str_symname_p(VALUE sym) +{ + rb_encoding *enc; + const char *ptr; + long len; + rb_encoding *resenc = rb_default_internal_encoding(); + + if (resenc == NULL) resenc = rb_default_external_encoding(); + enc = STR_ENC_GET(sym); + ptr = RSTRING_PTR(sym); + len = RSTRING_LEN(sym); + if ((resenc != enc && !rb_str_is_ascii_only_p(sym)) || len != (long)strlen(ptr) || + !rb_enc_symname2_p(ptr, len, enc) || !sym_printable(ptr, ptr + len, enc)) { + return FALSE; + } + return TRUE; } +VALUE +rb_str_quote_unprintable(VALUE str) +{ + rb_encoding *enc; + const char *ptr; + long len; + rb_encoding *resenc; + + Check_Type(str, T_STRING); + resenc = rb_default_internal_encoding(); + if (resenc == NULL) resenc = rb_default_external_encoding(); + enc = STR_ENC_GET(str); + ptr = RSTRING_PTR(str); + len = RSTRING_LEN(str); + if ((resenc != enc && !rb_str_is_ascii_only_p(str)) || + !sym_printable(ptr, ptr + len, enc)) { + return rb_str_escape(str); + } + return str; +} + +VALUE +rb_id_quote_unprintable(ID id) +{ + VALUE str = rb_id2str(id); + if (!rb_str_symname_p(str)) { + return rb_str_escape(str); + } + return str; +} /* * call-seq: - * str.center(integer) => new_str - * - * If <i>integer</i> is greater than the length of <i>str</i>, returns a new - * <code>String</code> of length <i>integer</i> with <i>str</i> centered - * between spaces; otherwise, returns <i>str</i>. - * - * "hello".center(4) #=> "hello" - * "hello".center(20) #=> " hello " + * inspect -> string + * + * Returns a string representation of +self+ (including the leading colon): + * + * :foo.inspect # => ":foo" + * + * Related: Symbol#to_s, Symbol#name. + * */ static VALUE -rb_str_center(argc, argv, str) - int argc; - VALUE *argv; - VALUE str; +sym_inspect(VALUE sym) { - return rb_str_justify(argc, argv, str, 'c'); + VALUE str = rb_sym2str(sym); + const char *ptr; + long len; + char *dest; + + if (!rb_str_symname_p(str)) { + str = rb_str_inspect(str); + len = RSTRING_LEN(str); + rb_str_resize(str, len + 1); + dest = RSTRING_PTR(str); + memmove(dest + 1, dest, len); + } + else { + rb_encoding *enc = STR_ENC_GET(str); + VALUE orig_str = str; + + len = RSTRING_LEN(orig_str); + str = rb_enc_str_new(0, len + 1, enc); + + // Get data pointer after allocation + ptr = RSTRING_PTR(orig_str); + dest = RSTRING_PTR(str); + memcpy(dest + 1, ptr, len); + + RB_GC_GUARD(orig_str); + } + dest[0] = ':'; + + RUBY_ASSERT_BUILTIN_TYPE(str, T_STRING); + + return str; } -void -rb_str_setter(val, id, var) - VALUE val; - ID id; - VALUE *var; +VALUE +rb_sym_to_s(VALUE sym) { - if (!NIL_P(val) && TYPE(val) != T_STRING) { - rb_raise(rb_eTypeError, "value of %s must be String", rb_id2name(id)); + VALUE str = str_new_shared(rb_cString, rb_sym2str(sym)); + FL_SET_RAW(str, STR_CHILLED_SYMBOL_TO_S); + return str; +} + +VALUE +rb_sym_proc_call(ID mid, int argc, const VALUE *argv, int kw_splat, VALUE passed_proc) +{ + VALUE obj; + + if (argc < 1) { + rb_raise(rb_eArgError, "no receiver given"); } - *var = val; + obj = argv[0]; + return rb_funcall_with_block_kw(obj, mid, argc - 1, argv + 1, passed_proc, kw_splat); } +/* + * call-seq: + * succ + * + * Equivalent to <tt>self.to_s.succ.to_sym</tt>: + * + * :foo.succ # => :fop + * + * Related: String#succ. + */ + +static VALUE +sym_succ(VALUE sym) +{ + return rb_str_intern(rb_str_succ(rb_sym2str(sym))); +} /* - * A <code>String</code> object holds and manipulates an arbitrary sequence of - * bytes, typically representing characters. String objects may be created - * using <code>String::new</code> or as literals. - * - * Because of aliasing issues, users of strings should be aware of the methods - * that modify the contents of a <code>String</code> object. Typically, - * methods with names ending in ``!'' modify their receiver, while those - * without a ``!'' return a new <code>String</code>. However, there are - * exceptions, such as <code>String#[]=</code>. - * + * call-seq: + * self <=> other -> -1, 0, 1, or nil + * + * Compares +self+ and +other+, using String#<=>. + * + * Returns: + * + * - <tt>self.to_s <=> other.to_s</tt>, if +other+ is a symbol. + * - +nil+, otherwise. + * + * Examples: + * + * :bar <=> :foo # => -1 + * :foo <=> :foo # => 0 + * :foo <=> :bar # => 1 + * :foo <=> 'bar' # => nil + * + * \Class \Symbol includes module Comparable, + * each of whose methods uses Symbol#<=> for comparison. + * + * Related: String#<=>. + */ + +static VALUE +sym_cmp(VALUE sym, VALUE other) +{ + if (!SYMBOL_P(other)) { + return Qnil; + } + return rb_str_cmp_m(rb_sym2str(sym), rb_sym2str(other)); +} + +/* + * call-seq: + * casecmp(object) -> -1, 0, 1, or nil + * + * :include: doc/symbol/casecmp.rdoc + * + */ + +static VALUE +sym_casecmp(VALUE sym, VALUE other) +{ + if (!SYMBOL_P(other)) { + return Qnil; + } + return str_casecmp(rb_sym2str(sym), rb_sym2str(other)); +} + +/* + * call-seq: + * casecmp?(object) -> true, false, or nil + * + * :include: doc/symbol/casecmp_p.rdoc + * + */ + +static VALUE +sym_casecmp_p(VALUE sym, VALUE other) +{ + if (!SYMBOL_P(other)) { + return Qnil; + } + return str_casecmp_p(rb_sym2str(sym), rb_sym2str(other)); +} + +/* + * call-seq: + * self =~ other -> integer or nil + * + * Equivalent to <tt>self.to_s =~ other</tt>, + * including possible updates to global variables; + * see String#=~. + * + */ + +static VALUE +sym_match(VALUE sym, VALUE other) +{ + return rb_str_match(rb_sym2str(sym), other); +} + +/* + * call-seq: + * match(pattern, offset = 0) -> matchdata or nil + * match(pattern, offset = 0) {|matchdata| } -> object + * + * Equivalent to <tt>self.to_s.match</tt>, + * including possible updates to global variables; + * see String#match. + * + */ + +static VALUE +sym_match_m(int argc, VALUE *argv, VALUE sym) +{ + return rb_str_match_m(argc, argv, rb_sym2str(sym)); +} + +/* + * call-seq: + * match?(pattern, offset) -> true or false + * + * Equivalent to <tt>sym.to_s.match?</tt>; + * see String#match. + * + */ + +static VALUE +sym_match_m_p(int argc, VALUE *argv, VALUE sym) +{ + return rb_str_match_m_p(argc, argv, sym); +} + +/* + * call-seq: + * self[offset] -> string or nil + * self[offset, size] -> string or nil + * self[range] -> string or nil + * self[regexp, capture = 0] -> string or nil + * self[substring] -> string or nil + * + * Equivalent to <tt>symbol.to_s[]</tt>; see String#[]. + * + */ + +static VALUE +sym_aref(int argc, VALUE *argv, VALUE sym) +{ + return rb_str_aref_m(argc, argv, rb_sym2str(sym)); +} + +/* + * call-seq: + * length -> integer + * + * Equivalent to <tt>self.to_s.length</tt>; see String#length. + */ + +static VALUE +sym_length(VALUE sym) +{ + return rb_str_length(rb_sym2str(sym)); +} + +/* + * call-seq: + * empty? -> true or false + * + * Returns +true+ if +self+ is <tt>:''</tt>, +false+ otherwise. + * + */ + +static VALUE +sym_empty(VALUE sym) +{ + return rb_str_empty(rb_sym2str(sym)); +} + +/* + * call-seq: + * upcase(mapping) -> symbol + * + * Equivalent to <tt>sym.to_s.upcase.to_sym</tt>. + * + * See String#upcase. + * + */ + +static VALUE +sym_upcase(int argc, VALUE *argv, VALUE sym) +{ + return rb_str_intern(rb_str_upcase(argc, argv, rb_sym2str(sym))); +} + +/* + * call-seq: + * downcase(mapping) -> symbol + * + * Equivalent to <tt>sym.to_s.downcase.to_sym</tt>. + * + * See String#downcase. + * + * Related: Symbol#upcase. + * + */ + +static VALUE +sym_downcase(int argc, VALUE *argv, VALUE sym) +{ + return rb_str_intern(rb_str_downcase(argc, argv, rb_sym2str(sym))); +} + +/* + * call-seq: + * capitalize(mapping) -> symbol + * + * Equivalent to <tt>sym.to_s.capitalize.to_sym</tt>. + * + * See String#capitalize. + * + */ + +static VALUE +sym_capitalize(int argc, VALUE *argv, VALUE sym) +{ + return rb_str_intern(rb_str_capitalize(argc, argv, rb_sym2str(sym))); +} + +/* + * call-seq: + * swapcase(mapping) -> symbol + * + * Equivalent to <tt>sym.to_s.swapcase.to_sym</tt>. + * + * See String#swapcase. + * + */ + +static VALUE +sym_swapcase(int argc, VALUE *argv, VALUE sym) +{ + return rb_str_intern(rb_str_swapcase(argc, argv, rb_sym2str(sym))); +} + +/* + * call-seq: + * start_with?(*string_or_regexp) -> true or false + * + * Equivalent to <tt>self.to_s.start_with?</tt>; see String#start_with?. + * + */ + +static VALUE +sym_start_with(int argc, VALUE *argv, VALUE sym) +{ + return rb_str_start_with(argc, argv, rb_sym2str(sym)); +} + +/* + * call-seq: + * end_with?(*strings) -> true or false + * + * + * Equivalent to <tt>self.to_s.end_with?</tt>; see String#end_with?. + * + */ + +static VALUE +sym_end_with(int argc, VALUE *argv, VALUE sym) +{ + return rb_str_end_with(argc, argv, rb_sym2str(sym)); +} + +/* + * call-seq: + * encoding -> encoding + * + * Equivalent to <tt>self.to_s.encoding</tt>; see String#encoding. + * + */ + +static VALUE +sym_encoding(VALUE sym) +{ + return rb_obj_encoding(rb_sym2str(sym)); +} + +static VALUE +string_for_symbol(VALUE name) +{ + if (!RB_TYPE_P(name, T_STRING)) { + VALUE tmp = rb_check_string_type(name); + if (NIL_P(tmp)) { + rb_raise(rb_eTypeError, "%+"PRIsVALUE" is not a symbol nor a string", + name); + } + name = tmp; + } + return name; +} + +ID +rb_to_id(VALUE name) +{ + if (SYMBOL_P(name)) { + return SYM2ID(name); + } + name = string_for_symbol(name); + return rb_intern_str(name); +} + +VALUE +rb_to_symbol(VALUE name) +{ + if (SYMBOL_P(name)) { + return name; + } + name = string_for_symbol(name); + return rb_str_intern(name); +} + +/* + * call-seq: + * Symbol.all_symbols -> array_of_symbols + * + * Returns an array of all symbols currently in Ruby's symbol table: + * + * Symbol.all_symbols.size # => 9334 + * Symbol.all_symbols.take(3) # => [:!, :"\"", :"#"] + * */ +static VALUE +sym_all_symbols(VALUE _) +{ + return rb_sym_all_symbols(); +} + +VALUE +rb_str_to_interned_str(VALUE str) +{ + return rb_fstring(str); +} + +VALUE +rb_interned_str(const char *ptr, long len) +{ + struct RString fake_str = {RBASIC_INIT}; + int encidx = ENCINDEX_US_ASCII; + int coderange = ENC_CODERANGE_7BIT; + if (len > 0 && search_nonascii(ptr, ptr + len)) { + encidx = ENCINDEX_ASCII_8BIT; + coderange = ENC_CODERANGE_VALID; + } + VALUE str = setup_fake_str(&fake_str, ptr, len, encidx); + ENC_CODERANGE_SET(str, coderange); + return register_fstring(str, true, false); +} + +VALUE +rb_interned_str_cstr(const char *ptr) +{ + return rb_interned_str(ptr, strlen(ptr)); +} + +VALUE +rb_enc_interned_str(const char *ptr, long len, rb_encoding *enc) +{ + if (enc != NULL && UNLIKELY(rb_enc_autoload_p(enc))) { + rb_enc_autoload(enc); + } + + struct RString fake_str = {RBASIC_INIT}; + return register_fstring(rb_setup_fake_str(&fake_str, ptr, len, enc), true, false); +} + +VALUE +rb_enc_literal_str(const char *ptr, long len, rb_encoding *enc) +{ + if (enc != NULL && UNLIKELY(rb_enc_autoload_p(enc))) { + rb_enc_autoload(enc); + } + + struct RString fake_str = {RBASIC_INIT}; + VALUE str = register_fstring(rb_setup_fake_str(&fake_str, ptr, len, enc), true, true); + RUBY_ASSERT(RB_OBJ_SHAREABLE_P(str) && (rb_gc_verify_shareable(str), 1)); + return str; +} + +VALUE +rb_enc_interned_str_cstr(const char *ptr, rb_encoding *enc) +{ + return rb_enc_interned_str(ptr, strlen(ptr), enc); +} + +#if USE_YJIT || USE_ZJIT void -Init_String() +rb_jit_str_concat_codepoint(VALUE str, VALUE codepoint) +{ + if (RB_LIKELY(ENCODING_GET_INLINED(str) == rb_ascii8bit_encindex())) { + ssize_t code = RB_NUM2SSIZE(codepoint); + + if (RB_LIKELY(code >= 0 && code < 0xff)) { + rb_str_buf_cat_byte(str, (char) code); + return; + } + } + + rb_str_concat(str, codepoint); +} +#endif + +static int +fstring_set_class_i(VALUE *str, void *data) +{ + RBASIC_SET_CLASS(*str, rb_cString); + + return ST_CONTINUE; +} + +void +Init_String(void) { rb_cString = rb_define_class("String", rb_cObject); + + rb_concurrent_set_foreach_with_replace(fstring_table_obj, fstring_set_class_i, NULL); + rb_include_module(rb_cString, rb_mComparable); - rb_include_module(rb_cString, rb_mEnumerable); - rb_define_alloc_func(rb_cString, str_alloc); + rb_define_alloc_func(rb_cString, empty_str_alloc); + rb_define_singleton_method(rb_cString, "new", rb_str_s_new, -1); + rb_define_singleton_method(rb_cString, "try_convert", rb_str_s_try_convert, 1); rb_define_method(rb_cString, "initialize", rb_str_init, -1); - rb_define_method(rb_cString, "initialize_copy", rb_str_replace, 1); + rb_define_method(rb_cString, "replace", rb_str_replace, 1); + rb_define_method(rb_cString, "initialize_copy", rb_str_replace, 1); rb_define_method(rb_cString, "<=>", rb_str_cmp_m, 1); rb_define_method(rb_cString, "==", rb_str_equal, 1); + rb_define_method(rb_cString, "===", rb_str_equal, 1); rb_define_method(rb_cString, "eql?", rb_str_eql, 1); rb_define_method(rb_cString, "hash", rb_str_hash_m, 0); rb_define_method(rb_cString, "casecmp", rb_str_casecmp, 1); + rb_define_method(rb_cString, "casecmp?", rb_str_casecmp_p, 1); rb_define_method(rb_cString, "+", rb_str_plus, 1); rb_define_method(rb_cString, "*", rb_str_times, 1); - rb_define_method(rb_cString, "%", rb_str_format, 1); + rb_define_method(rb_cString, "%", rb_str_format_m, 1); rb_define_method(rb_cString, "[]", rb_str_aref_m, -1); rb_define_method(rb_cString, "[]=", rb_str_aset_m, -1); rb_define_method(rb_cString, "insert", rb_str_insert, 2); rb_define_method(rb_cString, "length", rb_str_length, 0); rb_define_method(rb_cString, "size", rb_str_length, 0); + rb_define_method(rb_cString, "bytesize", rb_str_bytesize, 0); rb_define_method(rb_cString, "empty?", rb_str_empty, 0); rb_define_method(rb_cString, "=~", rb_str_match, 1); - rb_define_method(rb_cString, "~", rb_str_match2, 0); - rb_define_method(rb_cString, "match", rb_str_match_m, 1); + rb_define_method(rb_cString, "match", rb_str_match_m, -1); + rb_define_method(rb_cString, "match?", rb_str_match_m_p, -1); rb_define_method(rb_cString, "succ", rb_str_succ, 0); rb_define_method(rb_cString, "succ!", rb_str_succ_bang, 0); rb_define_method(rb_cString, "next", rb_str_succ, 0); rb_define_method(rb_cString, "next!", rb_str_succ_bang, 0); - rb_define_method(rb_cString, "upto", rb_str_upto_m, 1); + rb_define_method(rb_cString, "upto", rb_str_upto, -1); rb_define_method(rb_cString, "index", rb_str_index_m, -1); + rb_define_method(rb_cString, "byteindex", rb_str_byteindex_m, -1); rb_define_method(rb_cString, "rindex", rb_str_rindex_m, -1); - rb_define_method(rb_cString, "replace", rb_str_replace, 1); + rb_define_method(rb_cString, "byterindex", rb_str_byterindex_m, -1); + rb_define_method(rb_cString, "clear", rb_str_clear, 0); + rb_define_method(rb_cString, "chr", rb_str_chr, 0); + rb_define_method(rb_cString, "getbyte", rb_str_getbyte, 1); + rb_define_method(rb_cString, "setbyte", rb_str_setbyte, 2); + rb_define_method(rb_cString, "byteslice", rb_str_byteslice, -1); + rb_define_method(rb_cString, "bytesplice", rb_str_bytesplice, -1); + rb_define_method(rb_cString, "scrub", str_scrub, -1); + rb_define_method(rb_cString, "scrub!", str_scrub_bang, -1); + rb_define_method(rb_cString, "freeze", rb_str_freeze, 0); + rb_define_method(rb_cString, "+@", str_uplus, 0); + rb_define_method(rb_cString, "-@", str_uminus, 0); + rb_define_method(rb_cString, "dup", rb_str_dup_m, 0); + rb_define_alias(rb_cString, "dedup", "-@"); rb_define_method(rb_cString, "to_i", rb_str_to_i, -1); rb_define_method(rb_cString, "to_f", rb_str_to_f, 0); @@ -4611,29 +12857,45 @@ Init_String() rb_define_method(rb_cString, "to_str", rb_str_to_s, 0); rb_define_method(rb_cString, "inspect", rb_str_inspect, 0); rb_define_method(rb_cString, "dump", rb_str_dump, 0); + rb_define_method(rb_cString, "undump", str_undump, 0); + + sym_ascii = ID2SYM(rb_intern_const("ascii")); + sym_turkic = ID2SYM(rb_intern_const("turkic")); + sym_lithuanian = ID2SYM(rb_intern_const("lithuanian")); + sym_fold = ID2SYM(rb_intern_const("fold")); - rb_define_method(rb_cString, "upcase", rb_str_upcase, 0); - rb_define_method(rb_cString, "downcase", rb_str_downcase, 0); - rb_define_method(rb_cString, "capitalize", rb_str_capitalize, 0); - rb_define_method(rb_cString, "swapcase", rb_str_swapcase, 0); + rb_define_method(rb_cString, "upcase", rb_str_upcase, -1); + rb_define_method(rb_cString, "downcase", rb_str_downcase, -1); + rb_define_method(rb_cString, "capitalize", rb_str_capitalize, -1); + rb_define_method(rb_cString, "swapcase", rb_str_swapcase, -1); - rb_define_method(rb_cString, "upcase!", rb_str_upcase_bang, 0); - rb_define_method(rb_cString, "downcase!", rb_str_downcase_bang, 0); - rb_define_method(rb_cString, "capitalize!", rb_str_capitalize_bang, 0); - rb_define_method(rb_cString, "swapcase!", rb_str_swapcase_bang, 0); + rb_define_method(rb_cString, "upcase!", rb_str_upcase_bang, -1); + rb_define_method(rb_cString, "downcase!", rb_str_downcase_bang, -1); + rb_define_method(rb_cString, "capitalize!", rb_str_capitalize_bang, -1); + rb_define_method(rb_cString, "swapcase!", rb_str_swapcase_bang, -1); rb_define_method(rb_cString, "hex", rb_str_hex, 0); rb_define_method(rb_cString, "oct", rb_str_oct, 0); rb_define_method(rb_cString, "split", rb_str_split_m, -1); + rb_define_method(rb_cString, "lines", rb_str_lines, -1); + rb_define_method(rb_cString, "bytes", rb_str_bytes, 0); + rb_define_method(rb_cString, "chars", rb_str_chars, 0); + rb_define_method(rb_cString, "codepoints", rb_str_codepoints, 0); + rb_define_method(rb_cString, "grapheme_clusters", rb_str_grapheme_clusters, 0); rb_define_method(rb_cString, "reverse", rb_str_reverse, 0); rb_define_method(rb_cString, "reverse!", rb_str_reverse_bang, 0); - rb_define_method(rb_cString, "concat", rb_str_concat, 1); + rb_define_method(rb_cString, "concat", rb_str_concat_multi, -1); + rb_define_method(rb_cString, "append_as_bytes", rb_str_append_as_bytes, -1); rb_define_method(rb_cString, "<<", rb_str_concat, 1); + rb_define_method(rb_cString, "prepend", rb_str_prepend_multi, -1); rb_define_method(rb_cString, "crypt", rb_str_crypt, 1); - rb_define_method(rb_cString, "intern", rb_str_intern, 0); - rb_define_method(rb_cString, "to_sym", rb_str_intern, 0); + rb_define_method(rb_cString, "intern", rb_str_intern, 0); /* in symbol.c */ + rb_define_method(rb_cString, "to_sym", rb_str_intern, 0); /* in symbol.c */ + rb_define_method(rb_cString, "ord", rb_str_ord, 0); rb_define_method(rb_cString, "include?", rb_str_include, 1); + rb_define_method(rb_cString, "start_with?", rb_str_start_with, -1); + rb_define_method(rb_cString, "end_with?", rb_str_end_with, -1); rb_define_method(rb_cString, "scan", rb_str_scan, 1); @@ -4645,17 +12907,21 @@ Init_String() rb_define_method(rb_cString, "gsub", rb_str_gsub, -1); rb_define_method(rb_cString, "chop", rb_str_chop, 0); rb_define_method(rb_cString, "chomp", rb_str_chomp, -1); - rb_define_method(rb_cString, "strip", rb_str_strip, 0); - rb_define_method(rb_cString, "lstrip", rb_str_lstrip, 0); - rb_define_method(rb_cString, "rstrip", rb_str_rstrip, 0); + rb_define_method(rb_cString, "strip", rb_str_strip, -1); + rb_define_method(rb_cString, "lstrip", rb_str_lstrip, -1); + rb_define_method(rb_cString, "rstrip", rb_str_rstrip, -1); + rb_define_method(rb_cString, "delete_prefix", rb_str_delete_prefix, 1); + rb_define_method(rb_cString, "delete_suffix", rb_str_delete_suffix, 1); rb_define_method(rb_cString, "sub!", rb_str_sub_bang, -1); rb_define_method(rb_cString, "gsub!", rb_str_gsub_bang, -1); rb_define_method(rb_cString, "chop!", rb_str_chop_bang, 0); rb_define_method(rb_cString, "chomp!", rb_str_chomp_bang, -1); - rb_define_method(rb_cString, "strip!", rb_str_strip_bang, 0); - rb_define_method(rb_cString, "lstrip!", rb_str_lstrip_bang, 0); - rb_define_method(rb_cString, "rstrip!", rb_str_rstrip_bang, 0); + rb_define_method(rb_cString, "strip!", rb_str_strip_bang, -1); + rb_define_method(rb_cString, "lstrip!", rb_str_lstrip_bang, -1); + rb_define_method(rb_cString, "rstrip!", rb_str_rstrip_bang, -1); + rb_define_method(rb_cString, "delete_prefix!", rb_str_delete_prefix_bang, 1); + rb_define_method(rb_cString, "delete_suffix!", rb_str_delete_suffix_bang, 1); rb_define_method(rb_cString, "tr", rb_str_tr, 2); rb_define_method(rb_cString, "tr_s", rb_str_tr_s, 2); @@ -4669,32 +12935,73 @@ Init_String() rb_define_method(rb_cString, "squeeze!", rb_str_squeeze_bang, -1); rb_define_method(rb_cString, "each_line", rb_str_each_line, -1); - rb_define_method(rb_cString, "each", rb_str_each_line, -1); rb_define_method(rb_cString, "each_byte", rb_str_each_byte, 0); + rb_define_method(rb_cString, "each_char", rb_str_each_char, 0); + rb_define_method(rb_cString, "each_codepoint", rb_str_each_codepoint, 0); + rb_define_method(rb_cString, "each_grapheme_cluster", rb_str_each_grapheme_cluster, 0); rb_define_method(rb_cString, "sum", rb_str_sum, -1); - rb_define_global_function("sub", rb_f_sub, -1); - rb_define_global_function("gsub", rb_f_gsub, -1); + rb_define_method(rb_cString, "slice", rb_str_aref_m, -1); + rb_define_method(rb_cString, "slice!", rb_str_slice_bang, -1); - rb_define_global_function("sub!", rb_f_sub_bang, -1); - rb_define_global_function("gsub!", rb_f_gsub_bang, -1); + rb_define_method(rb_cString, "partition", rb_str_partition, 1); + rb_define_method(rb_cString, "rpartition", rb_str_rpartition, 1); - rb_define_global_function("chop", rb_f_chop, 0); - rb_define_global_function("chop!", rb_f_chop_bang, 0); + rb_define_method(rb_cString, "encoding", rb_obj_encoding, 0); /* in encoding.c */ + rb_define_method(rb_cString, "force_encoding", rb_str_force_encoding, 1); + rb_define_method(rb_cString, "b", rb_str_b, 0); + rb_define_method(rb_cString, "valid_encoding?", rb_str_valid_encoding_p, 0); + rb_define_method(rb_cString, "ascii_only?", rb_str_is_ascii_only_p, 0); - rb_define_global_function("chomp", rb_f_chomp, -1); - rb_define_global_function("chomp!", rb_f_chomp_bang, -1); + /* define UnicodeNormalize module here so that we don't have to look it up */ + mUnicodeNormalize = rb_define_module("UnicodeNormalize"); + id_normalize = rb_intern_const("normalize"); + id_normalized_p = rb_intern_const("normalized?"); - rb_define_global_function("split", rb_f_split, -1); - rb_define_global_function("scan", rb_f_scan, 1); - - rb_define_method(rb_cString, "slice", rb_str_aref_m, -1); - rb_define_method(rb_cString, "slice!", rb_str_slice_bang, -1); - - id_to_s = rb_intern("to_s"); + rb_define_method(rb_cString, "unicode_normalize", rb_str_unicode_normalize, -1); + rb_define_method(rb_cString, "unicode_normalize!", rb_str_unicode_normalize_bang, -1); + rb_define_method(rb_cString, "unicode_normalized?", rb_str_unicode_normalized_p, -1); rb_fs = Qnil; - rb_define_variable("$;", &rb_fs); - rb_define_variable("$-F", &rb_fs); + rb_define_hooked_variable("$;", &rb_fs, 0, rb_fs_setter); + rb_define_hooked_variable("$-F", &rb_fs, 0, rb_fs_setter); + rb_gc_register_address(&rb_fs); + + rb_cSymbol = rb_define_class("Symbol", rb_cObject); + rb_include_module(rb_cSymbol, rb_mComparable); + rb_undef_alloc_func(rb_cSymbol); + rb_undef_method(CLASS_OF(rb_cSymbol), "new"); + rb_define_singleton_method(rb_cSymbol, "all_symbols", sym_all_symbols, 0); + + rb_define_method(rb_cSymbol, "==", sym_equal, 1); + rb_define_method(rb_cSymbol, "===", sym_equal, 1); + rb_define_method(rb_cSymbol, "inspect", sym_inspect, 0); + rb_define_method(rb_cSymbol, "to_proc", rb_sym_to_proc, 0); /* in proc.c */ + rb_define_method(rb_cSymbol, "succ", sym_succ, 0); + rb_define_method(rb_cSymbol, "next", sym_succ, 0); + + rb_define_method(rb_cSymbol, "<=>", sym_cmp, 1); + rb_define_method(rb_cSymbol, "casecmp", sym_casecmp, 1); + rb_define_method(rb_cSymbol, "casecmp?", sym_casecmp_p, 1); + rb_define_method(rb_cSymbol, "=~", sym_match, 1); + + rb_define_method(rb_cSymbol, "[]", sym_aref, -1); + rb_define_method(rb_cSymbol, "slice", sym_aref, -1); + rb_define_method(rb_cSymbol, "length", sym_length, 0); + rb_define_method(rb_cSymbol, "size", sym_length, 0); + rb_define_method(rb_cSymbol, "empty?", sym_empty, 0); + rb_define_method(rb_cSymbol, "match", sym_match_m, -1); + rb_define_method(rb_cSymbol, "match?", sym_match_m_p, -1); + + rb_define_method(rb_cSymbol, "upcase", sym_upcase, -1); + rb_define_method(rb_cSymbol, "downcase", sym_downcase, -1); + rb_define_method(rb_cSymbol, "capitalize", sym_capitalize, -1); + rb_define_method(rb_cSymbol, "swapcase", sym_swapcase, -1); + + rb_define_method(rb_cSymbol, "start_with?", sym_start_with, -1); + rb_define_method(rb_cSymbol, "end_with?", sym_end_with, -1); + + rb_define_method(rb_cSymbol, "encoding", sym_encoding, 0); } + |