diff options
Diffstat (limited to 'transcode.c')
| -rw-r--r-- | transcode.c | 1184 |
1 files changed, 451 insertions, 733 deletions
diff --git a/transcode.c b/transcode.c index f1d871e292..514c181699 100644 --- a/transcode.c +++ b/transcode.c @@ -9,48 +9,23 @@ **********************************************************************/ -#include "ruby/internal/config.h" - -#include <ctype.h> - -#include "internal.h" -#include "internal/array.h" -#include "internal/inits.h" -#include "internal/object.h" -#include "internal/string.h" -#include "internal/transcode.h" +#include "ruby/ruby.h" #include "ruby/encoding.h" - #include "transcode_data.h" -#include "id.h" - -#define ENABLE_ECONV_NEWLINE_OPTION 1 +#include <ctype.h> /* VALUE rb_cEncoding = rb_define_class("Encoding", rb_cObject); */ -static VALUE rb_eUndefinedConversionError; -static VALUE rb_eInvalidByteSequenceError; -static VALUE rb_eConverterNotFoundError; +VALUE rb_eUndefinedConversionError; +VALUE rb_eInvalidByteSequenceError; +VALUE rb_eConverterNotFoundError; VALUE rb_cEncodingConverter; -static ID id_destination_encoding; -static ID id_destination_encoding_name; -static ID id_error_bytes; -static ID id_error_char; -static ID id_incomplete_input; -static ID id_readagain_bytes; -static ID id_source_encoding; -static ID id_source_encoding_name; - static VALUE sym_invalid, sym_undef, sym_replace, sym_fallback; static VALUE sym_xml, sym_text, sym_attr; static VALUE sym_universal_newline; static VALUE sym_crlf_newline; static VALUE sym_cr_newline; -static VALUE sym_lf_newline; -#ifdef ENABLE_ECONV_NEWLINE_OPTION -static VALUE sym_newline, sym_universal, sym_crlf, sym_cr, sym_lf; -#endif static VALUE sym_partial_input; static VALUE sym_invalid_byte_sequence; @@ -129,21 +104,21 @@ typedef struct { struct rb_econv_t { int flags; - int started; /* bool */ - const char *source_encoding_name; const char *destination_encoding_name; + int started; + const unsigned char *replacement_str; size_t replacement_len; const char *replacement_enc; + int replacement_allocated; unsigned char *in_buf_start; unsigned char *in_data_start; unsigned char *in_data_end; unsigned char *in_buf_end; rb_econv_elem_t *elems; - int replacement_allocated; /* bool */ int num_allocated; int num_trans; int num_finished; @@ -175,7 +150,7 @@ struct rb_econv_t { typedef struct { const char *sname; const char *dname; - const char *lib; /* null means no need to load a library */ + const char *lib; /* null means means no need to load a library */ const rb_transcoder *transcoder; } transcoder_entry_t; @@ -230,8 +205,8 @@ rb_register_transcoder(const rb_transcoder *tr) entry = make_transcoder_entry(sname, dname); if (entry->transcoder) { - rb_raise(rb_eArgError, "transcoder from %s to %s has been already registered", - sname, dname); + rb_raise(rb_eArgError, "transcoder from %s to %s has been already registered", + sname, dname); } entry->transcoder = tr; @@ -246,18 +221,20 @@ declare_transcoder(const char *sname, const char *dname, const char *lib) entry->lib = lib; } +#define MAX_TRANSCODER_LIBNAME_LEN 64 static const char transcoder_lib_prefix[] = "enc/trans/"; void rb_declare_transcoder(const char *enc1, const char *enc2, const char *lib) { - if (!lib) { - rb_raise(rb_eArgError, "invalid library name - (null)"); + if (!lib || strlen(lib) > MAX_TRANSCODER_LIBNAME_LEN) { + rb_raise(rb_eArgError, "invalid library name - %s", + lib ? lib : "(null)"); } declare_transcoder(enc1, enc2, lib); } -#define encoding_equal(enc1, enc2) (STRCASECMP((enc1), (enc2)) == 0) +#define encoding_equal(enc1, enc2) (STRCASECMP(enc1, enc2) == 0) typedef struct search_path_queue_tag { struct search_path_queue_tag *next; @@ -377,8 +354,6 @@ transcode_search_path(const char *sname, const char *dname, return pathlen; /* is -1 if not found */ } -int rb_require_internal_silent(VALUE fname); - static const rb_transcoder * load_transcoder_entry(transcoder_entry_t *entry) { @@ -386,17 +361,18 @@ load_transcoder_entry(transcoder_entry_t *entry) return entry->transcoder; if (entry->lib) { - const char *const lib = entry->lib; - const size_t len = strlen(lib); - const size_t total_len = sizeof(transcoder_lib_prefix) - 1 + len; - const VALUE fn = rb_str_new(0, total_len); - char *const path = RSTRING_PTR(fn); + const char *lib = entry->lib; + size_t len = strlen(lib); + char path[sizeof(transcoder_lib_prefix) + MAX_TRANSCODER_LIBNAME_LEN]; + + entry->lib = NULL; + if (len > MAX_TRANSCODER_LIBNAME_LEN) + return NULL; memcpy(path, transcoder_lib_prefix, sizeof(transcoder_lib_prefix) - 1); - memcpy(path + sizeof(transcoder_lib_prefix) - 1, lib, len); - rb_str_set_len(fn, total_len); - OBJ_FREEZE(fn); - rb_require_internal_silent(fn); + memcpy(path + sizeof(transcoder_lib_prefix) - 1, lib, len + 1); + if (!rb_require(path)) + return NULL; } if (entry->transcoder) @@ -476,7 +452,7 @@ transcode_restartable0(const unsigned char **in_pos, unsigned char **out_pos, tc->recognized_len -= readagain_len; \ tc->readagain_len = readagain_len; \ } \ - return (ret); \ + return ret; \ resume_label ## num:; \ } while (0) #define SUSPEND_OBUF(num) \ @@ -536,7 +512,7 @@ transcode_restartable0(const unsigned char **in_pos, unsigned char **out_pos, while (1) { inchar_start = in_p; tc->recognized_len = 0; - next_table = tr->conv_tree_start; + next_table = tr->conv_tree_start; SUSPEND_AFTER_OUTPUT(24); @@ -556,7 +532,7 @@ transcode_restartable0(const unsigned char **in_pos, unsigned char **out_pos, #define BL_OFFSET(byte) (BL_BASE[2+(byte)-BL_MIN_BYTE]) #define BL_ACTION(byte) (BL_INFO[BL_OFFSET((byte))]) - next_byte = (unsigned char)*in_p++; + next_byte = (unsigned char)*in_p++; follow_byte: if (next_byte < BL_MIN_BYTE || BL_MAX_BYTE < next_byte) next_info = INVALID; @@ -564,8 +540,8 @@ transcode_restartable0(const unsigned char **in_pos, unsigned char **out_pos, next_info = (VALUE)BL_ACTION(next_byte); } follow_info: - switch (next_info & 0x1F) { - case NOMAP: + switch (next_info & 0x1F) { + case NOMAP: { const unsigned char *p = inchar_start; writebuf_off = 0; @@ -580,43 +556,43 @@ transcode_restartable0(const unsigned char **in_pos, unsigned char **out_pos, } } continue; - case 0x00: case 0x04: case 0x08: case 0x0C: - case 0x10: case 0x14: case 0x18: case 0x1C: + case 0x00: case 0x04: case 0x08: case 0x0C: + case 0x10: case 0x14: case 0x18: case 0x1C: SUSPEND_AFTER_OUTPUT(25); - while (in_p >= in_stop) { + while (in_p >= in_stop) { if (!(opt & ECONV_PARTIAL_INPUT)) goto incomplete; SUSPEND(econv_source_buffer_empty, 5); - } - next_byte = (unsigned char)*in_p++; - next_table = (unsigned int)next_info; - goto follow_byte; - case ZERObt: /* drop input */ - continue; - case ONEbt: + } + next_byte = (unsigned char)*in_p++; + next_table = (unsigned int)next_info; + goto follow_byte; + case ZERObt: /* drop input */ + continue; + case ONEbt: SUSPEND_OBUF(9); *out_p++ = getBT1(next_info); - continue; - case TWObt: + continue; + case TWObt: SUSPEND_OBUF(10); *out_p++ = getBT1(next_info); SUSPEND_OBUF(21); *out_p++ = getBT2(next_info); - continue; - case THREEbt: + continue; + case THREEbt: SUSPEND_OBUF(11); *out_p++ = getBT1(next_info); SUSPEND_OBUF(15); *out_p++ = getBT2(next_info); SUSPEND_OBUF(16); *out_p++ = getBT3(next_info); - continue; - case FOURbt: + continue; + case FOURbt: SUSPEND_OBUF(12); *out_p++ = getBT0(next_info); SUSPEND_OBUF(17); *out_p++ = getBT1(next_info); SUSPEND_OBUF(18); *out_p++ = getBT2(next_info); SUSPEND_OBUF(19); *out_p++ = getBT3(next_info); - continue; - case GB4bt: + continue; + case GB4bt: SUSPEND_OBUF(29); *out_p++ = getGB4bt0(next_info); SUSPEND_OBUF(30); *out_p++ = getGB4bt1(next_info); SUSPEND_OBUF(31); *out_p++ = getGB4bt2(next_info); SUSPEND_OBUF(32); *out_p++ = getGB4bt3(next_info); - continue; + continue; case STR1: tc->output_index = 0; while (tc->output_index < STR1_LENGTH(BYTE_ADDR(STR1_BYTEINDEX(next_info)))) { @@ -624,10 +600,10 @@ transcode_restartable0(const unsigned char **in_pos, unsigned char **out_pos, tc->output_index++; } continue; - case FUNii: - next_info = (VALUE)(*tr->func_ii)(TRANSCODING_STATE(tc), next_info); - goto follow_info; - case FUNsi: + case FUNii: + next_info = (VALUE)(*tr->func_ii)(TRANSCODING_STATE(tc), next_info); + goto follow_info; + case FUNsi: { const unsigned char *char_start; size_t char_len; @@ -635,7 +611,7 @@ transcode_restartable0(const unsigned char **in_pos, unsigned char **out_pos, next_info = (VALUE)(*tr->func_si)(TRANSCODING_STATE(tc), char_start, (size_t)char_len); goto follow_info; } - case FUNio: + case FUNio: SUSPEND_OBUF(13); if (tr->max_output <= out_stop - out_p) out_p += tr->func_io(TRANSCODING_STATE(tc), @@ -650,8 +626,8 @@ transcode_restartable0(const unsigned char **in_pos, unsigned char **out_pos, *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++]; } } - break; - case FUNso: + break; + case FUNso: { const unsigned char *char_start; size_t char_len; @@ -699,7 +675,7 @@ transcode_restartable0(const unsigned char **in_pos, unsigned char **out_pos, } break; } - case INVALID: + case INVALID: if (tc->recognized_len + (in_p - inchar_start) <= unitlen) { if (tc->recognized_len + (in_p - inchar_start) < unitlen) SUSPEND_AFTER_OUTPUT(26); @@ -722,12 +698,12 @@ transcode_restartable0(const unsigned char **in_pos, unsigned char **out_pos, readagain_len = invalid_len - discard_len; } goto invalid; - case UNDEF: - goto undef; - default: - rb_raise(rb_eRuntimeError, "unknown transcoding instruction"); - } - continue; + case UNDEF: + goto undef; + default: + rb_raise(rb_eRuntimeError, "unknown transcoding instruction"); + } + continue; invalid: SUSPEND(econv_invalid_byte_sequence, 1); @@ -857,13 +833,13 @@ rb_transcoding_memsize(rb_transcoding *tc) const rb_transcoder *tr = tc->transcoder; if (TRANSCODING_STATE_EMBED_MAX < tr->state_size) { - size += tr->state_size; + size += tr->state_size; } if ((int)sizeof(tc->readbuf.ary) < tr->max_input) { - size += tr->max_input; + size += tr->max_input; } if ((int)sizeof(tc->writebuf.ary) < tr->max_output) { - size += tr->max_output; + size += tr->max_output; } return size; } @@ -995,15 +971,28 @@ rb_econv_open0(const char *sname, const char *dname, int ecflags) int num_trans; rb_econv_t *ec; - /* Just check if sname and dname are defined */ - /* (This check is needed?) */ - if (*sname) rb_enc_find_index(sname); - if (*dname) rb_enc_find_index(dname); + rb_encoding *senc, *denc; + int sidx, didx; + + senc = NULL; + if (*sname) { + sidx = rb_enc_find_index(sname); + if (0 <= sidx) { + senc = rb_enc_from_index(sidx); + } + } + + denc = NULL; + if (*dname) { + didx = rb_enc_find_index(dname); + if (0 <= didx) { + denc = rb_enc_from_index(didx); + } + } if (*sname == '\0' && *dname == '\0') { num_trans = 0; entries = NULL; - sname = dname = ""; } else { struct trans_open_t toarg; @@ -1036,16 +1025,13 @@ decorator_names(int ecflags, const char **decorators_ret) { int num_decorators; - switch (ecflags & ECONV_NEWLINE_DECORATOR_MASK) { - case ECONV_UNIVERSAL_NEWLINE_DECORATOR: - case ECONV_CRLF_NEWLINE_DECORATOR: - case ECONV_CR_NEWLINE_DECORATOR: - case ECONV_LF_NEWLINE_DECORATOR: - case 0: - break; - default: + if ((ecflags & ECONV_CRLF_NEWLINE_DECORATOR) && + (ecflags & ECONV_CR_NEWLINE_DECORATOR)) + return -1; + + if ((ecflags & (ECONV_CRLF_NEWLINE_DECORATOR|ECONV_CR_NEWLINE_DECORATOR)) && + (ecflags & ECONV_UNIVERSAL_NEWLINE_DECORATOR)) return -1; - } if ((ecflags & ECONV_XML_TEXT_DECORATOR) && (ecflags & ECONV_XML_ATTR_CONTENT_DECORATOR)) @@ -1064,8 +1050,6 @@ decorator_names(int ecflags, const char **decorators_ret) decorators_ret[num_decorators++] = "crlf_newline"; if (ecflags & ECONV_CR_NEWLINE_DECORATOR) decorators_ret[num_decorators++] = "cr_newline"; - if (ecflags & ECONV_LF_NEWLINE_DECORATOR) - decorators_ret[num_decorators++] = "lf_newline"; if (ecflags & ECONV_UNIVERSAL_NEWLINE_DECORATOR) decorators_ret[num_decorators++] = "universal_newline"; @@ -1207,6 +1191,7 @@ rb_trans_conv(rb_econv_t *ec, if (ec->elems[0].last_result == econv_after_output) ec->elems[0].last_result = econv_source_buffer_empty; + needreport_index = -1; for (i = ec->num_trans-1; 0 <= i; i--) { switch (ec->elems[i].last_result) { case econv_invalid_byte_sequence: @@ -1215,6 +1200,7 @@ rb_trans_conv(rb_econv_t *ec, case econv_after_output: case econv_finished: sweep_start = i+1; + needreport_index = i; goto found_needreport; case econv_destination_buffer_full: @@ -1477,22 +1463,22 @@ rb_econv_convert(rb_econv_t *ec, if (ret == econv_invalid_byte_sequence || ret == econv_incomplete_input) { - /* deal with invalid byte sequence */ - /* todo: add more alternative behaviors */ + /* deal with invalid byte sequence */ + /* todo: add more alternative behaviors */ switch (ec->flags & ECONV_INVALID_MASK) { case ECONV_INVALID_REPLACE: - if (output_replacement_character(ec) == 0) + if (output_replacement_character(ec) == 0) goto resume; - } + } } if (ret == econv_undefined_conversion) { - /* valid character in source encoding - * but no related character(s) in destination encoding */ - /* todo: add more alternative behaviors */ + /* valid character in source encoding + * but no related character(s) in destination encoding */ + /* todo: add more alternative behaviors */ switch (ec->flags & ECONV_UNDEF_MASK) { case ECONV_UNDEF_REPLACE: - if (output_replacement_character(ec) == 0) + if (output_replacement_character(ec) == 0) goto resume; break; @@ -1730,14 +1716,14 @@ rb_econv_memsize(rb_econv_t *ec) int i; if (ec->replacement_allocated) { - size += ec->replacement_len; + size += ec->replacement_len; } for (i = 0; i < ec->num_trans; i++) { - size += rb_transcoding_memsize(ec->elems[i].tc); + size += rb_transcoding_memsize(ec->elems[i].tc); - if (ec->elems[i].out_buf_start) { + if (ec->elems[i].out_buf_start) { size += ec->elems[i].out_buf_end - ec->elems[i].out_buf_start; - } + } } size += ec->in_buf_end - ec->in_buf_start; size += sizeof(rb_econv_elem_t) * ec->num_allocated; @@ -1816,32 +1802,18 @@ rb_econv_asciicompat_encoding(const char *ascii_incompat_name) return data.ascii_compat_name; } -/* - * Append `len` bytes pointed by `ss` to `dst` with converting with `ec`. - * - * If the result of the conversion is not compatible with the encoding of - * `dst`, `dst` may not be valid encoding. - */ VALUE -rb_econv_append(rb_econv_t *ec, const char *ss, long len, VALUE dst, int flags) +rb_econv_substr_append(rb_econv_t *ec, VALUE src, long off, long len, VALUE dst, int flags) { - unsigned const char *sp, *se; + unsigned const char *ss, *sp, *se; unsigned char *ds, *dp, *de; rb_econv_result_t res; int max_output; - enum ruby_coderange_type coderange; - rb_encoding *dst_enc = ec->destination_encoding; if (NIL_P(dst)) { dst = rb_str_buf_new(len); - if (dst_enc) { - rb_enc_associate(dst, dst_enc); - } - coderange = ENC_CODERANGE_7BIT; // scan from the start - } - else { - dst_enc = rb_enc_get(dst); - coderange = rb_enc_str_coderange(dst); + if (ec->destination_encoding) + rb_enc_associate(dst, ec->destination_encoding); } if (ec->last_tc) @@ -1849,48 +1821,28 @@ rb_econv_append(rb_econv_t *ec, const char *ss, long len, VALUE dst, int flags) else max_output = 1; - do { - int cr; + res = econv_destination_buffer_full; + while (res == econv_destination_buffer_full) { long dlen = RSTRING_LEN(dst); if (rb_str_capacity(dst) - dlen < (size_t)len + max_output) { unsigned long new_capa = (unsigned long)dlen + len + max_output; if (LONG_MAX < new_capa) rb_raise(rb_eArgError, "too long string"); - rb_str_modify_expand(dst, new_capa - dlen); + rb_str_resize(dst, new_capa); + rb_str_set_len(dst, dlen); } - sp = (const unsigned char *)ss; - se = sp + len; + ss = sp = (const unsigned char *)RSTRING_PTR(src) + off; + se = ss + len; ds = (unsigned char *)RSTRING_PTR(dst); de = ds + rb_str_capacity(dst); dp = ds += dlen; res = rb_econv_convert(ec, &sp, se, &dp, de, flags); - switch (coderange) { - case ENC_CODERANGE_7BIT: - case ENC_CODERANGE_VALID: - cr = (int)coderange; - rb_str_coderange_scan_restartable((char *)ds, (char *)dp, dst_enc, &cr); - coderange = cr; - ENC_CODERANGE_SET(dst, coderange); - break; - case ENC_CODERANGE_UNKNOWN: - case ENC_CODERANGE_BROKEN: - break; - } - len -= (const char *)sp - ss; - ss = (const char *)sp; + off += sp - ss; + len -= sp - ss; rb_str_set_len(dst, dlen + (dp - ds)); rb_econv_check_error(ec); - } while (res == econv_destination_buffer_full); - - return dst; -} + } -VALUE -rb_econv_substr_append(rb_econv_t *ec, VALUE src, long off, long len, VALUE dst, int flags) -{ - src = rb_str_new_frozen(src); - dst = rb_econv_append(ec, RSTRING_PTR(src) + off, len, dst, flags); - RB_GC_GUARD(src); return dst; } @@ -1926,7 +1878,6 @@ rb_econv_add_converter(rb_econv_t *ec, const char *sname, const char *dname, int return -1; tr = load_transcoder_entry(entry); - if (!tr) return -1; return rb_econv_add_transcoder_at(ec, tr, n); } @@ -1974,40 +1925,48 @@ rb_econv_decorate_at_last(rb_econv_t *ec, const char *decorator_name) void rb_econv_binmode(rb_econv_t *ec) { - const char *dname = 0; - - switch (ec->flags & ECONV_NEWLINE_DECORATOR_MASK) { - case ECONV_UNIVERSAL_NEWLINE_DECORATOR: - dname = "universal_newline"; - break; - case ECONV_CRLF_NEWLINE_DECORATOR: - dname = "crlf_newline"; - break; - case ECONV_CR_NEWLINE_DECORATOR: - dname = "cr_newline"; - break; - case ECONV_LF_NEWLINE_DECORATOR: - dname = "lf_newline"; - break; - } - - if (dname) { - const rb_transcoder *transcoder = get_transcoder_entry("", dname)->transcoder; - int num_trans = ec->num_trans; - int i, j = 0; - - for (i=0; i < num_trans; i++) { - if (transcoder == ec->elems[i].tc->transcoder) { - rb_transcoding_close(ec->elems[i].tc); - xfree(ec->elems[i].out_buf_start); - ec->num_trans--; - } - else - ec->elems[j++] = ec->elems[i]; + const rb_transcoder *trs[3]; + int n, i, j; + transcoder_entry_t *entry; + int num_trans; + + n = 0; + if (ec->flags & ECONV_UNIVERSAL_NEWLINE_DECORATOR) { + entry = get_transcoder_entry("", "universal_newline"); + if (entry->transcoder) + trs[n++] = entry->transcoder; + } + if (ec->flags & ECONV_CRLF_NEWLINE_DECORATOR) { + entry = get_transcoder_entry("", "crlf_newline"); + if (entry->transcoder) + trs[n++] = entry->transcoder; + } + if (ec->flags & ECONV_CR_NEWLINE_DECORATOR) { + entry = get_transcoder_entry("", "cr_newline"); + if (entry->transcoder) + trs[n++] = entry->transcoder; + } + + num_trans = ec->num_trans; + j = 0; + for (i = 0; i < num_trans; i++) { + int k; + for (k = 0; k < n; k++) + if (trs[k] == ec->elems[i].tc->transcoder) + break; + if (k == n) { + ec->elems[j] = ec->elems[i]; + j++; + } + else { + rb_transcoding_close(ec->elems[i].tc); + xfree(ec->elems[i].out_buf_start); + ec->num_trans--; } } - ec->flags &= ~ECONV_NEWLINE_DECORATOR_MASK; + ec->flags &= ~(ECONV_UNIVERSAL_NEWLINE_DECORATOR|ECONV_CRLF_NEWLINE_DECORATOR|ECONV_CR_NEWLINE_DECORATOR); + } static VALUE @@ -2028,7 +1987,9 @@ econv_description(const char *sname, const char *dname, int ecflags, VALUE mesg) has_description = 1; } - if (ecflags & (ECONV_NEWLINE_DECORATOR_MASK| + if (ecflags & (ECONV_UNIVERSAL_NEWLINE_DECORATOR| + ECONV_CRLF_NEWLINE_DECORATOR| + ECONV_CR_NEWLINE_DECORATOR| ECONV_XML_TEXT_DECORATOR| ECONV_XML_ATTR_CONTENT_DECORATOR| ECONV_XML_ATTR_QUOTE_DECORATOR)) { @@ -2047,10 +2008,6 @@ econv_description(const char *sname, const char *dname, int ecflags, VALUE mesg) rb_str_cat2(mesg, pre); pre = ","; rb_str_cat2(mesg, "cr_newline"); } - if (ecflags & ECONV_LF_NEWLINE_DECORATOR) { - rb_str_cat2(mesg, pre); pre = ","; - rb_str_cat2(mesg, "lf_newline"); - } if (ecflags & ECONV_XML_TEXT_DECORATOR) { rb_str_cat2(mesg, pre); pre = ","; rb_str_cat2(mesg, "xml_text"); @@ -2096,6 +2053,7 @@ make_econv_exception(rb_econv_t *ec) size_t readagain_len = ec->last_error.readagain_len; VALUE bytes2 = Qnil; VALUE dumped2; + int idx; if (ec->last_error.result == econv_incomplete_input) { mesg = rb_sprintf("incomplete %s on %s", StringValueCStr(dumped), @@ -2116,10 +2074,20 @@ make_econv_exception(rb_econv_t *ec) } exc = rb_exc_new3(rb_eInvalidByteSequenceError, mesg); - rb_ivar_set(exc, id_error_bytes, bytes); - rb_ivar_set(exc, id_readagain_bytes, bytes2); - rb_ivar_set(exc, id_incomplete_input, RBOOL(ec->last_error.result == econv_incomplete_input)); - goto set_encs; + rb_ivar_set(exc, rb_intern("error_bytes"), bytes); + rb_ivar_set(exc, rb_intern("readagain_bytes"), bytes2); + rb_ivar_set(exc, rb_intern("incomplete_input"), ec->last_error.result == econv_incomplete_input ? Qtrue : Qfalse); + + set_encs: + rb_ivar_set(exc, rb_intern("source_encoding_name"), rb_str_new2(ec->last_error.source_encoding)); + rb_ivar_set(exc, rb_intern("destination_encoding_name"), rb_str_new2(ec->last_error.destination_encoding)); + idx = rb_enc_find_index(ec->last_error.source_encoding); + if (0 <= idx) + rb_ivar_set(exc, rb_intern("source_encoding"), rb_enc_from_encoding(rb_enc_from_index(idx))); + idx = rb_enc_find_index(ec->last_error.destination_encoding); + if (0 <= idx) + rb_ivar_set(exc, rb_intern("destination_encoding"), rb_enc_from_encoding(rb_enc_from_index(idx))); + return exc; } if (ec->last_error.result == econv_undefined_conversion) { VALUE bytes = rb_str_new((const char *)ec->last_error.error_bytes_start, @@ -2139,7 +2107,7 @@ make_econv_exception(rb_econv_t *ec) dumped = rb_sprintf("U+%04X", cc); } } - if (NIL_P(dumped)) + if (dumped == Qnil) dumped = rb_str_dump(bytes); if (strcmp(ec->last_error.source_encoding, ec->source_encoding_name) == 0 && @@ -2167,21 +2135,10 @@ make_econv_exception(rb_econv_t *ec) idx = rb_enc_find_index(ec->last_error.source_encoding); if (0 <= idx) rb_enc_associate_index(bytes, idx); - rb_ivar_set(exc, id_error_char, bytes); + rb_ivar_set(exc, rb_intern("error_char"), bytes); goto set_encs; } return Qnil; - - set_encs: - rb_ivar_set(exc, id_source_encoding_name, rb_str_new2(ec->last_error.source_encoding)); - rb_ivar_set(exc, id_destination_encoding_name, rb_str_new2(ec->last_error.destination_encoding)); - int idx = rb_enc_find_index(ec->last_error.source_encoding); - if (0 <= idx) - rb_ivar_set(exc, id_source_encoding, rb_enc_from_encoding(rb_enc_from_index(idx))); - idx = rb_enc_find_index(ec->last_error.destination_encoding); - if (0 <= idx) - rb_ivar_set(exc, id_destination_encoding, rb_enc_from_encoding(rb_enc_from_index(idx))); - return exc; } static void @@ -2205,6 +2162,7 @@ make_replacement(rb_econv_t *ec) { rb_transcoding *tc; const rb_transcoder *tr; + rb_encoding *enc; const unsigned char *replacement; const char *repl_enc; const char *ins_enc; @@ -2218,7 +2176,7 @@ make_replacement(rb_econv_t *ec) tc = ec->last_tc; if (*ins_enc) { tr = tc->transcoder; - rb_enc_find(tr->dst_encoding); + enc = rb_enc_find(tr->dst_encoding); replacement = (const unsigned char *)get_replacement_character(ins_enc, &len, &repl_enc); } else { @@ -2244,7 +2202,7 @@ rb_econv_set_replacement(rb_econv_t *ec, encname2 = rb_econv_encoding_to_insert_output(ec); - if (!*encname2 || encoding_equal(encname, encname2)) { + if (encoding_equal(encname, encname2)) { str2 = xmalloc(len); MEMCPY(str2, str, unsigned char, len); /* xxx: str may be invalid */ len2 = len; @@ -2282,29 +2240,9 @@ output_replacement_character(rb_econv_t *ec) } #if 1 -#define hash_fallback rb_hash_aref - -static VALUE -proc_fallback(VALUE fallback, VALUE c) -{ - return rb_proc_call(fallback, rb_ary_new4(1, &c)); -} - -static VALUE -method_fallback(VALUE fallback, VALUE c) -{ - return rb_method_call(1, &c, fallback); -} - -static VALUE -aref_fallback(VALUE fallback, VALUE c) -{ - return rb_funcallv_public(fallback, idAREF, 1, &c); -} - static void transcode_loop(const unsigned char **in_pos, unsigned char **out_pos, - const unsigned char *in_stop, unsigned char *out_stop, + const unsigned char *in_stop, unsigned char *out_stop, VALUE destination, unsigned char *(*resize_destination)(VALUE, size_t, size_t), const char *src_encoding, @@ -2319,27 +2257,13 @@ transcode_loop(const unsigned char **in_pos, unsigned char **out_pos, int max_output; VALUE exc; VALUE fallback = Qnil; - VALUE (*fallback_func)(VALUE, VALUE) = 0; ec = rb_econv_open_opts(src_encoding, dst_encoding, ecflags, ecopts); if (!ec) rb_exc_raise(rb_econv_open_exc(src_encoding, dst_encoding, ecflags)); - if (!NIL_P(ecopts) && RB_TYPE_P(ecopts, T_HASH)) { - fallback = rb_hash_aref(ecopts, sym_fallback); - if (RB_TYPE_P(fallback, T_HASH)) { - fallback_func = hash_fallback; - } - else if (rb_obj_is_proc(fallback)) { - fallback_func = proc_fallback; - } - else if (rb_obj_is_method(fallback)) { - fallback_func = method_fallback; - } - else { - fallback_func = aref_fallback; - } - } + if (!NIL_P(ecopts) && TYPE(ecopts) == T_HASH) + fallback = rb_hash_aref(ecopts, sym_fallback); last_tc = ec->last_tc; max_output = last_tc ? last_tc->transcoder->max_output : 1; @@ -2347,20 +2271,20 @@ transcode_loop(const unsigned char **in_pos, unsigned char **out_pos, ret = rb_econv_convert(ec, in_pos, in_stop, out_pos, out_stop, 0); if (!NIL_P(fallback) && ret == econv_undefined_conversion) { - VALUE rep = rb_enc_str_new( - (const char *)ec->last_error.error_bytes_start, - ec->last_error.error_bytes_len, - rb_enc_find(ec->last_error.source_encoding)); - rep = (*fallback_func)(fallback, rep); - if (!UNDEF_P(rep) && !NIL_P(rep)) { - StringValue(rep); - ret = rb_econv_insert_output(ec, (const unsigned char *)RSTRING_PTR(rep), - RSTRING_LEN(rep), rb_enc_name(rb_enc_get(rep))); - if ((int)ret == -1) { - rb_raise(rb_eArgError, "too big fallback string"); - } - goto resume; - } + VALUE rep = rb_enc_str_new( + (const char *)ec->last_error.error_bytes_start, + ec->last_error.error_bytes_len, + rb_enc_find(ec->last_error.source_encoding)); + rep = rb_hash_lookup2(fallback, rep, Qundef); + if (rep != Qundef) { + StringValue(rep); + ret = rb_econv_insert_output(ec, (const unsigned char *)RSTRING_PTR(rep), + RSTRING_LEN(rep), rb_enc_name(rb_enc_get(rep))); + if ((int)ret == -1) { + rb_raise(rb_eArgError, "too big fallback string"); + } + goto resume; + } } if (ret == econv_invalid_byte_sequence || @@ -2368,7 +2292,7 @@ transcode_loop(const unsigned char **in_pos, unsigned char **out_pos, ret == econv_undefined_conversion) { exc = make_econv_exception(ec); rb_econv_close(ec); - rb_exc_raise(exc); + rb_exc_raise(exc); } if (ret == econv_destination_buffer_full) { @@ -2383,7 +2307,7 @@ transcode_loop(const unsigned char **in_pos, unsigned char **out_pos, /* sample transcode_loop implementation in byte-by-byte stream style */ static void transcode_loop(const unsigned char **in_pos, unsigned char **out_pos, - const unsigned char *in_stop, unsigned char *out_stop, + const unsigned char *in_stop, unsigned char *out_stop, VALUE destination, unsigned char *(*resize_destination)(VALUE, size_t, size_t), const char *src_encoding, @@ -2465,10 +2389,10 @@ str_transcoding_resize(VALUE destination, size_t len, size_t new_len) } static int -econv_opts(VALUE opt, int ecflags) +econv_opts(VALUE opt) { VALUE v; - int newlineflag = 0; + int ecflags = 0; v = rb_hash_aref(opt, sym_invalid); if (NIL_P(v)) { @@ -2503,114 +2427,64 @@ econv_opts(VALUE opt, int ecflags) else if (v==sym_attr) { ecflags |= ECONV_XML_ATTR_CONTENT_DECORATOR|ECONV_XML_ATTR_QUOTE_DECORATOR|ECONV_UNDEF_HEX_CHARREF; } - else if (SYMBOL_P(v)) { - rb_raise(rb_eArgError, "unexpected value for xml option: %"PRIsVALUE, rb_sym2str(v)); + else if (TYPE(v) == T_SYMBOL) { + rb_raise(rb_eArgError, "unexpected value for xml option: %s", rb_id2name(SYM2ID(v))); } else { rb_raise(rb_eArgError, "unexpected value for xml option"); } } -#ifdef ENABLE_ECONV_NEWLINE_OPTION - v = rb_hash_aref(opt, sym_newline); - if (!NIL_P(v)) { - newlineflag = 2; - ecflags &= ~ECONV_NEWLINE_DECORATOR_MASK; - if (v == sym_universal) { - ecflags |= ECONV_UNIVERSAL_NEWLINE_DECORATOR; - } - else if (v == sym_crlf) { - ecflags |= ECONV_CRLF_NEWLINE_DECORATOR; - } - else if (v == sym_cr) { - ecflags |= ECONV_CR_NEWLINE_DECORATOR; - } - else if (v == sym_lf) { - ecflags |= ECONV_LF_NEWLINE_DECORATOR; - } - else if (SYMBOL_P(v)) { - rb_raise(rb_eArgError, "unexpected value for newline option: %"PRIsVALUE, - rb_sym2str(v)); - } - else { - rb_raise(rb_eArgError, "unexpected value for newline option"); - } - } -#endif - { - int setflags = 0; - - v = rb_hash_aref(opt, sym_universal_newline); - if (RTEST(v)) - setflags |= ECONV_UNIVERSAL_NEWLINE_DECORATOR; - newlineflag |= !NIL_P(v); - - v = rb_hash_aref(opt, sym_crlf_newline); - if (RTEST(v)) - setflags |= ECONV_CRLF_NEWLINE_DECORATOR; - newlineflag |= !NIL_P(v); - - v = rb_hash_aref(opt, sym_cr_newline); - if (RTEST(v)) - setflags |= ECONV_CR_NEWLINE_DECORATOR; - newlineflag |= !NIL_P(v); - - v = rb_hash_aref(opt, sym_lf_newline); - if (RTEST(v)) - setflags |= ECONV_LF_NEWLINE_DECORATOR; - newlineflag |= !NIL_P(v); + v = rb_hash_aref(opt, sym_universal_newline); + if (RTEST(v)) + ecflags |= ECONV_UNIVERSAL_NEWLINE_DECORATOR; - switch (newlineflag) { - case 1: - ecflags &= ~ECONV_NEWLINE_DECORATOR_MASK; - ecflags |= setflags; - break; + v = rb_hash_aref(opt, sym_crlf_newline); + if (RTEST(v)) + ecflags |= ECONV_CRLF_NEWLINE_DECORATOR; - case 3: - rb_warning(":newline option precedes other newline options"); - break; - } - } + v = rb_hash_aref(opt, sym_cr_newline); + if (RTEST(v)) + ecflags |= ECONV_CR_NEWLINE_DECORATOR; return ecflags; } int -rb_econv_prepare_options(VALUE opthash, VALUE *opts, int ecflags) +rb_econv_prepare_opts(VALUE opthash, VALUE *opts) { + int ecflags; VALUE newhash = Qnil; VALUE v; if (NIL_P(opthash)) { *opts = Qnil; - return ecflags; + return 0; } - ecflags = econv_opts(opthash, ecflags); + ecflags = econv_opts(opthash); v = rb_hash_aref(opthash, sym_replace); if (!NIL_P(v)) { - StringValue(v); - if (is_broken_string(v)) { - VALUE dumped = rb_str_dump(v); - rb_raise(rb_eArgError, "replacement string is broken: %s as %s", - StringValueCStr(dumped), - rb_enc_name(rb_enc_get(v))); - } - v = rb_str_new_frozen(v); - newhash = rb_hash_new(); - rb_hash_aset(newhash, sym_replace, v); + StringValue(v); + if (rb_enc_str_coderange(v) == ENC_CODERANGE_BROKEN) { + VALUE dumped = rb_str_dump(v); + rb_raise(rb_eArgError, "replacement string is broken: %s as %s", + StringValueCStr(dumped), + rb_enc_name(rb_enc_get(v))); + } + v = rb_str_new_frozen(v); + newhash = rb_hash_new(); + rb_hash_aset(newhash, sym_replace, v); } v = rb_hash_aref(opthash, sym_fallback); if (!NIL_P(v)) { - VALUE h = rb_check_hash_type(v); - if (NIL_P(h) - ? (rb_obj_is_proc(v) || rb_obj_is_method(v) || rb_respond_to(v, idAREF)) - : (v = h, 1)) { - if (NIL_P(newhash)) - newhash = rb_hash_new(); - rb_hash_aset(newhash, sym_fallback, v); - } + v = rb_convert_type(v, T_HASH, "Hash", "to_hash"); + if (!NIL_P(v)) { + if (NIL_P(newhash)) + newhash = rb_hash_new(); + rb_hash_aset(newhash, sym_fallback, v); + } } if (!NIL_P(newhash)) @@ -2620,12 +2494,6 @@ rb_econv_prepare_options(VALUE opthash, VALUE *opts, int ecflags) return ecflags; } -int -rb_econv_prepare_opts(VALUE opthash, VALUE *opts) -{ - return rb_econv_prepare_options(opthash, opts, 0); -} - rb_econv_t * rb_econv_open_opts(const char *source_encoding, const char *destination_encoding, int ecflags, VALUE opthash) { @@ -2636,7 +2504,7 @@ rb_econv_open_opts(const char *source_encoding, const char *destination_encoding replacement = Qnil; } else { - if (!RB_TYPE_P(opthash, T_HASH) || !OBJ_FROZEN(opthash)) + if (TYPE(opthash) != T_HASH || !OBJ_FROZEN(opthash)) rb_bug("rb_econv_open_opts called with invalid opthash"); replacement = rb_hash_aref(opthash, sym_replace); } @@ -2662,7 +2530,7 @@ rb_econv_open_opts(const char *source_encoding, const char *destination_encoding } static int -enc_arg(VALUE *arg, const char **name_p, rb_encoding **enc_p) +enc_arg(volatile VALUE *arg, const char **name_p, rb_encoding **enc_p) { rb_encoding *enc; const char *n; @@ -2670,13 +2538,13 @@ enc_arg(VALUE *arg, const char **name_p, rb_encoding **enc_p) VALUE encval; if (((encidx = rb_to_encoding_index(encval = *arg)) < 0) || - !(enc = rb_enc_from_index(encidx))) { - enc = NULL; - encidx = 0; - n = StringValueCStr(*arg); + !(enc = rb_enc_from_index(encidx))) { + enc = NULL; + encidx = 0; + n = StringValueCStr(*arg); } else { - n = rb_enc_name(enc); + n = rb_enc_name(enc); } *name_p = n; @@ -2686,7 +2554,7 @@ enc_arg(VALUE *arg, const char **name_p, rb_encoding **enc_p) } static int -str_transcode_enc_args(VALUE str, VALUE *arg1, VALUE *arg2, +str_transcode_enc_args(VALUE str, volatile VALUE *arg1, volatile VALUE *arg2, const char **sname_p, rb_encoding **senc_p, const char **dname_p, rb_encoding **denc_p) { @@ -2697,9 +2565,9 @@ str_transcode_enc_args(VALUE str, VALUE *arg1, VALUE *arg2, dencidx = enc_arg(arg1, &dname, &denc); if (NIL_P(*arg2)) { - sencidx = rb_enc_get_index(str); - senc = rb_enc_from_index(sencidx); - sname = rb_enc_name(senc); + sencidx = rb_enc_get_index(str); + senc = rb_enc_from_index(sencidx); + sname = rb_enc_name(senc); } else { sencidx = enc_arg(arg2, &sname, &senc); @@ -2717,53 +2585,43 @@ str_transcode0(int argc, VALUE *argv, VALUE *self, int ecflags, VALUE ecopts) { VALUE dest; VALUE str = *self; - VALUE arg1, arg2; + volatile VALUE arg1, arg2; long blen, slen; unsigned char *buf, *bp, *sp; const unsigned char *fromp; rb_encoding *senc, *denc; const char *sname, *dname; int dencidx; - int explicitly_invalid_replace = TRUE; - rb_check_arity(argc, 0, 2); + if (argc <0 || argc > 2) { + rb_raise(rb_eArgError, "wrong number of arguments (%d for 0..2)", argc); + } if (argc == 0) { - arg1 = rb_enc_default_internal(); - if (NIL_P(arg1)) { - if (!ecflags) return -1; - arg1 = rb_obj_encoding(str); - } - if (!(ecflags & ECONV_INVALID_MASK)) { - explicitly_invalid_replace = FALSE; - } - ecflags |= ECONV_INVALID_REPLACE | ECONV_UNDEF_REPLACE; + arg1 = rb_enc_default_internal(); + if (NIL_P(arg1)) { + if (!ecflags) return -1; + arg1 = rb_obj_encoding(str); + } + ecflags |= ECONV_INVALID_REPLACE | ECONV_UNDEF_REPLACE; } else { - arg1 = argv[0]; + arg1 = argv[0]; } arg2 = argc<=1 ? Qnil : argv[1]; dencidx = str_transcode_enc_args(str, &arg1, &arg2, &sname, &senc, &dname, &denc); - if ((ecflags & (ECONV_NEWLINE_DECORATOR_MASK| + if ((ecflags & (ECONV_UNIVERSAL_NEWLINE_DECORATOR| + ECONV_CRLF_NEWLINE_DECORATOR| + ECONV_CR_NEWLINE_DECORATOR| ECONV_XML_TEXT_DECORATOR| ECONV_XML_ATTR_CONTENT_DECORATOR| ECONV_XML_ATTR_QUOTE_DECORATOR)) == 0) { if (senc && senc == denc) { - if ((ecflags & ECONV_INVALID_MASK) && explicitly_invalid_replace) { - VALUE rep = Qnil; - if (!NIL_P(ecopts)) { - rep = rb_hash_aref(ecopts, sym_replace); - } - dest = rb_enc_str_scrub(senc, str, rep); - if (NIL_P(dest)) dest = str; - *self = dest; - return dencidx; - } return NIL_P(arg2) ? -1 : dencidx; } if (senc && denc && rb_enc_asciicompat(senc) && rb_enc_asciicompat(denc)) { - if (is_ascii_string(str)) { + if (rb_enc_str_coderange(str) == ENC_CODERANGE_7BIT) { return dencidx; } } @@ -2772,12 +2630,6 @@ str_transcode0(int argc, VALUE *argv, VALUE *self, int ecflags, VALUE ecopts) } } else { - if (senc && denc && !rb_enc_asciicompat(senc) && !rb_enc_asciicompat(denc)) { - rb_encoding *utf8 = rb_utf8_encoding(); - str = rb_str_conv_enc(str, senc, utf8); - senc = utf8; - sname = "UTF-8"; - } if (encoding_equal(sname, dname)) { sname = ""; dname = ""; @@ -2800,9 +2652,7 @@ str_transcode0(int argc, VALUE *argv, VALUE *self, int ecflags, VALUE ecopts) /* set encoding */ if (!denc) { - dencidx = rb_define_dummy_encoding(dname); - RB_GC_GUARD(arg1); - RB_GC_GUARD(arg2); + dencidx = rb_define_dummy_encoding(dname); } *self = dest; @@ -2816,9 +2666,12 @@ str_transcode(int argc, VALUE *argv, VALUE *self) int ecflags = 0; VALUE ecopts = Qnil; - argc = rb_scan_args(argc, argv, "02:", NULL, NULL, &opt); - if (!NIL_P(opt)) { - ecflags = rb_econv_prepare_opts(opt, &ecopts); + if (0 < argc) { + opt = rb_check_convert_type(argv[argc-1], T_HASH, "Hash", "to_hash"); + if (!NIL_P(opt)) { + argc--; + ecflags = rb_econv_prepare_opts(opt, &ecopts); + } } return str_transcode0(argc, argv, self, ecflags, ecopts); } @@ -2832,10 +2685,10 @@ str_encode_associate(VALUE str, int encidx) /* transcoded string never be broken. */ if (rb_enc_asciicompat(rb_enc_from_index(encidx))) { - rb_str_coderange_scan_restartable(RSTRING_PTR(str), RSTRING_END(str), 0, &cr); + rb_str_coderange_scan_restartable(RSTRING_PTR(str), RSTRING_END(str), 0, &cr); } else { - cr = ENC_CODERANGE_VALID; + cr = ENC_CODERANGE_VALID; } ENC_CODERANGE_SET(str, cr); return str; @@ -2843,11 +2696,16 @@ str_encode_associate(VALUE str, int encidx) /* * call-seq: - * encode!(dst_encoding = Encoding.default_internal, **enc_opts) -> self - * encode!(dst_encoding, src_encoding, **enc_opts) -> self - * - * Like #encode, but applies encoding changes to +self+; returns +self+. + * str.encode!(encoding [, options] ) -> str + * str.encode!(dst_encoding, src_encoding [, options] ) -> str * + * The first form transcodes the contents of <i>str</i> from + * str.encoding to +encoding+. + * The second form transcodes the contents of <i>str</i> from + * src_encoding to dst_encoding. + * The options Hash gives details for conversion. See String#encode + * for details. + * Returns the string even if no changes were made. */ static VALUE @@ -2856,28 +2714,86 @@ str_encode_bang(int argc, VALUE *argv, VALUE str) VALUE newstr; int encidx; - rb_check_frozen(str); + if (OBJ_FROZEN(str)) { /* in future, may use str_frozen_check from string.c, but that's currently static */ + rb_raise(rb_eRuntimeError, "string frozen"); + } newstr = str; encidx = str_transcode(argc, argv, &newstr); if (encidx < 0) return str; - if (newstr == str) { - rb_enc_associate_index(str, encidx); - return str; - } rb_str_shared_replace(str, newstr); return str_encode_associate(str, encidx); } -static VALUE encoded_dup(VALUE newstr, VALUE str, int encidx); +/* + * call-seq: + * str.encode(encoding [, options] ) -> str + * str.encode(dst_encoding, src_encoding [, options] ) -> str + * str.encode([options]) -> str + * + * The first form returns a copy of <i>str</i> transcoded + * to encoding +encoding+. + * The second form returns a copy of <i>str</i> transcoded + * from src_encoding to dst_encoding. + * The last form returns a copy of <i>str</i> transcoded to + * <code>Encoding.default_internal</code>. + * By default, the first and second form raise + * Encoding::UndefinedConversionError for characters that are + * undefined in the destination encoding, and + * Encoding::InvalidByteSequenceError for invalid byte sequences + * in the source encoding. The last form by default does not raise + * exceptions but uses replacement strings. + * The <code>options</code> Hash gives details for conversion. + * + * === options + * The hash <code>options</code> can have the following keys: + * :invalid :: + * If the value is <code>:replace</code>, <code>#encode</code> replaces + * invalid byte sequences in <code>str</code> with the replacement character. + * The default is to raise the exception + * :undef :: + * If the value is <code>:replace</code>, <code>#encode</code> replaces + * characters which are undefined in the destination encoding with + * the replacement character. + * :replace :: + * Sets the replacement string to the value. The default replacement + * string is "\uFFFD" for Unicode encoding forms, and "?" otherwise. + * :fallback :: + * Sets the replacement string by the hash for undefined character. + * Its key is a such undefined character encoded in source encoding + * of current transcoder. Its value can be any encoding until it + * can be converted into the destination encoding of the transcoder. + * :xml :: + * The value must be <code>:text</code> or <code>:attr</code>. + * If the value is <code>:text</code> <code>#encode</code> replaces + * undefined characters with their (upper-case hexadecimal) numeric + * character references. '&', '<', and '>' are converted to "&", + * "<", and ">", respectively. + * If the value is <code>:attr</code>, <code>#encode</code> also quotes + * the replacement result (using '"'), and replaces '"' with """. + * :cr_newline :: + * Replaces LF ("\n") with CR ("\r") if value is true. + * :crlf_newline :: + * Replaces LF ("\n") with CRLF ("\r\n") if value is true. + * :universal_newline :: + * Replaces CRLF ("\r\n") and CR ("\r") with LF ("\n") if value is true. + */ static VALUE str_encode(int argc, VALUE *argv, VALUE str) { VALUE newstr = str; int encidx = str_transcode(argc, argv, &newstr); - return encoded_dup(newstr, str, encidx); + + if (encidx < 0) return rb_str_dup(str); + if (newstr == str) { + newstr = rb_str_dup(str); + } + else { + RBASIC(newstr)->klass = rb_obj_class(str); + } + return str_encode_associate(newstr, encidx); } VALUE @@ -2887,29 +2803,17 @@ rb_str_encode(VALUE str, VALUE to, int ecflags, VALUE ecopts) VALUE *argv = &to; VALUE newstr = str; int encidx = str_transcode0(argc, argv, &newstr, ecflags, ecopts); - return encoded_dup(newstr, str, encidx); -} -static VALUE -encoded_dup(VALUE newstr, VALUE str, int encidx) -{ if (encidx < 0) return rb_str_dup(str); if (newstr == str) { - newstr = rb_str_dup(str); - rb_enc_associate_index(newstr, encidx); - return newstr; + newstr = rb_str_dup(str); } else { - RBASIC_SET_CLASS(newstr, rb_obj_class(str)); + RBASIC(newstr)->klass = rb_obj_class(str); } return str_encode_associate(newstr, encidx); } -/* - * Document-class: Encoding::Converter - * - * Encoding conversion class. - */ static void econv_free(void *ptr) { @@ -2920,13 +2824,12 @@ econv_free(void *ptr) static size_t econv_memsize(const void *ptr) { - return sizeof(rb_econv_t); + return ptr ? sizeof(rb_econv_t) : 0; } static const rb_data_type_t econv_data_type = { "econv", - {0, econv_free, econv_memsize,}, - 0, 0, RUBY_TYPED_FREE_IMMEDIATELY + NULL, econv_free, econv_memsize, }; static VALUE @@ -2970,7 +2873,7 @@ make_encobj(const char *name) * * Returns nil if the argument is an ASCII compatible encoding. * - * "corresponding ASCII compatible encoding" is an ASCII compatible encoding which + * "corresponding ASCII compatible encoding" is a ASCII compatible encoding which * can represents exactly the same characters as the given ASCII incompatible encoding. * So, no conversion undefined error occurs when converting between the two encodings. * @@ -2999,33 +2902,31 @@ econv_s_asciicompat_encoding(VALUE klass, VALUE arg) static void econv_args(int argc, VALUE *argv, - VALUE *snamev_p, VALUE *dnamev_p, + volatile VALUE *snamev_p, volatile VALUE *dnamev_p, const char **sname_p, const char **dname_p, rb_encoding **senc_p, rb_encoding **denc_p, int *ecflags_p, VALUE *ecopts_p) { - VALUE opt, flags_v, ecopts; + VALUE opt, opthash, flags_v, ecopts; int sidx, didx; const char *sname, *dname; rb_encoding *senc, *denc; int ecflags; - argc = rb_scan_args(argc, argv, "21:", snamev_p, dnamev_p, &flags_v, &opt); + rb_scan_args(argc, argv, "21", snamev_p, dnamev_p, &opt); - if (!NIL_P(flags_v)) { - if (!NIL_P(opt)) { - rb_error_arity(argc + 1, 2, 3); - } - ecflags = NUM2INT(rb_to_int(flags_v)); + if (NIL_P(opt)) { + ecflags = 0; ecopts = Qnil; } - else if (!NIL_P(opt)) { - ecflags = rb_econv_prepare_opts(opt, &ecopts); + else if (!NIL_P(flags_v = rb_check_to_integer(opt, "to_int"))) { + ecflags = NUM2INT(flags_v); + ecopts = Qnil; } else { - ecflags = 0; - ecopts = Qnil; + opthash = rb_convert_type(opt, T_HASH, "Hash", "to_hash"); + ecflags = rb_econv_prepare_opts(opthash, &ecopts); } senc = NULL; @@ -3071,23 +2972,23 @@ decorate_convpath(VALUE convpath, int ecflags) len = n = RARRAY_LENINT(convpath); if (n != 0) { - VALUE pair = RARRAY_AREF(convpath, n-1); - if (RB_TYPE_P(pair, T_ARRAY)) { - const char *sname = rb_enc_name(rb_to_encoding(RARRAY_AREF(pair, 0))); - const char *dname = rb_enc_name(rb_to_encoding(RARRAY_AREF(pair, 1))); - transcoder_entry_t *entry = get_transcoder_entry(sname, dname); - const rb_transcoder *tr = load_transcoder_entry(entry); - if (!tr) - return -1; - if (!DECORATOR_P(tr->src_encoding, tr->dst_encoding) && - tr->asciicompat_type == asciicompat_encoder) { - n--; - rb_ary_store(convpath, len + num_decorators - 1, pair); - } - } - else { - rb_ary_store(convpath, len + num_decorators - 1, pair); - } + VALUE pair = RARRAY_PTR(convpath)[n-1]; + if (TYPE(pair) == T_ARRAY) { + const char *sname = rb_enc_name(rb_to_encoding(RARRAY_PTR(pair)[0])); + const char *dname = rb_enc_name(rb_to_encoding(RARRAY_PTR(pair)[1])); + transcoder_entry_t *entry = get_transcoder_entry(sname, dname); + const rb_transcoder *tr = load_transcoder_entry(entry); + if (!tr) + return -1; + if (!DECORATOR_P(tr->src_encoding, tr->dst_encoding) && + tr->asciicompat_type == asciicompat_encoder) { + n--; + rb_ary_store(convpath, len + num_decorators - 1, pair); + } + } + else { + rb_ary_store(convpath, len + num_decorators - 1, pair); + } } for (i = 0; i < num_decorators; i++) @@ -3102,7 +3003,7 @@ search_convpath_i(const char *sname, const char *dname, int depth, void *arg) VALUE *ary_p = arg; VALUE v; - if (NIL_P(*ary_p)) { + if (*ary_p == Qnil) { *ary_p = rb_ary_new(); } @@ -3127,15 +3028,11 @@ search_convpath_i(const char *sname, const char *dname, int depth, void *arg) * # [#<Encoding:UTF-8>, #<Encoding:EUC-JP>]] * * p Encoding::Converter.search_convpath("ISO-8859-1", "EUC-JP", universal_newline: true) - * or - * p Encoding::Converter.search_convpath("ISO-8859-1", "EUC-JP", newline: :universal) * #=> [[#<Encoding:ISO-8859-1>, #<Encoding:UTF-8>], * # [#<Encoding:UTF-8>, #<Encoding:EUC-JP>], * # "universal_newline"] * * p Encoding::Converter.search_convpath("ISO-8859-1", "UTF-32BE", universal_newline: true) - * or - * p Encoding::Converter.search_convpath("ISO-8859-1", "UTF-32BE", newline: :universal) * #=> [[#<Encoding:ISO-8859-1>, #<Encoding:UTF-8>], * # "universal_newline", * # [#<Encoding:UTF-8>, #<Encoding:UTF-32BE>]] @@ -3143,7 +3040,7 @@ search_convpath_i(const char *sname, const char *dname, int depth, void *arg) static VALUE econv_s_search_convpath(int argc, VALUE *argv, VALUE klass) { - VALUE snamev, dnamev; + volatile VALUE snamev, dnamev; const char *sname, *dname; rb_encoding *senc, *denc; int ecflags; @@ -3155,19 +3052,11 @@ econv_s_search_convpath(int argc, VALUE *argv, VALUE klass) convpath = Qnil; transcode_search_path(sname, dname, search_convpath_i, &convpath); - if (NIL_P(convpath)) { - VALUE exc = rb_econv_open_exc(sname, dname, ecflags); - RB_GC_GUARD(snamev); - RB_GC_GUARD(dnamev); - rb_exc_raise(exc); - } + if (NIL_P(convpath)) + rb_exc_raise(rb_econv_open_exc(sname, dname, ecflags)); - if (decorate_convpath(convpath, ecflags) == -1) { - VALUE exc = rb_econv_open_exc(sname, dname, ecflags); - RB_GC_GUARD(snamev); - RB_GC_GUARD(dnamev); - rb_exc_raise(exc); - } + if (decorate_convpath(convpath, ecflags) == -1) + rb_exc_raise(rb_econv_open_exc(sname, dname, ecflags)); return convpath; } @@ -3182,7 +3071,7 @@ rb_econv_has_convpath_p(const char* from_encoding, const char* to_encoding) { VALUE convpath = Qnil; transcode_search_path(from_encoding, to_encoding, search_convpath_i, - &convpath); + &convpath); return RTEST(convpath); } @@ -3223,7 +3112,7 @@ rb_econv_init_by_convpath(VALUE self, VALUE convpath, DATA_PTR(self) = ec; for (i = 0; i < RARRAY_LEN(convpath); i++) { - VALUE snamev, dnamev; + volatile VALUE snamev, dnamev; VALUE pair; elt = rb_ary_entry(convpath, i); if (!NIL_P(pair = rb_check_array_type(elt))) { @@ -3240,12 +3129,8 @@ rb_econv_init_by_convpath(VALUE self, VALUE convpath, } if (DECORATOR_P(sname, dname)) { ret = rb_econv_add_converter(ec, sname, dname, ec->num_trans); - if (ret == -1) { - VALUE msg = rb_sprintf("decoration failed: %s", dname); - RB_GC_GUARD(snamev); - RB_GC_GUARD(dnamev); - rb_exc_raise(rb_exc_new_str(rb_eArgError, msg)); - } + if (ret == -1) + rb_raise(rb_eArgError, "decoration failed: %s", dname); } else { int j = ec->num_trans; @@ -3254,12 +3139,8 @@ rb_econv_init_by_convpath(VALUE self, VALUE convpath, arg.index = ec->num_trans; arg.ret = 0; ret = transcode_search_path(sname, dname, rb_econv_init_by_convpath_i, &arg); - if (ret == -1 || arg.ret == -1) { - VALUE msg = rb_sprintf("adding conversion failed: %s to %s", sname, dname); - RB_GC_GUARD(snamev); - RB_GC_GUARD(dnamev); - rb_exc_raise(rb_exc_new_str(rb_eArgError, msg)); - } + if (ret == -1 || arg.ret == -1) + rb_raise(rb_eArgError, "adding conversion failed: %s to %s", sname, dname); if (first) { first = 0; *senc_p = senc; @@ -3271,10 +3152,10 @@ rb_econv_init_by_convpath(VALUE self, VALUE convpath, } if (first) { - *senc_p = NULL; - *denc_p = NULL; - *sname_p = ""; - *dname_p = ""; + *senc_p = NULL; + *denc_p = NULL; + *sname_p = ""; + *dname_p = ""; } ec->source_encoding_name = *sname_p; @@ -3296,14 +3177,9 @@ rb_econv_init_by_convpath(VALUE self, VALUE convpath, * :undef => nil # raise error on undefined conversion (default) * :undef => :replace # replace undefined conversion * :replace => string # replacement string ("?" or "\uFFFD" if not specified) - * :newline => :universal # decorator for converting CRLF and CR to LF - * :newline => :lf # decorator for converting CRLF and CR to LF when writing - * :newline => :crlf # decorator for converting LF to CRLF - * :newline => :cr # decorator for converting LF to CR * :universal_newline => true # decorator for converting CRLF and CR to LF - * :crlf_newline => true # decorator for converting LF to CRLF - * :cr_newline => true # decorator for converting LF to CR - * :lf_newline => true # decorator for converting CRLF and CR to LF when writing + * :crlf_newline => true # decorator for converting CRLF to LF + * :cr_newline => true # decorator for converting CR to LF * :xml => :text # escape as XML CharData. * :xml => :attr # escape as XML AttValue * integer form: @@ -3311,7 +3187,6 @@ rb_econv_init_by_convpath(VALUE self, VALUE convpath, * Encoding::Converter::UNDEF_REPLACE * Encoding::Converter::UNDEF_HEX_CHARREF * Encoding::Converter::UNIVERSAL_NEWLINE_DECORATOR - * Encoding::Converter::LF_NEWLINE_DECORATOR * Encoding::Converter::CRLF_NEWLINE_DECORATOR * Encoding::Converter::CR_NEWLINE_DECORATOR * Encoding::Converter::XML_TEXT_DECORATOR @@ -3354,11 +3229,9 @@ rb_econv_init_by_convpath(VALUE self, VALUE convpath, * Convert LF to CRLF. * [:cr_newline => true] * Convert LF to CR. - * [:lf_newline => true] - * Convert CRLF and CR to LF (when writing). * [:xml => :text] * Escape as XML CharData. - * This form can be used as an HTML 4.0 #PCDATA. + * This form can be used as a HTML 4.0 #PCDATA. * - '&' -> '&' * - '<' -> '<' * - '>' -> '>' @@ -3366,7 +3239,7 @@ rb_econv_init_by_convpath(VALUE self, VALUE convpath, * [:xml => :attr] * Escape as XML AttValue. * The converted result is quoted as "...". - * This form can be used as an HTML 4.0 attribute value. + * This form can be used as a HTML 4.0 attribute value. * - '&' -> '&' * - '<' -> '<' * - '>' -> '>' @@ -3398,7 +3271,7 @@ static VALUE econv_init(int argc, VALUE *argv, VALUE self) { VALUE ecopts; - VALUE snamev, dnamev; + volatile VALUE snamev, dnamev; const char *sname, *dname; rb_encoding *senc, *denc; rb_econv_t *ec; @@ -3420,10 +3293,7 @@ econv_init(int argc, VALUE *argv, VALUE self) } if (!ec) { - VALUE exc = rb_econv_open_exc(sname, dname, ecflags); - RB_GC_GUARD(snamev); - RB_GC_GUARD(dnamev); - rb_exc_raise(exc); + rb_exc_raise(rb_econv_open_exc(sname, dname, ecflags)); } if (!DECORATOR_P(sname, dname)) { @@ -3431,8 +3301,6 @@ econv_init(int argc, VALUE *argv, VALUE self) senc = make_dummy_encoding(sname); if (!denc) denc = make_dummy_encoding(dname); - RB_GC_GUARD(snamev); - RB_GC_GUARD(dnamev); } ec->source_encoding = senc; @@ -3485,14 +3353,6 @@ check_econv(VALUE self) return ec; } -static VALUE -econv_get_encoding(rb_encoding *encoding) -{ - if (!encoding) - return Qnil; - return rb_enc_from_encoding(encoding); -} - /* * call-seq: * ec.source_encoding -> encoding @@ -3503,7 +3363,9 @@ static VALUE econv_source_encoding(VALUE self) { rb_econv_t *ec = check_econv(self); - return econv_get_encoding(ec->source_encoding); + if (!ec->source_encoding) + return Qnil; + return rb_enc_from_encoding(ec->source_encoding); } /* @@ -3516,7 +3378,9 @@ static VALUE econv_destination_encoding(VALUE self) { rb_econv_t *ec = check_econv(self); - return econv_get_encoding(ec->destination_encoding); + if (!ec->destination_encoding) + return Qnil; + return rb_enc_from_encoding(ec->destination_encoding); } /* @@ -3561,45 +3425,6 @@ econv_convpath(VALUE self) return result; } -/* - * call-seq: - * ec == other -> true or false - */ -static VALUE -econv_equal(VALUE self, VALUE other) -{ - rb_econv_t *ec1 = check_econv(self); - rb_econv_t *ec2; - int i; - - if (!rb_typeddata_is_kind_of(other, &econv_data_type)) { - return Qnil; - } - ec2 = DATA_PTR(other); - if (!ec2) return Qfalse; - if (ec1->source_encoding_name != ec2->source_encoding_name && - strcmp(ec1->source_encoding_name, ec2->source_encoding_name)) - return Qfalse; - if (ec1->destination_encoding_name != ec2->destination_encoding_name && - strcmp(ec1->destination_encoding_name, ec2->destination_encoding_name)) - return Qfalse; - if (ec1->flags != ec2->flags) return Qfalse; - if (ec1->replacement_enc != ec2->replacement_enc && - strcmp(ec1->replacement_enc, ec2->replacement_enc)) - return Qfalse; - if (ec1->replacement_len != ec2->replacement_len) return Qfalse; - if (ec1->replacement_str != ec2->replacement_str && - memcmp(ec1->replacement_str, ec2->replacement_str, ec2->replacement_len)) - return Qfalse; - - if (ec1->num_trans != ec2->num_trans) return Qfalse; - for (i = 0; i < ec1->num_trans; i++) { - if (ec1->elems[i].tc->transcoder != ec2->elems[i].tc->transcoder) - return Qfalse; - } - return Qtrue; -} - static VALUE econv_result_to_symbol(rb_econv_result_t res) { @@ -3642,7 +3467,7 @@ econv_result_to_symbol(rb_econv_result_t res) * primitive_convert converts source_buffer into destination_buffer. * * source_buffer should be a string or nil. - * nil means an empty string. + * nil means a empty string. * * destination_buffer should be a string. * @@ -3679,12 +3504,9 @@ econv_result_to_symbol(rb_econv_result_t res) * * primitive_convert stops conversion when one of following condition met. * - invalid byte sequence found in source buffer (:invalid_byte_sequence) - * +primitive_errinfo+ and +last_error+ methods returns the detail of the error. * - unexpected end of source buffer (:incomplete_input) * this occur only when :partial_input is not specified. - * +primitive_errinfo+ and +last_error+ methods returns the detail of the error. * - character not representable in output encoding (:undefined_conversion) - * +primitive_errinfo+ and +last_error+ methods returns the detail of the error. * - after some output is generated, before input is done (:after_output) * this occur only when :after_output is specified. * - destination buffer is full (:destination_buffer_full) @@ -3721,7 +3543,7 @@ econv_primitive_convert(int argc, VALUE *argv, VALUE self) unsigned long output_byteend; int flags; - argc = rb_scan_args(argc, argv, "23:", &input, &output, &output_byteoffset_v, &output_bytesize_v, &flags_v, &opt); + rb_scan_args(argc, argv, "23", &input, &output, &output_byteoffset_v, &output_bytesize_v, &opt); if (NIL_P(output_byteoffset_v)) output_byteoffset = 0; /* dummy */ @@ -3733,14 +3555,15 @@ econv_primitive_convert(int argc, VALUE *argv, VALUE self) else output_bytesize = NUM2LONG(output_bytesize_v); - if (!NIL_P(flags_v)) { - if (!NIL_P(opt)) { - rb_error_arity(argc + 1, 2, 5); - } - flags = NUM2INT(rb_to_int(flags_v)); + if (NIL_P(opt)) { + flags = 0; + } + else if (!NIL_P(flags_v = rb_check_to_integer(opt, "to_int"))) { + flags = NUM2INT(flags_v); } - else if (!NIL_P(opt)) { + else { VALUE v; + opt = rb_convert_type(opt, T_HASH, "Hash", "to_hash"); flags = 0; v = rb_hash_aref(opt, sym_partial_input); if (RTEST(v)) @@ -3749,9 +3572,6 @@ econv_primitive_convert(int argc, VALUE *argv, VALUE self) if (RTEST(v)) flags |= ECONV_AFTER_OUTPUT; } - else { - flags = 0; - } StringValue(output); if (!NIL_P(input)) @@ -3759,11 +3579,7 @@ econv_primitive_convert(int argc, VALUE *argv, VALUE self) rb_str_modify(output); if (NIL_P(output_bytesize_v)) { -#if USE_RVARGC - output_bytesize = rb_str_capacity(output); -#else output_bytesize = RSTRING_EMBED_LEN_MAX; -#endif if (!NIL_P(input) && output_bytesize < RSTRING_LEN(input)) output_bytesize = RSTRING_LEN(input); } @@ -3805,9 +3621,8 @@ econv_primitive_convert(int argc, VALUE *argv, VALUE self) res = rb_econv_convert(ec, &ip, is, &op, os, flags); rb_str_set_len(output, op-(unsigned char *)RSTRING_PTR(output)); - if (!NIL_P(input)) { + if (!NIL_P(input)) rb_str_drop_bytes(input, ip - (unsigned char *)RSTRING_PTR(input)); - } if (NIL_P(output_bytesize_v) && res == econv_destination_buffer_full) { if (LONG_MAX / 2 < output_bytesize) @@ -3922,7 +3737,7 @@ econv_finish(VALUE self) av[1] = dst; av[2] = Qnil; av[3] = Qnil; - av[4] = INT2FIX(0); + av[4] = INT2NUM(0); ac = 5; ret = econv_primitive_convert(ac, av, self); @@ -3970,7 +3785,7 @@ econv_finish(VALUE self) * ec = Encoding::Converter.new("EUC-JP", "Shift_JIS") * ec.primitive_convert(src="\xff", dst="", nil, 10) * p ec.primitive_errinfo - * #=> [:invalid_byte_sequence, "EUC-JP", "Shift_JIS", "\xFF", ""] + * #=> [:invalid_byte_sequence, "EUC-JP", "UTF-8", "\xFF", ""] * * # HIRAGANA LETTER A (\xa4\xa2 in EUC-JP) is not representable in ISO-8859-1. * # Since this error is occur in UTF-8 to ISO-8859-1 conversion, @@ -4089,14 +3904,14 @@ econv_insert_output(VALUE self, VALUE string) ret = rb_econv_insert_output(ec, (const unsigned char *)RSTRING_PTR(string), RSTRING_LEN(string), insert_enc); if (ret == -1) { - rb_raise(rb_eArgError, "too big string"); + rb_raise(rb_eArgError, "too big string"); } return Qnil; } /* - * call-seq: + * call-seq * ec.putback -> string * ec.putback(max_numbytes) -> string * @@ -4127,9 +3942,10 @@ econv_putback(int argc, VALUE *argv, VALUE self) int putbackable; VALUE str, max; - if (!rb_check_arity(argc, 0, 1) || NIL_P(max = argv[0])) { + rb_scan_args(argc, argv, "01", &max); + + if (NIL_P(max)) n = rb_econv_putbackable(ec); - } else { n = NUM2INT(max); putbackable = rb_econv_putbackable(ec); @@ -4267,7 +4083,7 @@ rb_econv_check_error(rb_econv_t *ec) static VALUE ecerr_source_encoding_name(VALUE self) { - return rb_attr_get(self, id_source_encoding_name); + return rb_attr_get(self, rb_intern("source_encoding_name")); } /* @@ -4293,7 +4109,7 @@ ecerr_source_encoding_name(VALUE self) static VALUE ecerr_source_encoding(VALUE self) { - return rb_attr_get(self, id_source_encoding); + return rb_attr_get(self, rb_intern("source_encoding")); } /* @@ -4305,7 +4121,7 @@ ecerr_source_encoding(VALUE self) static VALUE ecerr_destination_encoding_name(VALUE self) { - return rb_attr_get(self, id_destination_encoding_name); + return rb_attr_get(self, rb_intern("destination_encoding_name")); } /* @@ -4317,7 +4133,7 @@ ecerr_destination_encoding_name(VALUE self) static VALUE ecerr_destination_encoding(VALUE self) { - return rb_attr_get(self, id_destination_encoding); + return rb_attr_get(self, rb_intern("destination_encoding")); } /* @@ -4338,7 +4154,7 @@ ecerr_destination_encoding(VALUE self) static VALUE ecerr_error_char(VALUE self) { - return rb_attr_get(self, id_error_char); + return rb_attr_get(self, rb_intern("error_char")); } /* @@ -4359,7 +4175,7 @@ ecerr_error_char(VALUE self) static VALUE ecerr_error_bytes(VALUE self) { - return rb_attr_get(self, id_error_bytes); + return rb_attr_get(self, rb_intern("error_bytes")); } /* @@ -4371,7 +4187,7 @@ ecerr_error_bytes(VALUE self) static VALUE ecerr_readagain_bytes(VALUE self) { - return rb_attr_get(self, id_readagain_bytes); + return rb_attr_get(self, rb_intern("readagain_bytes")); } /* @@ -4401,9 +4217,11 @@ ecerr_readagain_bytes(VALUE self) static VALUE ecerr_incomplete_input(VALUE self) { - return rb_attr_get(self, id_incomplete_input); + return rb_attr_get(self, rb_intern("incomplete_input")); } +extern void Init_newline(void); + /* * Document-class: Encoding::UndefinedConversionError * @@ -4429,60 +4247,36 @@ ecerr_incomplete_input(VALUE self) void Init_transcode(void) { - transcoder_table = st_init_strcasetable(); - - id_destination_encoding = rb_intern_const("destination_encoding"); - id_destination_encoding_name = rb_intern_const("destination_encoding_name"); - id_error_bytes = rb_intern_const("error_bytes"); - id_error_char = rb_intern_const("error_char"); - id_incomplete_input = rb_intern_const("incomplete_input"); - id_readagain_bytes = rb_intern_const("readagain_bytes"); - id_source_encoding = rb_intern_const("source_encoding"); - id_source_encoding_name = rb_intern_const("source_encoding_name"); - - sym_invalid = ID2SYM(rb_intern_const("invalid")); - sym_undef = ID2SYM(rb_intern_const("undef")); - sym_replace = ID2SYM(rb_intern_const("replace")); - sym_fallback = ID2SYM(rb_intern_const("fallback")); - sym_xml = ID2SYM(rb_intern_const("xml")); - sym_text = ID2SYM(rb_intern_const("text")); - sym_attr = ID2SYM(rb_intern_const("attr")); - - sym_invalid_byte_sequence = ID2SYM(rb_intern_const("invalid_byte_sequence")); - sym_undefined_conversion = ID2SYM(rb_intern_const("undefined_conversion")); - sym_destination_buffer_full = ID2SYM(rb_intern_const("destination_buffer_full")); - sym_source_buffer_empty = ID2SYM(rb_intern_const("source_buffer_empty")); - sym_finished = ID2SYM(rb_intern_const("finished")); - sym_after_output = ID2SYM(rb_intern_const("after_output")); - sym_incomplete_input = ID2SYM(rb_intern_const("incomplete_input")); - sym_universal_newline = ID2SYM(rb_intern_const("universal_newline")); - sym_crlf_newline = ID2SYM(rb_intern_const("crlf_newline")); - sym_cr_newline = ID2SYM(rb_intern_const("cr_newline")); - sym_lf_newline = ID2SYM(rb_intern("lf_newline")); - sym_partial_input = ID2SYM(rb_intern_const("partial_input")); - -#ifdef ENABLE_ECONV_NEWLINE_OPTION - sym_newline = ID2SYM(rb_intern_const("newline")); - sym_universal = ID2SYM(rb_intern_const("universal")); - sym_crlf = ID2SYM(rb_intern_const("crlf")); - sym_cr = ID2SYM(rb_intern_const("cr")); - sym_lf = ID2SYM(rb_intern_const("lf")); -#endif - - InitVM(transcode); -} - -void -InitVM_transcode(void) -{ rb_eUndefinedConversionError = rb_define_class_under(rb_cEncoding, "UndefinedConversionError", rb_eEncodingError); rb_eInvalidByteSequenceError = rb_define_class_under(rb_cEncoding, "InvalidByteSequenceError", rb_eEncodingError); rb_eConverterNotFoundError = rb_define_class_under(rb_cEncoding, "ConverterNotFoundError", rb_eEncodingError); + transcoder_table = st_init_strcasetable(); + + sym_invalid = ID2SYM(rb_intern("invalid")); + sym_undef = ID2SYM(rb_intern("undef")); + sym_replace = ID2SYM(rb_intern("replace")); + sym_fallback = ID2SYM(rb_intern("fallback")); + sym_xml = ID2SYM(rb_intern("xml")); + sym_text = ID2SYM(rb_intern("text")); + sym_attr = ID2SYM(rb_intern("attr")); + + sym_invalid_byte_sequence = ID2SYM(rb_intern("invalid_byte_sequence")); + sym_undefined_conversion = ID2SYM(rb_intern("undefined_conversion")); + sym_destination_buffer_full = ID2SYM(rb_intern("destination_buffer_full")); + sym_source_buffer_empty = ID2SYM(rb_intern("source_buffer_empty")); + sym_finished = ID2SYM(rb_intern("finished")); + sym_after_output = ID2SYM(rb_intern("after_output")); + sym_incomplete_input = ID2SYM(rb_intern("incomplete_input")); + sym_universal_newline = ID2SYM(rb_intern("universal_newline")); + sym_crlf_newline = ID2SYM(rb_intern("crlf_newline")); + sym_cr_newline = ID2SYM(rb_intern("cr_newline")); + sym_partial_input = ID2SYM(rb_intern("partial_input")); + rb_define_method(rb_cString, "encode", str_encode, -1); rb_define_method(rb_cString, "encode!", str_encode_bang, -1); - rb_cEncodingConverter = rb_define_class_under(rb_cEncoding, "Converter", rb_cObject); + rb_cEncodingConverter = rb_define_class_under(rb_cEncoding, "Converter", rb_cData); rb_define_alloc_func(rb_cEncodingConverter, econv_s_allocate); rb_define_singleton_method(rb_cEncodingConverter, "asciicompat_encoding", econv_s_asciicompat_encoding, 1); rb_define_singleton_method(rb_cEncodingConverter, "search_convpath", econv_s_search_convpath, -1); @@ -4500,95 +4294,19 @@ InitVM_transcode(void) rb_define_method(rb_cEncodingConverter, "last_error", econv_last_error, 0); rb_define_method(rb_cEncodingConverter, "replacement", econv_get_replacement, 0); rb_define_method(rb_cEncodingConverter, "replacement=", econv_set_replacement, 1); - rb_define_method(rb_cEncodingConverter, "==", econv_equal, 1); - /* Document-const: INVALID_MASK - * - * Mask for invalid byte sequences - */ rb_define_const(rb_cEncodingConverter, "INVALID_MASK", INT2FIX(ECONV_INVALID_MASK)); - - /* Document-const: INVALID_REPLACE - * - * Replace invalid byte sequences - */ rb_define_const(rb_cEncodingConverter, "INVALID_REPLACE", INT2FIX(ECONV_INVALID_REPLACE)); - - /* Document-const: UNDEF_MASK - * - * Mask for a valid character in the source encoding but no related - * character(s) in destination encoding. - */ rb_define_const(rb_cEncodingConverter, "UNDEF_MASK", INT2FIX(ECONV_UNDEF_MASK)); - - /* Document-const: UNDEF_REPLACE - * - * Replace byte sequences that are undefined in the destination encoding. - */ rb_define_const(rb_cEncodingConverter, "UNDEF_REPLACE", INT2FIX(ECONV_UNDEF_REPLACE)); - - /* Document-const: UNDEF_HEX_CHARREF - * - * Replace byte sequences that are undefined in the destination encoding - * with an XML hexadecimal character reference. This is valid for XML - * conversion. - */ rb_define_const(rb_cEncodingConverter, "UNDEF_HEX_CHARREF", INT2FIX(ECONV_UNDEF_HEX_CHARREF)); - - /* Document-const: PARTIAL_INPUT - * - * Indicates the source may be part of a larger string. See - * primitive_convert for an example. - */ rb_define_const(rb_cEncodingConverter, "PARTIAL_INPUT", INT2FIX(ECONV_PARTIAL_INPUT)); - - /* Document-const: AFTER_OUTPUT - * - * Stop converting after some output is complete but before all of the - * input was consumed. See primitive_convert for an example. - */ rb_define_const(rb_cEncodingConverter, "AFTER_OUTPUT", INT2FIX(ECONV_AFTER_OUTPUT)); - - /* Document-const: UNIVERSAL_NEWLINE_DECORATOR - * - * Decorator for converting CRLF and CR to LF - */ rb_define_const(rb_cEncodingConverter, "UNIVERSAL_NEWLINE_DECORATOR", INT2FIX(ECONV_UNIVERSAL_NEWLINE_DECORATOR)); - - /* Document-const: LF_NEWLINE_DECORATOR - * - * Decorator for converting CRLF and CR to LF when writing - */ - rb_define_const(rb_cEncodingConverter, "LF_NEWLINE_DECORATOR", INT2FIX(ECONV_LF_NEWLINE_DECORATOR)); - - /* Document-const: CRLF_NEWLINE_DECORATOR - * - * Decorator for converting LF to CRLF - */ rb_define_const(rb_cEncodingConverter, "CRLF_NEWLINE_DECORATOR", INT2FIX(ECONV_CRLF_NEWLINE_DECORATOR)); - - /* Document-const: CR_NEWLINE_DECORATOR - * - * Decorator for converting LF to CR - */ rb_define_const(rb_cEncodingConverter, "CR_NEWLINE_DECORATOR", INT2FIX(ECONV_CR_NEWLINE_DECORATOR)); - - /* Document-const: XML_TEXT_DECORATOR - * - * Escape as XML CharData - */ rb_define_const(rb_cEncodingConverter, "XML_TEXT_DECORATOR", INT2FIX(ECONV_XML_TEXT_DECORATOR)); - - /* Document-const: XML_ATTR_CONTENT_DECORATOR - * - * Escape as XML AttValue - */ rb_define_const(rb_cEncodingConverter, "XML_ATTR_CONTENT_DECORATOR", INT2FIX(ECONV_XML_ATTR_CONTENT_DECORATOR)); - - /* Document-const: XML_ATTR_QUOTE_DECORATOR - * - * Escape as XML AttValue - */ rb_define_const(rb_cEncodingConverter, "XML_ATTR_QUOTE_DECORATOR", INT2FIX(ECONV_XML_ATTR_QUOTE_DECORATOR)); rb_define_method(rb_eUndefinedConversionError, "source_encoding_name", ecerr_source_encoding_name, 0); |