diff options
Diffstat (limited to 'transcode.c')
-rw-r--r-- | transcode.c | 615 |
1 files changed, 348 insertions, 267 deletions
diff --git a/transcode.c b/transcode.c index e6e585e901..892814ee3b 100644 --- a/transcode.c +++ b/transcode.c @@ -47,6 +47,7 @@ static VALUE sym_xml, sym_text, sym_attr; static VALUE sym_universal_newline; static VALUE sym_crlf_newline; static VALUE sym_cr_newline; +static VALUE sym_lf_newline; #ifdef ENABLE_ECONV_NEWLINE_OPTION static VALUE sym_newline, sym_universal, sym_crlf, sym_cr, sym_lf; #endif @@ -180,6 +181,28 @@ typedef struct { static st_table *transcoder_table; +static int +free_inner_transcode_i(st_data_t key, st_data_t val, st_data_t arg) +{ + xfree((void *)val); + return ST_DELETE; +} + +static int +free_transcode_i(st_data_t key, st_data_t val, st_data_t arg) +{ + st_foreach((void *)val, free_inner_transcode_i, 0); + st_free_table((void *)val); + return ST_DELETE; +} + +void +rb_free_transcoder_table(void) +{ + st_foreach(transcoder_table, free_transcode_i, 0); + st_free_table(transcoder_table); +} + static transcoder_entry_t * make_transcoder_entry(const char *sname, const char *dname) { @@ -229,8 +252,8 @@ rb_register_transcoder(const rb_transcoder *tr) entry = make_transcoder_entry(sname, dname); if (entry->transcoder) { - rb_raise(rb_eArgError, "transcoder from %s to %s has been already registered", - sname, dname); + rb_raise(rb_eArgError, "transcoder from %s to %s has been already registered", + sname, dname); } entry->transcoder = tr; @@ -251,7 +274,7 @@ void rb_declare_transcoder(const char *enc1, const char *enc2, const char *lib) { if (!lib) { - rb_raise(rb_eArgError, "invalid library name - (null)"); + rb_raise(rb_eArgError, "invalid library name - (null)"); } declare_transcoder(enc1, enc2, lib); } @@ -535,7 +558,7 @@ transcode_restartable0(const unsigned char **in_pos, unsigned char **out_pos, while (1) { inchar_start = in_p; tc->recognized_len = 0; - next_table = tr->conv_tree_start; + next_table = tr->conv_tree_start; SUSPEND_AFTER_OUTPUT(24); @@ -555,7 +578,7 @@ transcode_restartable0(const unsigned char **in_pos, unsigned char **out_pos, #define BL_OFFSET(byte) (BL_BASE[2+(byte)-BL_MIN_BYTE]) #define BL_ACTION(byte) (BL_INFO[BL_OFFSET((byte))]) - next_byte = (unsigned char)*in_p++; + next_byte = (unsigned char)*in_p++; follow_byte: if (next_byte < BL_MIN_BYTE || BL_MAX_BYTE < next_byte) next_info = INVALID; @@ -563,8 +586,8 @@ transcode_restartable0(const unsigned char **in_pos, unsigned char **out_pos, next_info = (VALUE)BL_ACTION(next_byte); } follow_info: - switch (next_info & 0x1F) { - case NOMAP: + switch (next_info & 0x1F) { + case NOMAP: { const unsigned char *p = inchar_start; writebuf_off = 0; @@ -579,43 +602,43 @@ transcode_restartable0(const unsigned char **in_pos, unsigned char **out_pos, } } continue; - case 0x00: case 0x04: case 0x08: case 0x0C: - case 0x10: case 0x14: case 0x18: case 0x1C: + case 0x00: case 0x04: case 0x08: case 0x0C: + case 0x10: case 0x14: case 0x18: case 0x1C: SUSPEND_AFTER_OUTPUT(25); - while (in_p >= in_stop) { + while (in_p >= in_stop) { if (!(opt & ECONV_PARTIAL_INPUT)) goto incomplete; SUSPEND(econv_source_buffer_empty, 5); - } - next_byte = (unsigned char)*in_p++; - next_table = (unsigned int)next_info; - goto follow_byte; - case ZERObt: /* drop input */ - continue; - case ONEbt: + } + next_byte = (unsigned char)*in_p++; + next_table = (unsigned int)next_info; + goto follow_byte; + case ZERObt: /* drop input */ + continue; + case ONEbt: SUSPEND_OBUF(9); *out_p++ = getBT1(next_info); - continue; - case TWObt: + continue; + case TWObt: SUSPEND_OBUF(10); *out_p++ = getBT1(next_info); SUSPEND_OBUF(21); *out_p++ = getBT2(next_info); - continue; - case THREEbt: + continue; + case THREEbt: SUSPEND_OBUF(11); *out_p++ = getBT1(next_info); SUSPEND_OBUF(15); *out_p++ = getBT2(next_info); SUSPEND_OBUF(16); *out_p++ = getBT3(next_info); - continue; - case FOURbt: + continue; + case FOURbt: SUSPEND_OBUF(12); *out_p++ = getBT0(next_info); SUSPEND_OBUF(17); *out_p++ = getBT1(next_info); SUSPEND_OBUF(18); *out_p++ = getBT2(next_info); SUSPEND_OBUF(19); *out_p++ = getBT3(next_info); - continue; - case GB4bt: + continue; + case GB4bt: SUSPEND_OBUF(29); *out_p++ = getGB4bt0(next_info); SUSPEND_OBUF(30); *out_p++ = getGB4bt1(next_info); SUSPEND_OBUF(31); *out_p++ = getGB4bt2(next_info); SUSPEND_OBUF(32); *out_p++ = getGB4bt3(next_info); - continue; + continue; case STR1: tc->output_index = 0; while (tc->output_index < STR1_LENGTH(BYTE_ADDR(STR1_BYTEINDEX(next_info)))) { @@ -623,10 +646,10 @@ transcode_restartable0(const unsigned char **in_pos, unsigned char **out_pos, tc->output_index++; } continue; - case FUNii: - next_info = (VALUE)(*tr->func_ii)(TRANSCODING_STATE(tc), next_info); - goto follow_info; - case FUNsi: + case FUNii: + next_info = (VALUE)(*tr->func_ii)(TRANSCODING_STATE(tc), next_info); + goto follow_info; + case FUNsi: { const unsigned char *char_start; size_t char_len; @@ -634,7 +657,7 @@ transcode_restartable0(const unsigned char **in_pos, unsigned char **out_pos, next_info = (VALUE)(*tr->func_si)(TRANSCODING_STATE(tc), char_start, (size_t)char_len); goto follow_info; } - case FUNio: + case FUNio: SUSPEND_OBUF(13); if (tr->max_output <= out_stop - out_p) out_p += tr->func_io(TRANSCODING_STATE(tc), @@ -649,8 +672,8 @@ transcode_restartable0(const unsigned char **in_pos, unsigned char **out_pos, *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++]; } } - break; - case FUNso: + break; + case FUNso: { const unsigned char *char_start; size_t char_len; @@ -674,7 +697,7 @@ transcode_restartable0(const unsigned char **in_pos, unsigned char **out_pos, } break; } - case FUNsio: + case FUNsio: { const unsigned char *char_start; size_t char_len; @@ -698,7 +721,7 @@ transcode_restartable0(const unsigned char **in_pos, unsigned char **out_pos, } break; } - case INVALID: + case INVALID: if (tc->recognized_len + (in_p - inchar_start) <= unitlen) { if (tc->recognized_len + (in_p - inchar_start) < unitlen) SUSPEND_AFTER_OUTPUT(26); @@ -721,12 +744,12 @@ transcode_restartable0(const unsigned char **in_pos, unsigned char **out_pos, readagain_len = invalid_len - discard_len; } goto invalid; - case UNDEF: - goto undef; - default: - rb_raise(rb_eRuntimeError, "unknown transcoding instruction"); - } - continue; + case UNDEF: + goto undef; + default: + rb_raise(rb_eRuntimeError, "unknown transcoding instruction"); + } + continue; invalid: SUSPEND(econv_invalid_byte_sequence, 1); @@ -856,13 +879,13 @@ rb_transcoding_memsize(rb_transcoding *tc) const rb_transcoder *tr = tc->transcoder; if (TRANSCODING_STATE_EMBED_MAX < tr->state_size) { - size += tr->state_size; + size += tr->state_size; } if ((int)sizeof(tc->readbuf.ary) < tr->max_input) { - size += tr->max_input; + size += tr->max_input; } if ((int)sizeof(tc->writebuf.ary) < tr->max_output) { - size += tr->max_output; + size += tr->max_output; } return size; } @@ -1002,7 +1025,7 @@ rb_econv_open0(const char *sname, const char *dname, int ecflags) if (*sname == '\0' && *dname == '\0') { num_trans = 0; entries = NULL; - sname = dname = ""; + sname = dname = ""; } else { struct trans_open_t toarg; @@ -1039,8 +1062,9 @@ decorator_names(int ecflags, const char **decorators_ret) case ECONV_UNIVERSAL_NEWLINE_DECORATOR: case ECONV_CRLF_NEWLINE_DECORATOR: case ECONV_CR_NEWLINE_DECORATOR: + case ECONV_LF_NEWLINE_DECORATOR: case 0: - break; + break; default: return -1; } @@ -1062,6 +1086,8 @@ decorator_names(int ecflags, const char **decorators_ret) decorators_ret[num_decorators++] = "crlf_newline"; if (ecflags & ECONV_CR_NEWLINE_DECORATOR) decorators_ret[num_decorators++] = "cr_newline"; + if (ecflags & ECONV_LF_NEWLINE_DECORATOR) + decorators_ret[num_decorators++] = "lf_newline"; if (ecflags & ECONV_UNIVERSAL_NEWLINE_DECORATOR) decorators_ret[num_decorators++] = "universal_newline"; @@ -1473,22 +1499,22 @@ rb_econv_convert(rb_econv_t *ec, if (ret == econv_invalid_byte_sequence || ret == econv_incomplete_input) { - /* deal with invalid byte sequence */ - /* todo: add more alternative behaviors */ + /* deal with invalid byte sequence */ + /* todo: add more alternative behaviors */ switch (ec->flags & ECONV_INVALID_MASK) { case ECONV_INVALID_REPLACE: - if (output_replacement_character(ec) == 0) + if (output_replacement_character(ec) == 0) goto resume; - } + } } if (ret == econv_undefined_conversion) { - /* valid character in source encoding - * but no related character(s) in destination encoding */ - /* todo: add more alternative behaviors */ + /* valid character in source encoding + * but no related character(s) in destination encoding */ + /* todo: add more alternative behaviors */ switch (ec->flags & ECONV_UNDEF_MASK) { case ECONV_UNDEF_REPLACE: - if (output_replacement_character(ec) == 0) + if (output_replacement_character(ec) == 0) goto resume; break; @@ -1711,8 +1737,7 @@ rb_econv_close(rb_econv_t *ec) } for (i = 0; i < ec->num_trans; i++) { rb_transcoding_close(ec->elems[i].tc); - if (ec->elems[i].out_buf_start) - xfree(ec->elems[i].out_buf_start); + xfree(ec->elems[i].out_buf_start); } xfree(ec->in_buf_start); xfree(ec->elems); @@ -1726,14 +1751,14 @@ rb_econv_memsize(rb_econv_t *ec) int i; if (ec->replacement_allocated) { - size += ec->replacement_len; + size += ec->replacement_len; } for (i = 0; i < ec->num_trans; i++) { - size += rb_transcoding_memsize(ec->elems[i].tc); + size += rb_transcoding_memsize(ec->elems[i].tc); - if (ec->elems[i].out_buf_start) { + if (ec->elems[i].out_buf_start) { size += ec->elems[i].out_buf_end - ec->elems[i].out_buf_start; - } + } } size += ec->in_buf_end - ec->in_buf_start; size += sizeof(rb_econv_elem_t) * ec->num_allocated; @@ -1812,6 +1837,12 @@ rb_econv_asciicompat_encoding(const char *ascii_incompat_name) return data.ascii_compat_name; } +/* + * Append `len` bytes pointed by `ss` to `dst` with converting with `ec`. + * + * If the result of the conversion is not compatible with the encoding of + * `dst`, `dst` may not be valid encoding. + */ VALUE rb_econv_append(rb_econv_t *ec, const char *ss, long len, VALUE dst, int flags) { @@ -1819,11 +1850,19 @@ rb_econv_append(rb_econv_t *ec, const char *ss, long len, VALUE dst, int flags) unsigned char *ds, *dp, *de; rb_econv_result_t res; int max_output; + enum ruby_coderange_type coderange; + rb_encoding *dst_enc = ec->destination_encoding; if (NIL_P(dst)) { dst = rb_str_buf_new(len); - if (ec->destination_encoding) - rb_enc_associate(dst, ec->destination_encoding); + if (dst_enc) { + rb_enc_associate(dst, dst_enc); + } + coderange = ENC_CODERANGE_7BIT; // scan from the start + } + else { + dst_enc = rb_enc_get(dst); + coderange = rb_enc_str_coderange(dst); } if (ec->last_tc) @@ -1832,13 +1871,13 @@ rb_econv_append(rb_econv_t *ec, const char *ss, long len, VALUE dst, int flags) max_output = 1; do { + int cr; long dlen = RSTRING_LEN(dst); if (rb_str_capacity(dst) - dlen < (size_t)len + max_output) { unsigned long new_capa = (unsigned long)dlen + len + max_output; if (LONG_MAX < new_capa) rb_raise(rb_eArgError, "too long string"); - rb_str_resize(dst, new_capa); - rb_str_set_len(dst, dlen); + rb_str_modify_expand(dst, new_capa - dlen); } sp = (const unsigned char *)ss; se = sp + len; @@ -1846,6 +1885,18 @@ rb_econv_append(rb_econv_t *ec, const char *ss, long len, VALUE dst, int flags) de = ds + rb_str_capacity(dst); dp = ds += dlen; res = rb_econv_convert(ec, &sp, se, &dp, de, flags); + switch (coderange) { + case ENC_CODERANGE_7BIT: + case ENC_CODERANGE_VALID: + cr = (int)coderange; + rb_str_coderange_scan_restartable((char *)ds, (char *)dp, dst_enc, &cr); + coderange = cr; + ENC_CODERANGE_SET(dst, coderange); + break; + case ENC_CODERANGE_UNKNOWN: + case ENC_CODERANGE_BROKEN: + break; + } len -= (const char *)sp - ss; ss = (const char *)sp; rb_str_set_len(dst, dlen + (dp - ds)); @@ -1948,30 +1999,33 @@ rb_econv_binmode(rb_econv_t *ec) switch (ec->flags & ECONV_NEWLINE_DECORATOR_MASK) { case ECONV_UNIVERSAL_NEWLINE_DECORATOR: - dname = "universal_newline"; - break; + dname = "universal_newline"; + break; case ECONV_CRLF_NEWLINE_DECORATOR: - dname = "crlf_newline"; - break; + dname = "crlf_newline"; + break; case ECONV_CR_NEWLINE_DECORATOR: - dname = "cr_newline"; - break; + dname = "cr_newline"; + break; + case ECONV_LF_NEWLINE_DECORATOR: + dname = "lf_newline"; + break; } if (dname) { const rb_transcoder *transcoder = get_transcoder_entry("", dname)->transcoder; int num_trans = ec->num_trans; - int i, j = 0; + int i, j = 0; - for (i=0; i < num_trans; i++) { - if (transcoder == ec->elems[i].tc->transcoder) { - rb_transcoding_close(ec->elems[i].tc); - xfree(ec->elems[i].out_buf_start); - ec->num_trans--; - } - else - ec->elems[j++] = ec->elems[i]; - } + for (i=0; i < num_trans; i++) { + if (transcoder == ec->elems[i].tc->transcoder) { + rb_transcoding_close(ec->elems[i].tc); + xfree(ec->elems[i].out_buf_start); + ec->num_trans--; + } + else + ec->elems[j++] = ec->elems[i]; + } } ec->flags &= ~ECONV_NEWLINE_DECORATOR_MASK; @@ -2014,6 +2068,10 @@ econv_description(const char *sname, const char *dname, int ecflags, VALUE mesg) rb_str_cat2(mesg, pre); pre = ","; rb_str_cat2(mesg, "cr_newline"); } + if (ecflags & ECONV_LF_NEWLINE_DECORATOR) { + rb_str_cat2(mesg, pre); pre = ","; + rb_str_cat2(mesg, "lf_newline"); + } if (ecflags & ECONV_XML_TEXT_DECORATOR) { rb_str_cat2(mesg, pre); pre = ","; rb_str_cat2(mesg, "xml_text"); @@ -2267,7 +2325,7 @@ aref_fallback(VALUE fallback, VALUE c) static void transcode_loop(const unsigned char **in_pos, unsigned char **out_pos, - const unsigned char *in_stop, unsigned char *out_stop, + const unsigned char *in_stop, unsigned char *out_stop, VALUE destination, unsigned char *(*resize_destination)(VALUE, size_t, size_t), const char *src_encoding, @@ -2289,19 +2347,19 @@ transcode_loop(const unsigned char **in_pos, unsigned char **out_pos, rb_exc_raise(rb_econv_open_exc(src_encoding, dst_encoding, ecflags)); if (!NIL_P(ecopts) && RB_TYPE_P(ecopts, T_HASH)) { - fallback = rb_hash_aref(ecopts, sym_fallback); - if (RB_TYPE_P(fallback, T_HASH)) { - fallback_func = hash_fallback; - } - else if (rb_obj_is_proc(fallback)) { - fallback_func = proc_fallback; - } - else if (rb_obj_is_method(fallback)) { - fallback_func = method_fallback; - } - else { - fallback_func = aref_fallback; - } + fallback = rb_hash_aref(ecopts, sym_fallback); + if (RB_TYPE_P(fallback, T_HASH)) { + fallback_func = hash_fallback; + } + else if (rb_obj_is_proc(fallback)) { + fallback_func = proc_fallback; + } + else if (rb_obj_is_method(fallback)) { + fallback_func = method_fallback; + } + else { + fallback_func = aref_fallback; + } } last_tc = ec->last_tc; max_output = last_tc ? last_tc->transcoder->max_output : 1; @@ -2310,20 +2368,20 @@ transcode_loop(const unsigned char **in_pos, unsigned char **out_pos, ret = rb_econv_convert(ec, in_pos, in_stop, out_pos, out_stop, 0); if (!NIL_P(fallback) && ret == econv_undefined_conversion) { - VALUE rep = rb_enc_str_new( - (const char *)ec->last_error.error_bytes_start, - ec->last_error.error_bytes_len, - rb_enc_find(ec->last_error.source_encoding)); - rep = (*fallback_func)(fallback, rep); - if (rep != Qundef && !NIL_P(rep)) { - StringValue(rep); - ret = rb_econv_insert_output(ec, (const unsigned char *)RSTRING_PTR(rep), - RSTRING_LEN(rep), rb_enc_name(rb_enc_get(rep))); - if ((int)ret == -1) { - rb_raise(rb_eArgError, "too big fallback string"); - } - goto resume; - } + VALUE rep = rb_enc_str_new( + (const char *)ec->last_error.error_bytes_start, + ec->last_error.error_bytes_len, + rb_enc_find(ec->last_error.source_encoding)); + rep = (*fallback_func)(fallback, rep); + if (!UNDEF_P(rep) && !NIL_P(rep)) { + StringValue(rep); + ret = rb_econv_insert_output(ec, (const unsigned char *)RSTRING_PTR(rep), + RSTRING_LEN(rep), rb_enc_name(rb_enc_get(rep))); + if ((int)ret == -1) { + rb_raise(rb_eArgError, "too big fallback string"); + } + goto resume; + } } if (ret == econv_invalid_byte_sequence || @@ -2331,7 +2389,7 @@ transcode_loop(const unsigned char **in_pos, unsigned char **out_pos, ret == econv_undefined_conversion) { exc = make_econv_exception(ec); rb_econv_close(ec); - rb_exc_raise(exc); + rb_exc_raise(exc); } if (ret == econv_destination_buffer_full) { @@ -2346,7 +2404,7 @@ transcode_loop(const unsigned char **in_pos, unsigned char **out_pos, /* sample transcode_loop implementation in byte-by-byte stream style */ static void transcode_loop(const unsigned char **in_pos, unsigned char **out_pos, - const unsigned char *in_stop, unsigned char *out_stop, + const unsigned char *in_stop, unsigned char *out_stop, VALUE destination, unsigned char *(*resize_destination)(VALUE, size_t, size_t), const char *src_encoding, @@ -2478,56 +2536,61 @@ econv_opts(VALUE opt, int ecflags) v = rb_hash_aref(opt, sym_newline); if (!NIL_P(v)) { newlineflag = 2; - ecflags &= ~ECONV_NEWLINE_DECORATOR_MASK; - if (v == sym_universal) { - ecflags |= ECONV_UNIVERSAL_NEWLINE_DECORATOR; - } - else if (v == sym_crlf) { - ecflags |= ECONV_CRLF_NEWLINE_DECORATOR; - } - else if (v == sym_cr) { - ecflags |= ECONV_CR_NEWLINE_DECORATOR; - } - else if (v == sym_lf) { - /* ecflags |= ECONV_LF_NEWLINE_DECORATOR; */ - } - else if (SYMBOL_P(v)) { - rb_raise(rb_eArgError, "unexpected value for newline option: %"PRIsVALUE, - rb_sym2str(v)); - } - else { - rb_raise(rb_eArgError, "unexpected value for newline option"); - } + ecflags &= ~ECONV_NEWLINE_DECORATOR_MASK; + if (v == sym_universal) { + ecflags |= ECONV_UNIVERSAL_NEWLINE_DECORATOR; + } + else if (v == sym_crlf) { + ecflags |= ECONV_CRLF_NEWLINE_DECORATOR; + } + else if (v == sym_cr) { + ecflags |= ECONV_CR_NEWLINE_DECORATOR; + } + else if (v == sym_lf) { + ecflags |= ECONV_LF_NEWLINE_DECORATOR; + } + else if (SYMBOL_P(v)) { + rb_raise(rb_eArgError, "unexpected value for newline option: %"PRIsVALUE, + rb_sym2str(v)); + } + else { + rb_raise(rb_eArgError, "unexpected value for newline option"); + } } #endif { int setflags = 0; - v = rb_hash_aref(opt, sym_universal_newline); - if (RTEST(v)) - setflags |= ECONV_UNIVERSAL_NEWLINE_DECORATOR; - newlineflag |= !NIL_P(v); + v = rb_hash_aref(opt, sym_universal_newline); + if (RTEST(v)) + setflags |= ECONV_UNIVERSAL_NEWLINE_DECORATOR; + newlineflag |= !NIL_P(v); - v = rb_hash_aref(opt, sym_crlf_newline); - if (RTEST(v)) - setflags |= ECONV_CRLF_NEWLINE_DECORATOR; - newlineflag |= !NIL_P(v); + v = rb_hash_aref(opt, sym_crlf_newline); + if (RTEST(v)) + setflags |= ECONV_CRLF_NEWLINE_DECORATOR; + newlineflag |= !NIL_P(v); - v = rb_hash_aref(opt, sym_cr_newline); - if (RTEST(v)) - setflags |= ECONV_CR_NEWLINE_DECORATOR; - newlineflag |= !NIL_P(v); + v = rb_hash_aref(opt, sym_cr_newline); + if (RTEST(v)) + setflags |= ECONV_CR_NEWLINE_DECORATOR; + newlineflag |= !NIL_P(v); + + v = rb_hash_aref(opt, sym_lf_newline); + if (RTEST(v)) + setflags |= ECONV_LF_NEWLINE_DECORATOR; + newlineflag |= !NIL_P(v); switch (newlineflag) { case 1: - ecflags &= ~ECONV_NEWLINE_DECORATOR_MASK; - ecflags |= setflags; + ecflags &= ~ECONV_NEWLINE_DECORATOR_MASK; + ecflags |= setflags; break; case 3: rb_warning(":newline option precedes other newline options"); break; - } + } } return ecflags; @@ -2547,28 +2610,28 @@ rb_econv_prepare_options(VALUE opthash, VALUE *opts, int ecflags) v = rb_hash_aref(opthash, sym_replace); if (!NIL_P(v)) { - StringValue(v); - if (rb_enc_str_coderange(v) == ENC_CODERANGE_BROKEN) { - VALUE dumped = rb_str_dump(v); - rb_raise(rb_eArgError, "replacement string is broken: %s as %s", - StringValueCStr(dumped), - rb_enc_name(rb_enc_get(v))); - } - v = rb_str_new_frozen(v); - newhash = rb_hash_new(); - rb_hash_aset(newhash, sym_replace, v); + StringValue(v); + if (is_broken_string(v)) { + VALUE dumped = rb_str_dump(v); + rb_raise(rb_eArgError, "replacement string is broken: %s as %s", + StringValueCStr(dumped), + rb_enc_name(rb_enc_get(v))); + } + v = rb_str_new_frozen(v); + newhash = rb_hash_new(); + rb_hash_aset(newhash, sym_replace, v); } v = rb_hash_aref(opthash, sym_fallback); if (!NIL_P(v)) { - VALUE h = rb_check_hash_type(v); - if (NIL_P(h) - ? (rb_obj_is_proc(v) || rb_obj_is_method(v) || rb_respond_to(v, idAREF)) - : (v = h, 1)) { - if (NIL_P(newhash)) - newhash = rb_hash_new(); - rb_hash_aset(newhash, sym_fallback, v); - } + VALUE h = rb_check_hash_type(v); + if (NIL_P(h) + ? (rb_obj_is_proc(v) || rb_obj_is_method(v) || rb_respond_to(v, idAREF)) + : (v = h, 1)) { + if (NIL_P(newhash)) + newhash = rb_hash_new(); + rb_hash_aset(newhash, sym_fallback, v); + } } if (!NIL_P(newhash)) @@ -2628,13 +2691,13 @@ enc_arg(VALUE *arg, const char **name_p, rb_encoding **enc_p) VALUE encval; if (((encidx = rb_to_encoding_index(encval = *arg)) < 0) || - !(enc = rb_enc_from_index(encidx))) { - enc = NULL; - encidx = 0; - n = StringValueCStr(*arg); + !(enc = rb_enc_from_index(encidx))) { + enc = NULL; + encidx = 0; + n = StringValueCStr(*arg); } else { - n = rb_enc_name(enc); + n = rb_enc_name(enc); } *name_p = n; @@ -2655,9 +2718,9 @@ str_transcode_enc_args(VALUE str, VALUE *arg1, VALUE *arg2, dencidx = enc_arg(arg1, &dname, &denc); if (NIL_P(*arg2)) { - sencidx = rb_enc_get_index(str); - senc = rb_enc_from_index(sencidx); - sname = rb_enc_name(senc); + sencidx = rb_enc_get_index(str); + senc = rb_enc_from_index(sencidx); + sname = rb_enc_name(senc); } else { sencidx = enc_arg(arg2, &sname, &senc); @@ -2687,18 +2750,18 @@ str_transcode0(int argc, VALUE *argv, VALUE *self, int ecflags, VALUE ecopts) rb_check_arity(argc, 0, 2); if (argc == 0) { - arg1 = rb_enc_default_internal(); - if (NIL_P(arg1)) { - if (!ecflags) return -1; - arg1 = rb_obj_encoding(str); - } - if (!(ecflags & ECONV_INVALID_MASK)) { - explicitly_invalid_replace = FALSE; - } - ecflags |= ECONV_INVALID_REPLACE | ECONV_UNDEF_REPLACE; + arg1 = rb_enc_default_internal(); + if (NIL_P(arg1)) { + if (!ecflags) return -1; + arg1 = rb_obj_encoding(str); + } + if (!(ecflags & ECONV_INVALID_MASK)) { + explicitly_invalid_replace = FALSE; + } + ecflags |= ECONV_INVALID_REPLACE | ECONV_UNDEF_REPLACE; } else { - arg1 = argv[0]; + arg1 = argv[0]; } arg2 = argc<=1 ? Qnil : argv[1]; dencidx = str_transcode_enc_args(str, &arg1, &arg2, &sname, &senc, &dname, &denc); @@ -2708,16 +2771,16 @@ str_transcode0(int argc, VALUE *argv, VALUE *self, int ecflags, VALUE ecopts) ECONV_XML_ATTR_CONTENT_DECORATOR| ECONV_XML_ATTR_QUOTE_DECORATOR)) == 0) { if (senc && senc == denc) { - if ((ecflags & ECONV_INVALID_MASK) && explicitly_invalid_replace) { - VALUE rep = Qnil; - if (!NIL_P(ecopts)) { - rep = rb_hash_aref(ecopts, sym_replace); - } - dest = rb_enc_str_scrub(senc, str, rep); - if (NIL_P(dest)) dest = str; - *self = dest; - return dencidx; - } + if ((ecflags & ECONV_INVALID_MASK) && explicitly_invalid_replace) { + VALUE rep = Qnil; + if (!NIL_P(ecopts)) { + rep = rb_hash_aref(ecopts, sym_replace); + } + dest = rb_enc_str_scrub(senc, str, rep); + if (NIL_P(dest)) dest = str; + *self = dest; + return dencidx; + } return NIL_P(arg2) ? -1 : dencidx; } if (senc && denc && rb_enc_asciicompat(senc) && rb_enc_asciicompat(denc)) { @@ -2758,9 +2821,9 @@ str_transcode0(int argc, VALUE *argv, VALUE *self, int ecflags, VALUE ecopts) /* set encoding */ if (!denc) { - dencidx = rb_define_dummy_encoding(dname); - RB_GC_GUARD(arg1); - RB_GC_GUARD(arg2); + dencidx = rb_define_dummy_encoding(dname); + RB_GC_GUARD(arg1); + RB_GC_GUARD(arg2); } *self = dest; @@ -2776,7 +2839,7 @@ str_transcode(int argc, VALUE *argv, VALUE *self) argc = rb_scan_args(argc, argv, "02:", NULL, NULL, &opt); if (!NIL_P(opt)) { - ecflags = rb_econv_prepare_opts(opt, &ecopts); + ecflags = rb_econv_prepare_opts(opt, &ecopts); } return str_transcode0(argc, argv, self, ecflags, ecopts); } @@ -2790,10 +2853,10 @@ str_encode_associate(VALUE str, int encidx) /* transcoded string never be broken. */ if (rb_enc_asciicompat(rb_enc_from_index(encidx))) { - rb_str_coderange_scan_restartable(RSTRING_PTR(str), RSTRING_END(str), 0, &cr); + rb_str_coderange_scan_restartable(RSTRING_PTR(str), RSTRING_END(str), 0, &cr); } else { - cr = ENC_CODERANGE_VALID; + cr = ENC_CODERANGE_VALID; } ENC_CODERANGE_SET(str, cr); return str; @@ -2821,8 +2884,8 @@ str_encode_bang(int argc, VALUE *argv, VALUE str) if (encidx < 0) return str; if (newstr == str) { - rb_enc_associate_index(str, encidx); - return str; + rb_enc_associate_index(str, encidx); + return str; } rb_str_shared_replace(str, newstr); return str_encode_associate(str, encidx); @@ -2830,6 +2893,15 @@ str_encode_bang(int argc, VALUE *argv, VALUE str) static VALUE encoded_dup(VALUE newstr, VALUE str, int encidx); +/* + * call-seq: + * encode(dst_encoding = Encoding.default_internal, **enc_opts) -> string + * encode(dst_encoding, src_encoding, **enc_opts) -> string + * + * :include: doc/string/encode.rdoc + * + */ + static VALUE str_encode(int argc, VALUE *argv, VALUE str) { @@ -2853,12 +2925,12 @@ encoded_dup(VALUE newstr, VALUE str, int encidx) { if (encidx < 0) return rb_str_dup(str); if (newstr == str) { - newstr = rb_str_dup(str); - rb_enc_associate_index(newstr, encidx); - return newstr; + newstr = rb_str_dup(str); + rb_enc_associate_index(newstr, encidx); + return newstr; } else { - RBASIC_SET_CLASS(newstr, rb_obj_class(str)); + RBASIC_SET_CLASS(newstr, rb_obj_class(str)); } return str_encode_associate(newstr, encidx); } @@ -2972,9 +3044,9 @@ econv_args(int argc, VALUE *argv, argc = rb_scan_args(argc, argv, "21:", snamev_p, dnamev_p, &flags_v, &opt); if (!NIL_P(flags_v)) { - if (!NIL_P(opt)) { - rb_error_arity(argc + 1, 2, 3); - } + if (!NIL_P(opt)) { + rb_error_arity(argc + 1, 2, 3); + } ecflags = NUM2INT(rb_to_int(flags_v)); ecopts = Qnil; } @@ -3030,22 +3102,22 @@ decorate_convpath(VALUE convpath, int ecflags) len = n = RARRAY_LENINT(convpath); if (n != 0) { VALUE pair = RARRAY_AREF(convpath, n-1); - if (RB_TYPE_P(pair, T_ARRAY)) { - const char *sname = rb_enc_name(rb_to_encoding(RARRAY_AREF(pair, 0))); - const char *dname = rb_enc_name(rb_to_encoding(RARRAY_AREF(pair, 1))); - transcoder_entry_t *entry = get_transcoder_entry(sname, dname); - const rb_transcoder *tr = load_transcoder_entry(entry); - if (!tr) - return -1; - if (!DECORATOR_P(tr->src_encoding, tr->dst_encoding) && - tr->asciicompat_type == asciicompat_encoder) { - n--; - rb_ary_store(convpath, len + num_decorators - 1, pair); - } - } - else { - rb_ary_store(convpath, len + num_decorators - 1, pair); - } + if (RB_TYPE_P(pair, T_ARRAY)) { + const char *sname = rb_enc_name(rb_to_encoding(RARRAY_AREF(pair, 0))); + const char *dname = rb_enc_name(rb_to_encoding(RARRAY_AREF(pair, 1))); + transcoder_entry_t *entry = get_transcoder_entry(sname, dname); + const rb_transcoder *tr = load_transcoder_entry(entry); + if (!tr) + return -1; + if (!DECORATOR_P(tr->src_encoding, tr->dst_encoding) && + tr->asciicompat_type == asciicompat_encoder) { + n--; + rb_ary_store(convpath, len + num_decorators - 1, pair); + } + } + else { + rb_ary_store(convpath, len + num_decorators - 1, pair); + } } for (i = 0; i < num_decorators; i++) @@ -3121,10 +3193,10 @@ econv_s_search_convpath(int argc, VALUE *argv, VALUE klass) } if (decorate_convpath(convpath, ecflags) == -1) { - VALUE exc = rb_econv_open_exc(sname, dname, ecflags); - RB_GC_GUARD(snamev); - RB_GC_GUARD(dnamev); - rb_exc_raise(exc); + VALUE exc = rb_econv_open_exc(sname, dname, ecflags); + RB_GC_GUARD(snamev); + RB_GC_GUARD(dnamev); + rb_exc_raise(exc); } return convpath; @@ -3140,7 +3212,7 @@ rb_econv_has_convpath_p(const char* from_encoding, const char* to_encoding) { VALUE convpath = Qnil; transcode_search_path(from_encoding, to_encoding, search_convpath_i, - &convpath); + &convpath); return RTEST(convpath); } @@ -3198,12 +3270,12 @@ rb_econv_init_by_convpath(VALUE self, VALUE convpath, } if (DECORATOR_P(sname, dname)) { ret = rb_econv_add_converter(ec, sname, dname, ec->num_trans); - if (ret == -1) { - VALUE msg = rb_sprintf("decoration failed: %s", dname); - RB_GC_GUARD(snamev); - RB_GC_GUARD(dnamev); - rb_exc_raise(rb_exc_new_str(rb_eArgError, msg)); - } + if (ret == -1) { + VALUE msg = rb_sprintf("decoration failed: %s", dname); + RB_GC_GUARD(snamev); + RB_GC_GUARD(dnamev); + rb_exc_raise(rb_exc_new_str(rb_eArgError, msg)); + } } else { int j = ec->num_trans; @@ -3212,12 +3284,12 @@ rb_econv_init_by_convpath(VALUE self, VALUE convpath, arg.index = ec->num_trans; arg.ret = 0; ret = transcode_search_path(sname, dname, rb_econv_init_by_convpath_i, &arg); - if (ret == -1 || arg.ret == -1) { - VALUE msg = rb_sprintf("adding conversion failed: %s to %s", sname, dname); - RB_GC_GUARD(snamev); - RB_GC_GUARD(dnamev); + if (ret == -1 || arg.ret == -1) { + VALUE msg = rb_sprintf("adding conversion failed: %s to %s", sname, dname); + RB_GC_GUARD(snamev); + RB_GC_GUARD(dnamev); rb_exc_raise(rb_exc_new_str(rb_eArgError, msg)); - } + } if (first) { first = 0; *senc_p = senc; @@ -3229,10 +3301,10 @@ rb_econv_init_by_convpath(VALUE self, VALUE convpath, } if (first) { - *senc_p = NULL; - *denc_p = NULL; - *sname_p = ""; - *dname_p = ""; + *senc_p = NULL; + *denc_p = NULL; + *sname_p = ""; + *dname_p = ""; } ec->source_encoding_name = *sname_p; @@ -3255,11 +3327,13 @@ rb_econv_init_by_convpath(VALUE self, VALUE convpath, * :undef => :replace # replace undefined conversion * :replace => string # replacement string ("?" or "\uFFFD" if not specified) * :newline => :universal # decorator for converting CRLF and CR to LF + * :newline => :lf # decorator for converting CRLF and CR to LF when writing * :newline => :crlf # decorator for converting LF to CRLF * :newline => :cr # decorator for converting LF to CR * :universal_newline => true # decorator for converting CRLF and CR to LF * :crlf_newline => true # decorator for converting LF to CRLF * :cr_newline => true # decorator for converting LF to CR + * :lf_newline => true # decorator for converting CRLF and CR to LF when writing * :xml => :text # escape as XML CharData. * :xml => :attr # escape as XML AttValue * integer form: @@ -3267,6 +3341,7 @@ rb_econv_init_by_convpath(VALUE self, VALUE convpath, * Encoding::Converter::UNDEF_REPLACE * Encoding::Converter::UNDEF_HEX_CHARREF * Encoding::Converter::UNIVERSAL_NEWLINE_DECORATOR + * Encoding::Converter::LF_NEWLINE_DECORATOR * Encoding::Converter::CRLF_NEWLINE_DECORATOR * Encoding::Converter::CR_NEWLINE_DECORATOR * Encoding::Converter::XML_TEXT_DECORATOR @@ -3309,6 +3384,8 @@ rb_econv_init_by_convpath(VALUE self, VALUE convpath, * Convert LF to CRLF. * [:cr_newline => true] * Convert LF to CR. + * [:lf_newline => true] + * Convert CRLF and CR to LF (when writing). * [:xml => :text] * Escape as XML CharData. * This form can be used as an HTML 4.0 #PCDATA. @@ -3373,10 +3450,10 @@ econv_init(int argc, VALUE *argv, VALUE self) } if (!ec) { - VALUE exc = rb_econv_open_exc(sname, dname, ecflags); - RB_GC_GUARD(snamev); - RB_GC_GUARD(dnamev); - rb_exc_raise(exc); + VALUE exc = rb_econv_open_exc(sname, dname, ecflags); + RB_GC_GUARD(snamev); + RB_GC_GUARD(dnamev); + rb_exc_raise(exc); } if (!DECORATOR_P(sname, dname)) { @@ -3384,8 +3461,8 @@ econv_init(int argc, VALUE *argv, VALUE self) senc = make_dummy_encoding(sname); if (!denc) denc = make_dummy_encoding(dname); - RB_GC_GUARD(snamev); - RB_GC_GUARD(dnamev); + RB_GC_GUARD(snamev); + RB_GC_GUARD(dnamev); } ec->source_encoding = senc; @@ -3526,29 +3603,29 @@ econv_equal(VALUE self, VALUE other) int i; if (!rb_typeddata_is_kind_of(other, &econv_data_type)) { - return Qnil; + return Qnil; } ec2 = DATA_PTR(other); if (!ec2) return Qfalse; if (ec1->source_encoding_name != ec2->source_encoding_name && - strcmp(ec1->source_encoding_name, ec2->source_encoding_name)) - return Qfalse; + strcmp(ec1->source_encoding_name, ec2->source_encoding_name)) + return Qfalse; if (ec1->destination_encoding_name != ec2->destination_encoding_name && - strcmp(ec1->destination_encoding_name, ec2->destination_encoding_name)) - return Qfalse; + strcmp(ec1->destination_encoding_name, ec2->destination_encoding_name)) + return Qfalse; if (ec1->flags != ec2->flags) return Qfalse; if (ec1->replacement_enc != ec2->replacement_enc && - strcmp(ec1->replacement_enc, ec2->replacement_enc)) - return Qfalse; + strcmp(ec1->replacement_enc, ec2->replacement_enc)) + return Qfalse; if (ec1->replacement_len != ec2->replacement_len) return Qfalse; if (ec1->replacement_str != ec2->replacement_str && - memcmp(ec1->replacement_str, ec2->replacement_str, ec2->replacement_len)) - return Qfalse; + memcmp(ec1->replacement_str, ec2->replacement_str, ec2->replacement_len)) + return Qfalse; if (ec1->num_trans != ec2->num_trans) return Qfalse; for (i = 0; i < ec1->num_trans; i++) { if (ec1->elems[i].tc->transcoder != ec2->elems[i].tc->transcoder) - return Qfalse; + return Qfalse; } return Qtrue; } @@ -3687,10 +3764,10 @@ econv_primitive_convert(int argc, VALUE *argv, VALUE self) output_bytesize = NUM2LONG(output_bytesize_v); if (!NIL_P(flags_v)) { - if (!NIL_P(opt)) { - rb_error_arity(argc + 1, 2, 5); - } - flags = NUM2INT(rb_to_int(flags_v)); + if (!NIL_P(opt)) { + rb_error_arity(argc + 1, 2, 5); + } + flags = NUM2INT(rb_to_int(flags_v)); } else if (!NIL_P(opt)) { VALUE v; @@ -3712,11 +3789,8 @@ econv_primitive_convert(int argc, VALUE *argv, VALUE self) rb_str_modify(output); if (NIL_P(output_bytesize_v)) { -#if USE_RVARGC output_bytesize = rb_str_capacity(output); -#else - output_bytesize = RSTRING_EMBED_LEN_MAX; -#endif + if (!NIL_P(input) && output_bytesize < RSTRING_LEN(input)) output_bytesize = RSTRING_LEN(input); } @@ -4042,7 +4116,7 @@ econv_insert_output(VALUE self, VALUE string) ret = rb_econv_insert_output(ec, (const unsigned char *)RSTRING_PTR(string), RSTRING_LEN(string), insert_enc); if (ret == -1) { - rb_raise(rb_eArgError, "too big string"); + rb_raise(rb_eArgError, "too big string"); } return Qnil; @@ -4411,6 +4485,7 @@ Init_transcode(void) sym_universal_newline = ID2SYM(rb_intern_const("universal_newline")); sym_crlf_newline = ID2SYM(rb_intern_const("crlf_newline")); sym_cr_newline = ID2SYM(rb_intern_const("cr_newline")); + sym_lf_newline = ID2SYM(rb_intern("lf_newline")); sym_partial_input = ID2SYM(rb_intern_const("partial_input")); #ifdef ENABLE_ECONV_NEWLINE_OPTION @@ -4507,6 +4582,12 @@ InitVM_transcode(void) */ rb_define_const(rb_cEncodingConverter, "UNIVERSAL_NEWLINE_DECORATOR", INT2FIX(ECONV_UNIVERSAL_NEWLINE_DECORATOR)); + /* Document-const: LF_NEWLINE_DECORATOR + * + * Decorator for converting CRLF and CR to LF when writing + */ + rb_define_const(rb_cEncodingConverter, "LF_NEWLINE_DECORATOR", INT2FIX(ECONV_LF_NEWLINE_DECORATOR)); + /* Document-const: CRLF_NEWLINE_DECORATOR * * Decorator for converting LF to CRLF |