/********************************************************************** transcode.c - $Author$ created at: Tue Oct 30 16:10:22 JST 2007 Copyright (C) 2007 Martin Duerst **********************************************************************/ #include "ruby/ruby.h" #include "ruby/encoding.h" #define PType (int) #include "transcode_data.h" #include VALUE rb_eConversionUndefined; VALUE rb_eInvalidByteSequence; VALUE rb_eNoConverter; VALUE rb_cEncodingConverter; static VALUE sym_invalid, sym_undef, sym_ignore, sym_replace; static VALUE sym_invalid_byte_sequence; static VALUE sym_undefined_conversion; static VALUE sym_destination_buffer_full; static VALUE sym_source_buffer_empty; static VALUE sym_finished; static VALUE sym_output_followed_by_input; static VALUE sym_incomplete_input; typedef struct { struct rb_transcoding *tc; unsigned char *out_buf_start; unsigned char *out_data_start; unsigned char *out_data_end; unsigned char *out_buf_end; rb_econv_result_t last_result; } rb_econv_elem_t; struct rb_econv_t { rb_econv_option_t opts; const char *source_encoding_name; const char *destination_encoding_name; unsigned char *in_buf_start; unsigned char *in_data_start; unsigned char *in_data_end; unsigned char *in_buf_end; rb_econv_elem_t *elems; int num_trans; int num_finished; int last_trans_index; /* last trans, not including universal newline */ struct rb_transcoding *last_tc; /* last error */ struct { rb_econv_result_t result; struct rb_transcoding *error_tc; const char *source_encoding; const char *destination_encoding; const unsigned char *error_bytes_start; size_t error_bytes_len; size_t readagain_len; int partial_input; } last_error; /* The following fields are only for Encoding::Converter. * rb_econv_open set them NULL. */ rb_encoding *source_encoding; rb_encoding *destination_encoding; }; /* * Dispatch data and logic */ typedef struct { const char *from; const char *to; const char *lib; /* maybe null. it means that don't load the library. */ const rb_transcoder *transcoder; } transcoder_entry_t; static st_table *transcoder_table; static transcoder_entry_t * make_transcoder_entry(const char *from, const char *to) { st_data_t val; st_table *table2; if (!st_lookup(transcoder_table, (st_data_t)from, &val)) { val = (st_data_t)st_init_strcasetable(); st_add_direct(transcoder_table, (st_data_t)from, val); } table2 = (st_table *)val; if (!st_lookup(table2, (st_data_t)to, &val)) { transcoder_entry_t *entry = ALLOC(transcoder_entry_t); entry->from = from; entry->to = to; entry->lib = NULL; entry->transcoder = NULL; val = (st_data_t)entry; st_add_direct(table2, (st_data_t)to, val); } return (transcoder_entry_t *)val; } static transcoder_entry_t * get_transcoder_entry(const char *from, const char *to) { st_data_t val; st_table *table2; if (!st_lookup(transcoder_table, (st_data_t)from, &val)) { return NULL; } table2 = (st_table *)val; if (!st_lookup(table2, (st_data_t)to, &val)) { return NULL; } return (transcoder_entry_t *)val; } void rb_register_transcoder(const rb_transcoder *tr) { const char *const from_e = tr->from_encoding; const char *const to_e = tr->to_encoding; transcoder_entry_t *entry; entry = make_transcoder_entry(from_e, to_e); if (entry->transcoder) { rb_raise(rb_eArgError, "transcoder from %s to %s has been already registered", from_e, to_e); } entry->transcoder = tr; } static void declare_transcoder(const char *from, const char *to, const char *lib) { transcoder_entry_t *entry; entry = make_transcoder_entry(from, to); entry->lib = lib; } #define MAX_TRANSCODER_LIBNAME_LEN 64 static const char transcoder_lib_prefix[] = "enc/trans/"; void rb_declare_transcoder(const char *enc1, const char *enc2, const char *lib) { if (!lib || strlen(lib) > MAX_TRANSCODER_LIBNAME_LEN) { rb_raise(rb_eArgError, "invalid library name - %s", lib ? lib : "(null)"); } declare_transcoder(enc1, enc2, lib); } #define encoding_equal(enc1, enc2) (STRCASECMP(enc1, enc2) == 0) typedef struct search_path_queue_tag { struct search_path_queue_tag *next; const char *enc; } search_path_queue_t; typedef struct { st_table *visited; search_path_queue_t *queue; search_path_queue_t **queue_last_ptr; const char *base_enc; } search_path_bfs_t; static int transcode_search_path_i(st_data_t key, st_data_t val, st_data_t arg) { const char *to = (const char *)key; search_path_bfs_t *bfs = (search_path_bfs_t *)arg; search_path_queue_t *q; if (st_lookup(bfs->visited, (st_data_t)to, &val)) { return ST_CONTINUE; } q = ALLOC(search_path_queue_t); q->enc = to; q->next = NULL; *bfs->queue_last_ptr = q; bfs->queue_last_ptr = &q->next; st_add_direct(bfs->visited, (st_data_t)to, (st_data_t)bfs->base_enc); return ST_CONTINUE; } static int transcode_search_path(const char *from, const char *to, void (*callback)(const char *from, const char *to, int depth, void *arg), void *arg) { search_path_bfs_t bfs; search_path_queue_t *q; st_data_t val; st_table *table2; int found; int pathlen; if (encoding_equal(from, to)) return -1; q = ALLOC(search_path_queue_t); q->enc = from; q->next = NULL; bfs.queue_last_ptr = &q->next; bfs.queue = q; bfs.visited = st_init_strcasetable(); st_add_direct(bfs.visited, (st_data_t)from, (st_data_t)NULL); while (bfs.queue) { q = bfs.queue; bfs.queue = q->next; if (!bfs.queue) bfs.queue_last_ptr = &bfs.queue; if (!st_lookup(transcoder_table, (st_data_t)q->enc, &val)) { xfree(q); continue; } table2 = (st_table *)val; if (st_lookup(table2, (st_data_t)to, &val)) { st_add_direct(bfs.visited, (st_data_t)to, (st_data_t)q->enc); xfree(q); found = 1; goto cleanup; } bfs.base_enc = q->enc; st_foreach(table2, transcode_search_path_i, (st_data_t)&bfs); bfs.base_enc = NULL; xfree(q); } found = 0; cleanup: while (bfs.queue) { q = bfs.queue; bfs.queue = q->next; xfree(q); } if (found) { const char *enc = to; int depth; pathlen = 0; while (1) { st_lookup(bfs.visited, (st_data_t)enc, &val); if (!val) break; pathlen++; enc = (const char *)val; } depth = pathlen; enc = to; while (1) { st_lookup(bfs.visited, (st_data_t)enc, &val); if (!val) break; callback((const char *)val, enc, --depth, arg); enc = (const char *)val; } } st_free_table(bfs.visited); if (found) return pathlen; else return -1; } static const rb_transcoder * load_transcoder_entry(transcoder_entry_t *entry) { if (entry->transcoder) return entry->transcoder; if (entry->lib) { const char *lib = entry->lib; int len = strlen(lib); char path[sizeof(transcoder_lib_prefix) + MAX_TRANSCODER_LIBNAME_LEN]; entry->lib = NULL; if (len > MAX_TRANSCODER_LIBNAME_LEN) return NULL; memcpy(path, transcoder_lib_prefix, sizeof(transcoder_lib_prefix) - 1); memcpy(path + sizeof(transcoder_lib_prefix) - 1, lib, len + 1); if (!rb_require(path)) return NULL; } if (entry->transcoder) return entry->transcoder; return NULL; } static const char* get_replacement_character(rb_encoding *enc, int *len_ret, const char **repl_enc_ptr) { static rb_encoding *utf16be_encoding, *utf16le_encoding; static rb_encoding *utf32be_encoding, *utf32le_encoding; if (!utf16be_encoding) { utf16be_encoding = rb_enc_find("UTF-16BE"); utf16le_encoding = rb_enc_find("UTF-16LE"); utf32be_encoding = rb_enc_find("UTF-32BE"); utf32le_encoding = rb_enc_find("UTF-32LE"); } if (rb_utf8_encoding() == enc) { *len_ret = 3; *repl_enc_ptr = "UTF-8"; return "\xEF\xBF\xBD"; } else if (utf16be_encoding == enc) { *len_ret = 2; *repl_enc_ptr = "UTF-16BE"; return "\xFF\xFD"; } else if (utf16le_encoding == enc) { *len_ret = 2; *repl_enc_ptr = "UTF-16LE"; return "\xFD\xFF"; } else if (utf32be_encoding == enc) { *len_ret = 4; *repl_enc_ptr = "UTF-32BE"; return "\x00\x00\xFF\xFD"; } else if (utf32le_encoding == enc) { *len_ret = 4; *repl_enc_ptr = "UTF-32LE"; return "\xFD\xFF\x00\x00"; } else { *len_ret = 1; *repl_enc_ptr = "US-ASCII"; return "?"; } } /* * Transcoding engine logic */ static const unsigned char * transcode_char_start(rb_transcoding *tc, const unsigned char *in_start, const unsigned char *inchar_start, const unsigned char *in_p, size_t *char_len_ptr) { const unsigned char *ptr; if (inchar_start - in_start < tc->recognized_len) { MEMCPY(TRANSCODING_READBUF(tc) + tc->recognized_len, inchar_start, unsigned char, in_p - inchar_start); ptr = TRANSCODING_READBUF(tc); } else { ptr = inchar_start - tc->recognized_len; } *char_len_ptr = tc->recognized_len + (in_p - inchar_start); return ptr; } static rb_econv_result_t transcode_restartable0(const unsigned char **in_pos, unsigned char **out_pos, const unsigned char *in_stop, unsigned char *out_stop, rb_transcoding *tc, const int opt) { const rb_transcoder *tr = tc->transcoder; int unitlen = tr->input_unit_length; int readagain_len = 0; const unsigned char *inchar_start; const unsigned char *in_p; unsigned char *out_p; unsigned char empty_buf; unsigned char *empty_ptr = &empty_buf; if (!in_pos) { in_pos = (const unsigned char **)&empty_ptr; in_stop = empty_ptr; } if (!out_pos) { out_pos = &empty_ptr; out_stop = empty_ptr; } in_p = inchar_start = *in_pos; out_p = *out_pos; #define SUSPEND(ret, num) \ do { \ tc->resume_position = (num); \ if (0 < in_p - inchar_start) \ MEMMOVE(TRANSCODING_READBUF(tc)+tc->recognized_len, \ inchar_start, unsigned char, in_p - inchar_start); \ *in_pos = in_p; \ *out_pos = out_p; \ tc->recognized_len += in_p - inchar_start; \ if (readagain_len) { \ tc->recognized_len -= readagain_len; \ tc->readagain_len = readagain_len; \ } \ return ret; \ resume_label ## num:; \ } while (0) #define SUSPEND_OBUF(num) \ do { \ while (out_stop - out_p < 1) { SUSPEND(econv_destination_buffer_full, num); } \ } while (0) #define SUSPEND_OUTPUT_FOLLOWED_BY_INPUT(num) \ if ((opt & ECONV_OUTPUT_FOLLOWED_BY_INPUT) && *out_pos != out_p) { \ SUSPEND(econv_output_followed_by_input, num); \ } #define next_table (tc->next_table) #define next_info (tc->next_info) #define next_byte (tc->next_byte) #define writebuf_len (tc->writebuf_len) #define writebuf_off (tc->writebuf_off) switch (tc->resume_position) { case 0: break; case 1: goto resume_label1; case 2: goto resume_label2; case 3: goto resume_label3; case 4: goto resume_label4; case 5: goto resume_label5; case 6: goto resume_label6; case 7: goto resume_label7; case 8: goto resume_label8; case 9: goto resume_label9; case 10: goto resume_label10; case 11: goto resume_label11; case 12: goto resume_label12; case 13: goto resume_label13; case 14: goto resume_label14; case 15: goto resume_label15; case 16: goto resume_label16; case 17: goto resume_label17; case 18: goto resume_label18; case 19: goto resume_label19; case 20: goto resume_label20; case 21: goto resume_label21; case 22: goto resume_label22; case 23: goto resume_label23; case 24: goto resume_label24; case 25: goto resume_label25; case 26: goto resume_label26; case 27: goto resume_label27; } while (1) { inchar_start = in_p; tc->recognized_len = 0; next_table = tr->conv_tree_start; SUSPEND_OUTPUT_FOLLOWED_BY_INPUT(24); if (in_stop <= in_p) { if (!(opt & ECONV_PARTIAL_INPUT)) break; SUSPEND(econv_source_buffer_empty, 7); continue; } next_byte = (unsigned char)*in_p++; follow_byte: if (next_byte < next_table->base[0] || next_table->base[1] < next_byte) next_info = INVALID; else { unsigned int next_offset = next_table->base[2+next_byte-next_table->base[0]]; next_info = (VALUE)next_table->info[next_offset]; } follow_info: switch (next_info & 0x1F) { case NOMAP: /* xxx: copy last byte only? */ SUSPEND_OBUF(3); *out_p++ = next_byte; continue; case 0x00: case 0x04: case 0x08: case 0x0C: case 0x10: case 0x14: case 0x18: case 0x1C: SUSPEND_OUTPUT_FOLLOWED_BY_INPUT(25); while (in_p >= in_stop) { if (!(opt & ECONV_PARTIAL_INPUT)) goto incomplete; SUSPEND(econv_source_buffer_empty, 5); } next_byte = (unsigned char)*in_p++; next_table = (const BYTE_LOOKUP *)next_info; goto follow_byte; case ZERObt: /* drop input */ continue; case ONEbt: SUSPEND_OBUF(9); *out_p++ = getBT1(next_info); continue; case TWObt: SUSPEND_OBUF(10); *out_p++ = getBT1(next_info); SUSPEND_OBUF(21); *out_p++ = getBT2(next_info); continue; case THREEbt: SUSPEND_OBUF(11); *out_p++ = getBT1(next_info); SUSPEND_OBUF(15); *out_p++ = getBT2(next_info); SUSPEND_OBUF(16); *out_p++ = getBT3(next_info); continue; case FOURbt: SUSPEND_OBUF(12); *out_p++ = getBT0(next_info); SUSPEND_OBUF(17); *out_p++ = getBT1(next_info); SUSPEND_OBUF(18); *out_p++ = getBT2(next_info); SUSPEND_OBUF(19); *out_p++ = getBT3(next_info); continue; case FUNii: next_info = (VALUE)(*tr->func_ii)(tc, next_info); goto follow_info; case FUNsi: { const unsigned char *char_start; size_t char_len; char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len); next_info = (VALUE)(*tr->func_si)(tc, char_start, (size_t)char_len); goto follow_info; } case FUNio: SUSPEND_OBUF(13); if (tr->max_output <= out_stop - out_p) out_p += (VALUE)(*tr->func_io)(tc, next_info, out_p); else { writebuf_len = (VALUE)(*tr->func_io)(tc, next_info, TRANSCODING_WRITEBUF(tc)); writebuf_off = 0; while (writebuf_off < writebuf_len) { SUSPEND_OBUF(20); *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++]; } } break; case FUNso: { const unsigned char *char_start; size_t char_len; SUSPEND_OBUF(14); if (tr->max_output <= out_stop - out_p) { char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len); out_p += (VALUE)(*tr->func_so)(tc, char_start, (size_t)char_len, out_p); } else { char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len); writebuf_len = (VALUE)(*tr->func_so)(tc, char_start, (size_t)char_len, TRANSCODING_WRITEBUF(tc)); writebuf_off = 0; while (writebuf_off < writebuf_len) { SUSPEND_OBUF(22); *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++]; } } break; } case INVALID: if (tc->recognized_len + (in_p - inchar_start) <= unitlen) { if (tc->recognized_len + (in_p - inchar_start) < unitlen) SUSPEND_OUTPUT_FOLLOWED_BY_INPUT(26); while ((opt & ECONV_PARTIAL_INPUT) && tc->recognized_len + (in_stop - inchar_start) < unitlen) { in_p = in_stop; SUSPEND(econv_source_buffer_empty, 8); } if (tc->recognized_len + (in_stop - inchar_start) <= unitlen) { in_p = in_stop; } else { in_p = inchar_start + (unitlen - tc->recognized_len); } } else { int invalid_len; /* including the last byte which causes invalid */ int discard_len; invalid_len = tc->recognized_len + (in_p - inchar_start); discard_len = ((invalid_len - 1) / unitlen) * unitlen; readagain_len = invalid_len - discard_len; } goto invalid; case UNDEF: goto undef; } continue; invalid: SUSPEND(econv_invalid_byte_sequence, 1); continue; incomplete: SUSPEND(econv_incomplete_input, 27); continue; undef: SUSPEND(econv_undefined_conversion, 2); continue; } /* cleanup */ if (tr->finish_func) { SUSPEND_OBUF(4); if (tr->max_output <= out_stop - out_p) { out_p += tr->finish_func(tc, out_p); } else { writebuf_len = tr->finish_func(tc, TRANSCODING_WRITEBUF(tc)); writebuf_off = 0; while (writebuf_off < writebuf_len) { SUSPEND_OBUF(23); *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++]; } } } while (1) SUSPEND(econv_finished, 6); #undef SUSPEND #undef next_table #undef next_info #undef next_byte #undef writebuf_len #undef writebuf_off } static rb_econv_result_t transcode_restartable(const unsigned char **in_pos, unsigned char **out_pos, const unsigned char *in_stop, unsigned char *out_stop, rb_transcoding *tc, const int opt) { if (tc->readagain_len) { unsigned char *readagain_buf = ALLOCA_N(unsigned char, tc->readagain_len); const unsigned char *readagain_pos = readagain_buf; const unsigned char *readagain_stop = readagain_buf + tc->readagain_len; rb_econv_result_t res; MEMCPY(readagain_buf, TRANSCODING_READBUF(tc) + tc->recognized_len, unsigned char, tc->readagain_len); tc->readagain_len = 0; res = transcode_restartable0(&readagain_pos, out_pos, readagain_stop, out_stop, tc, opt|ECONV_PARTIAL_INPUT); if (res != econv_source_buffer_empty) { MEMCPY(TRANSCODING_READBUF(tc) + tc->recognized_len + tc->readagain_len, readagain_pos, unsigned char, readagain_stop - readagain_pos); tc->readagain_len += readagain_stop - readagain_pos; return res; } } return transcode_restartable0(in_pos, out_pos, in_stop, out_stop, tc, opt); } static rb_transcoding * rb_transcoding_open_by_transcoder(const rb_transcoder *tr, int flags) { rb_transcoding *tc; tc = ALLOC(rb_transcoding); tc->transcoder = tr; tc->flags = flags; memset(tc->stateful, 0, sizeof(tc->stateful)); tc->resume_position = 0; tc->recognized_len = 0; tc->readagain_len = 0; tc->writebuf_len = 0; tc->writebuf_off = 0; if (sizeof(tc->readbuf.ary) < tr->max_input) { tc->readbuf.ptr = xmalloc(tr->max_input); } if (sizeof(tc->writebuf.ary) < tr->max_output) { tc->writebuf.ptr = xmalloc(tr->max_output); } return tc; } static rb_econv_result_t rb_transcoding_convert(rb_transcoding *tc, const unsigned char **input_ptr, const unsigned char *input_stop, unsigned char **output_ptr, unsigned char *output_stop, int flags) { return transcode_restartable( input_ptr, output_ptr, input_stop, output_stop, tc, flags); } static void rb_transcoding_close(rb_transcoding *tc) { const rb_transcoder *tr = tc->transcoder; if (sizeof(tc->readbuf.ary) < tr->max_input) xfree(tc->readbuf.ptr); if (sizeof(tc->writebuf.ary) < tr->max_output) xfree(tc->writebuf.ptr); xfree(tc); } static rb_econv_t * rb_econv_open_by_transcoder_entries(int n, transcoder_entry_t **entries) { rb_econv_t *ec; int i; for (i = 0; i < n; i++) { const rb_transcoder *tr; tr = load_transcoder_entry(entries[i]); if (!tr) return NULL; } ec = ALLOC(rb_econv_t); ec->opts.flags = 0; ec->source_encoding_name = NULL; ec->destination_encoding_name = NULL; ec->in_buf_start = NULL; ec->in_data_start = NULL; ec->in_data_end = NULL; ec->in_buf_end = NULL; ec->num_trans = n; ec->elems = ALLOC_N(rb_econv_elem_t, ec->num_trans); ec->num_finished = 0; ec->last_tc = NULL; ec->last_trans_index = -1; ec->last_error.result = econv_source_buffer_empty; ec->last_error.error_tc = NULL; ec->last_error.source_encoding = NULL; ec->last_error.destination_encoding = NULL; ec->last_error.error_bytes_start = NULL; ec->last_error.error_bytes_len = 0; ec->last_error.readagain_len = 0; ec->last_error.partial_input = 0; ec->source_encoding = NULL; ec->destination_encoding = NULL; for (i = 0; i < ec->num_trans; i++) { const rb_transcoder *tr = load_transcoder_entry(entries[i]); ec->elems[i].tc = rb_transcoding_open_by_transcoder(tr, 0); ec->elems[i].out_buf_start = NULL; ec->elems[i].out_data_start = NULL; ec->elems[i].out_data_end = NULL; ec->elems[i].out_buf_end = NULL; ec->elems[i].last_result = econv_source_buffer_empty; } if (ec->num_trans) ec->last_tc = ec->elems[ec->num_trans-1].tc; ec->last_trans_index = ec->num_trans-1; for (i = 0; i < ec->num_trans-1; i++) { int bufsize = 4096; unsigned char *p; p = xmalloc(bufsize); ec->elems[i].out_buf_start = p; ec->elems[i].out_buf_end = p + bufsize; ec->elems[i].out_data_start = p; ec->elems[i].out_data_end = p; } return ec; } static void trans_open_i(const char *from, const char *to, int depth, void *arg) { transcoder_entry_t ***entries_ptr = arg; transcoder_entry_t **entries; if (!*entries_ptr) { entries = ALLOC_N(transcoder_entry_t *, depth+1+2); *entries_ptr = entries; } else { entries = *entries_ptr; } entries[depth] = get_transcoder_entry(from, to); } rb_econv_t * rb_econv_open(const char *from, const char *to, rb_econv_option_t *opts) { transcoder_entry_t **entries = NULL; int num_trans; int num_additional; static rb_econv_t *ec; int flags = opts ? opts->flags : 0; int universal_newline_decoder_added = 0; rb_encoding *senc, *denc; int sidx, didx; senc = NULL; if (*from) { sidx = rb_enc_find_index(from); if (0 <= sidx) { senc = rb_enc_from_index(sidx); } } denc = NULL; if (*to) { didx = rb_enc_find_index(to); if (0 <= didx) { denc = rb_enc_from_index(didx); } } if (*from == '\0' && *to == '\0') { num_trans = 0; entries = ALLOC_N(transcoder_entry_t *, 1+2); } else { num_trans = transcode_search_path(from, to, trans_open_i, (void *)&entries); } if (num_trans < 0 || !entries) { xfree(entries); return NULL; } num_additional = 0; if ((!*from || (senc && rb_enc_asciicompat(senc))) && (flags & (ECONV_CRLF_NEWLINE_ENCODER|ECONV_CR_NEWLINE_ENCODER))) { const char *name = (flags & ECONV_CRLF_NEWLINE_ENCODER) ? "crlf_newline" : "cr_newline"; transcoder_entry_t *e = get_transcoder_entry("", name); if (flags & ECONV_CRLF_NEWLINE_ENCODER) flags &= ~ECONV_CR_NEWLINE_ENCODER; else flags &= ~ECONV_CRLF_NEWLINE_ENCODER; if (!e) { xfree(entries); return NULL; } MEMMOVE(entries+1, entries, transcoder_entry_t *, num_trans); entries[0] = e; num_trans++; num_additional++; } else { flags &= ~(ECONV_CRLF_NEWLINE_ENCODER|ECONV_CR_NEWLINE_ENCODER); } if ((!*to || (denc && rb_enc_asciicompat(denc))) && (flags & ECONV_UNIVERSAL_NEWLINE_DECODER)) { transcoder_entry_t *e = get_transcoder_entry("universal_newline", ""); if (!e) { xfree(entries); return NULL; } entries[num_trans++] = e; num_additional++; universal_newline_decoder_added = 1; } else { flags &= ~ECONV_UNIVERSAL_NEWLINE_DECODER; } ec = rb_econv_open_by_transcoder_entries(num_trans, entries); xfree(entries); if (!ec) return NULL; if (!opts) ec->opts.flags = 0; else ec->opts = *opts; ec->opts.flags = flags; ec->source_encoding_name = from; ec->destination_encoding_name = to; if (num_trans == num_additional) { ec->last_tc = NULL; ec->last_trans_index = -1; } else if (universal_newline_decoder_added) { ec->last_tc = ec->elems[ec->num_trans-2].tc; ec->last_trans_index = ec->num_trans-2; } return ec; } static int trans_sweep(rb_econv_t *ec, const unsigned char **input_ptr, const unsigned char *input_stop, unsigned char **output_ptr, unsigned char *output_stop, int flags, int start) { int try; int i, f; const unsigned char **ipp, *is, *iold; unsigned char **opp, *os, *oold; rb_econv_result_t res; try = 1; while (try) { try = 0; for (i = start; i < ec->num_trans; i++) { rb_econv_elem_t *te = &ec->elems[i]; if (i == 0) { ipp = input_ptr; is = input_stop; } else { rb_econv_elem_t *prev_te = &ec->elems[i-1]; ipp = (const unsigned char **)&prev_te->out_data_start; is = prev_te->out_data_end; } if (i == ec->num_trans-1) { opp = output_ptr; os = output_stop; } else { if (te->out_buf_start != te->out_data_start) { int len = te->out_data_end - te->out_data_start; int off = te->out_data_start - te->out_buf_start; MEMMOVE(te->out_buf_start, te->out_data_start, unsigned char, len); te->out_data_start = te->out_buf_start; te->out_data_end -= off; } opp = &te->out_data_end; os = te->out_buf_end; } f = flags; if (ec->num_finished != i) f |= ECONV_PARTIAL_INPUT; if (i == 0 && (flags & ECONV_OUTPUT_FOLLOWED_BY_INPUT)) { start = 1; flags &= ~ECONV_OUTPUT_FOLLOWED_BY_INPUT; } if (i != 0) f &= ~ECONV_OUTPUT_FOLLOWED_BY_INPUT; iold = *ipp; oold = *opp; te->last_result = res = rb_transcoding_convert(te->tc, ipp, is, opp, os, f); if (iold != *ipp || oold != *opp) try = 1; switch (res) { case econv_invalid_byte_sequence: case econv_incomplete_input: case econv_undefined_conversion: case econv_output_followed_by_input: return i; case econv_destination_buffer_full: case econv_source_buffer_empty: break; case econv_finished: ec->num_finished = i+1; break; } } } return -1; } static rb_econv_result_t rb_trans_conv(rb_econv_t *ec, const unsigned char **input_ptr, const unsigned char *input_stop, unsigned char **output_ptr, unsigned char *output_stop, int flags, int *result_position_ptr) { int i; int needreport_index; int sweep_start; unsigned char empty_buf; unsigned char *empty_ptr = &empty_buf; if (!input_ptr) { input_ptr = (const unsigned char **)&empty_ptr; input_stop = empty_ptr; } if (!output_ptr) { output_ptr = &empty_ptr; output_stop = empty_ptr; } if (ec->elems[0].last_result == econv_output_followed_by_input) ec->elems[0].last_result = econv_source_buffer_empty; needreport_index = -1; for (i = ec->num_trans-1; 0 <= i; i--) { switch (ec->elems[i].last_result) { case econv_invalid_byte_sequence: case econv_incomplete_input: case econv_undefined_conversion: case econv_output_followed_by_input: case econv_finished: sweep_start = i+1; needreport_index = i; goto found_needreport; case econv_destination_buffer_full: case econv_source_buffer_empty: break; default: rb_bug("unexpected transcode last result"); } } /* /^[sd]+$/ is confirmed. but actually /^s*d*$/. */ if (ec->elems[ec->num_trans-1].last_result == econv_destination_buffer_full && (flags & ECONV_OUTPUT_FOLLOWED_BY_INPUT)) { rb_econv_result_t res; res = rb_trans_conv(ec, NULL, NULL, output_ptr, output_stop, (flags & ~ECONV_OUTPUT_FOLLOWED_BY_INPUT)|ECONV_PARTIAL_INPUT, result_position_ptr); if (res == econv_source_buffer_empty) return econv_output_followed_by_input; return res; } sweep_start = 0; found_needreport: do { needreport_index = trans_sweep(ec, input_ptr, input_stop, output_ptr, output_stop, flags, sweep_start); sweep_start = needreport_index + 1; } while (needreport_index != -1 && needreport_index != ec->num_trans-1); for (i = ec->num_trans-1; 0 <= i; i--) { if (ec->elems[i].last_result != econv_source_buffer_empty) { rb_econv_result_t res = ec->elems[i].last_result; if (res == econv_invalid_byte_sequence || res == econv_incomplete_input || res == econv_undefined_conversion || res == econv_output_followed_by_input) { ec->elems[i].last_result = econv_source_buffer_empty; } if (result_position_ptr) *result_position_ptr = i; return res; } } if (result_position_ptr) *result_position_ptr = -1; return econv_source_buffer_empty; } static rb_econv_result_t rb_econv_convert0(rb_econv_t *ec, const unsigned char **input_ptr, const unsigned char *input_stop, unsigned char **output_ptr, unsigned char *output_stop, int flags) { rb_econv_result_t res; int result_position; int has_output = 0; memset(&ec->last_error, 0, sizeof(ec->last_error)); if (ec->num_trans == 0) { size_t len; if (ec->in_buf_start && ec->in_data_start != ec->in_data_end) { if (output_stop - *output_ptr < ec->in_data_end - ec->in_data_start) { len = output_stop - *output_ptr; memcpy(*output_ptr, ec->in_data_start, len); *output_ptr = output_stop; ec->in_data_start += len; res = econv_destination_buffer_full; goto gotresult; } len = ec->in_data_end - ec->in_data_start; memcpy(*output_ptr, ec->in_data_start, len); *output_ptr += len; ec->in_data_start = ec->in_data_end = ec->in_buf_start; if (flags & ECONV_OUTPUT_FOLLOWED_BY_INPUT) { res = econv_output_followed_by_input; goto gotresult; } } if (output_stop - *output_ptr < input_stop - *input_ptr) { len = output_stop - *output_ptr; } else { len = input_stop - *input_ptr; } if (0 < len && (flags & ECONV_OUTPUT_FOLLOWED_BY_INPUT)) { *(*output_ptr)++ = *(*input_ptr)++; res = econv_output_followed_by_input; goto gotresult; } memcpy(*output_ptr, *input_ptr, len); *output_ptr += len; *input_ptr += len; if (*input_ptr != input_stop) res = econv_destination_buffer_full; else if (flags & ECONV_PARTIAL_INPUT) res = econv_source_buffer_empty; else res = econv_finished; goto gotresult; } if (ec->elems[ec->num_trans-1].out_data_start) { unsigned char *data_start = ec->elems[ec->num_trans-1].out_data_start; unsigned char *data_end = ec->elems[ec->num_trans-1].out_data_end; if (data_start != data_end) { size_t len; if (output_stop - *output_ptr < data_end - data_start) { len = output_stop - *output_ptr; memcpy(*output_ptr, data_start, len); *output_ptr = output_stop; ec->elems[ec->num_trans-1].out_data_start += len; res = econv_destination_buffer_full; goto gotresult; } len = data_end - data_start; memcpy(*output_ptr, data_start, len); *output_ptr += len; ec->elems[ec->num_trans-1].out_data_start = ec->elems[ec->num_trans-1].out_data_end = ec->elems[ec->num_trans-1].out_buf_start; has_output = 1; } } if (ec->in_buf_start && ec->in_data_start != ec->in_data_end) { res = rb_trans_conv(ec, (const unsigned char **)&ec->in_data_start, ec->in_data_end, output_ptr, output_stop, (flags&~ECONV_OUTPUT_FOLLOWED_BY_INPUT)|ECONV_PARTIAL_INPUT, &result_position); if (res != econv_source_buffer_empty) goto gotresult; } if (has_output && (flags & ECONV_OUTPUT_FOLLOWED_BY_INPUT) && *input_ptr != input_stop) { input_stop = *input_ptr; res = rb_trans_conv(ec, input_ptr, input_stop, output_ptr, output_stop, flags, &result_position); if (res == econv_source_buffer_empty) res = econv_output_followed_by_input; } else if ((flags & ECONV_OUTPUT_FOLLOWED_BY_INPUT) || ec->num_trans == 1) { res = rb_trans_conv(ec, input_ptr, input_stop, output_ptr, output_stop, flags, &result_position); } else { flags |= ECONV_OUTPUT_FOLLOWED_BY_INPUT; do { res = rb_trans_conv(ec, input_ptr, input_stop, output_ptr, output_stop, flags, &result_position); } while (res == econv_output_followed_by_input); } gotresult: ec->last_error.result = res; ec->last_error.partial_input = flags & ECONV_PARTIAL_INPUT; if (res == econv_invalid_byte_sequence || res == econv_incomplete_input || res == econv_undefined_conversion) { rb_transcoding *error_tc = ec->elems[result_position].tc; ec->last_error.error_tc = error_tc; ec->last_error.source_encoding = error_tc->transcoder->from_encoding; ec->last_error.destination_encoding = error_tc->transcoder->to_encoding; ec->last_error.error_bytes_start = TRANSCODING_READBUF(error_tc); ec->last_error.error_bytes_len = error_tc->recognized_len; ec->last_error.readagain_len = error_tc->readagain_len; } return res; } static int output_replacement_character(rb_econv_t *ec); rb_econv_result_t rb_econv_convert(rb_econv_t *ec, const unsigned char **input_ptr, const unsigned char *input_stop, unsigned char **output_ptr, unsigned char *output_stop, int flags) { rb_econv_result_t ret; unsigned char empty_buf; unsigned char *empty_ptr = &empty_buf; if (!input_ptr) { input_ptr = (const unsigned char **)&empty_ptr; input_stop = empty_ptr; } if (!output_ptr) { output_ptr = &empty_ptr; output_stop = empty_ptr; } resume: ret = rb_econv_convert0(ec, input_ptr, input_stop, output_ptr, output_stop, flags); if (ret == econv_invalid_byte_sequence || ret == econv_incomplete_input) { /* deal with invalid byte sequence */ /* todo: add more alternative behaviors */ if (ec->opts.flags&ECONV_INVALID_IGNORE) { goto resume; } else if (ec->opts.flags&ECONV_INVALID_REPLACE) { if (output_replacement_character(ec) == 0) goto resume; } } if (ret == econv_undefined_conversion) { /* valid character in source encoding * but no related character(s) in destination encoding */ /* todo: add more alternative behaviors */ if (ec->opts.flags&ECONV_UNDEF_IGNORE) { goto resume; } else if (ec->opts.flags&ECONV_UNDEF_REPLACE) { if (output_replacement_character(ec) == 0) goto resume; } } return ret; } const char * rb_econv_encoding_to_insert_output(rb_econv_t *ec) { rb_transcoding *tc = ec->last_tc; const rb_transcoder *tr; if (tc == NULL) return ""; tr = tc->transcoder; if (tr->stateful_type == stateful_encoder) return tr->from_encoding; return tr->to_encoding; } static unsigned char * allocate_converted_string(const char *str_encoding, const char *insert_encoding, const unsigned char *str, size_t len, size_t *dst_len_ptr) { unsigned char *dst_str; size_t dst_len; size_t dst_bufsize = len; rb_econv_t *ec; rb_econv_result_t res; const unsigned char *sp; unsigned char *dp; if (dst_bufsize == 0) dst_bufsize += 1; ec = rb_econv_open(str_encoding, insert_encoding, NULL); if (ec == NULL) return NULL; dst_str = xmalloc(dst_bufsize); dst_len = 0; sp = str; dp = dst_str+dst_len; res = rb_econv_convert(ec, &sp, str+len, &dp, dst_str+dst_bufsize, 0); dst_len = dp - dst_str; while (res == econv_destination_buffer_full) { if (dst_bufsize * 2 < dst_bufsize) { xfree(dst_str); rb_econv_close(ec); return NULL; } dst_bufsize *= 2; dst_str = xrealloc(dst_str, dst_bufsize); dp = dst_str+dst_len; res = rb_econv_convert(ec, &sp, str+len, &dp, dst_str+dst_bufsize, 0); dst_len = dp - dst_str; } if (res != econv_finished) { xfree(dst_str); rb_econv_close(ec); return NULL; } rb_econv_close(ec); *dst_len_ptr = dst_len; return dst_str; } /* result: 0:success -1:failure */ int rb_econv_insert_output(rb_econv_t *ec, const unsigned char *str, size_t len, const char *str_encoding) { const char *insert_encoding = rb_econv_encoding_to_insert_output(ec); const unsigned char *insert_str; size_t insert_len; rb_transcoding *tc; unsigned char **buf_start_p; unsigned char **data_start_p; unsigned char **data_end_p; unsigned char **buf_end_p; size_t need; if (len == 0) return 0; if (encoding_equal(insert_encoding, str_encoding)) { insert_str = str; insert_len = len; } else { insert_str = allocate_converted_string(str_encoding, insert_encoding, str, len, &insert_len); if (insert_str == NULL) return -1; } need = insert_len; tc = ec->last_tc; if (!tc) { buf_start_p = &ec->in_buf_start; data_start_p = &ec->in_data_start; data_end_p = &ec->in_data_end; buf_end_p = &ec->in_buf_end; } else if (tc->transcoder->stateful_type == stateful_encoder) { need += tc->readagain_len; if (need < insert_len) goto fail; if (ec->last_trans_index == 0) { buf_start_p = &ec->in_buf_start; data_start_p = &ec->in_data_start; data_end_p = &ec->in_data_end; buf_end_p = &ec->in_buf_end; } else { rb_econv_elem_t *ee = &ec->elems[ec->last_trans_index-1]; buf_start_p = &ee->out_buf_start; data_start_p = &ee->out_data_start; data_end_p = &ee->out_data_end; buf_end_p = &ee->out_buf_end; } } else { rb_econv_elem_t *ee = &ec->elems[ec->last_trans_index]; buf_start_p = &ee->out_buf_start; data_start_p = &ee->out_data_start; data_end_p = &ee->out_data_end; buf_end_p = &ee->out_buf_end; } if (*buf_start_p == NULL) { unsigned char *buf = xmalloc(need); *buf_start_p = buf; *data_start_p = buf; *data_end_p = buf; *buf_end_p = buf+need; } else if (*buf_end_p - *data_end_p < need) { MEMMOVE(*buf_start_p, *data_start_p, unsigned char, *data_end_p - *data_start_p); *data_end_p = *buf_start_p + (*data_end_p - *data_start_p); *data_start_p = *buf_start_p; if (*buf_end_p - *data_end_p < need) { unsigned char *buf; size_t s = (*data_end_p - *buf_start_p) + need; if (s < need) goto fail; buf = xrealloc(*buf_start_p, s); *data_start_p = buf; *data_end_p = buf + (*data_end_p - *buf_start_p); *buf_start_p = buf; *buf_end_p = buf + s; } } if (tc && tc->transcoder->stateful_type == stateful_encoder) { memcpy(*data_end_p, TRANSCODING_READBUF(tc)+tc->recognized_len, tc->readagain_len); *data_end_p += tc->readagain_len; tc->readagain_len = 0; } memcpy(*data_end_p, insert_str, insert_len); *data_end_p += insert_len; if (insert_str != str) xfree((void*)insert_str); return 0; fail: if (insert_str != str) xfree((void*)insert_str); return -1; } void rb_econv_close(rb_econv_t *ec) { int i; for (i = 0; i < ec->num_trans; i++) { rb_transcoding_close(ec->elems[i].tc); if (ec->elems[i].out_buf_start) xfree(ec->elems[i].out_buf_start); } xfree(ec->in_buf_start); xfree(ec->elems); xfree(ec); } int rb_econv_putbackable(rb_econv_t *ec) { if (ec->num_trans == 0) return 0; return ec->elems[0].tc->readagain_len; } void rb_econv_putback(rb_econv_t *ec, unsigned char *p, int n) { rb_transcoding *tc; if (ec->num_trans == 0 || n == 0) return; tc = ec->elems[0].tc; memcpy(p, TRANSCODING_READBUF(tc) + tc->recognized_len + tc->readagain_len - n, n); tc->readagain_len -= n; } struct stateless_encoding_t { const char *stateless_enc; const char *stateful_enc; }; static int stateless_encoding_i(st_data_t key, st_data_t val, st_data_t arg) { struct stateless_encoding_t *data = (struct stateless_encoding_t *)arg; st_table *table2 = (st_table *)val; st_data_t v; if (st_lookup(table2, (st_data_t)data->stateful_enc, &v)) { transcoder_entry_t *entry = (transcoder_entry_t *)v; const rb_transcoder *tr = load_transcoder_entry(entry); if (tr && tr->stateful_type == stateful_encoder) { data->stateless_enc = tr->from_encoding; return ST_STOP; } } return ST_CONTINUE; } const char * rb_econv_stateless_encoding(const char *stateful_enc) { struct stateless_encoding_t data; data.stateful_enc = stateful_enc; data.stateless_enc = NULL; st_foreach(transcoder_table, stateless_encoding_i, (st_data_t)&data); if (data.stateless_enc) return data.stateless_enc; return NULL; } VALUE rb_econv_substr_append(rb_econv_t *ec, VALUE src, long off, long len, VALUE dst, int flags) { unsigned const char *ss, *sp, *se; unsigned char *ds, *dp, *de; rb_econv_result_t res; int max_output; if (NIL_P(dst)) { dst = rb_str_buf_new(len); if (ec->destination_encoding) rb_enc_associate(dst, ec->destination_encoding); } if (ec->last_tc) max_output = ec->last_tc->transcoder->max_output; else max_output = 1; res = econv_destination_buffer_full; while (res == econv_destination_buffer_full) { long dlen = RSTRING_LEN(dst); if (rb_str_capacity(dst) - dlen < (size_t)len + max_output) { unsigned long new_capa = (unsigned long)dlen + len + max_output; if (LONG_MAX < new_capa) rb_raise(rb_eArgError, "too long string"); rb_str_resize(dst, new_capa); rb_str_set_len(dst, dlen); } ss = sp = (const unsigned char *)RSTRING_PTR(src) + off; se = ss + len; ds = (unsigned char *)RSTRING_PTR(dst); de = ds + rb_str_capacity(dst); dp = ds += dlen; res = rb_econv_convert(ec, &sp, se, &dp, de, flags); off += sp - ss; len -= sp - ss; rb_str_set_len(dst, dlen + (dp - ds)); rb_econv_check_error(ec); } return dst; } VALUE rb_econv_str_append(rb_econv_t *ec, VALUE src, VALUE dst, int flags) { return rb_econv_substr_append(ec, src, 0, RSTRING_LEN(src), dst, flags); } VALUE rb_econv_substr_convert(rb_econv_t *ec, VALUE src, long byteoff, long bytesize, int flags) { return rb_econv_substr_append(ec, src, byteoff, bytesize, Qnil, flags); } VALUE rb_econv_str_convert(rb_econv_t *ec, VALUE src, int flags) { return rb_econv_substr_append(ec, src, 0, RSTRING_LEN(src), Qnil, flags); } void rb_econv_binmode(rb_econv_t *ec) { if (ec->opts.flags & ECONV_UNIVERSAL_NEWLINE_DECODER) { int i = ec->num_trans-1; rb_transcoding_close(ec->elems[i].tc); xfree(ec->elems[i].out_buf_start); ec->elems[i].tc = NULL; ec->elems[i].out_buf_start = NULL; ec->elems[i].out_data_start = NULL; ec->elems[i].out_data_end = NULL; ec->elems[i].out_buf_end = NULL; ec->num_trans--; ec->opts.flags &= ~ECONV_UNIVERSAL_NEWLINE_DECODER; } if (ec->opts.flags & (ECONV_CRLF_NEWLINE_ENCODER|ECONV_CR_NEWLINE_ENCODER)) { rb_transcoding_close(ec->elems[0].tc); xfree(ec->elems[0].out_buf_start); MEMMOVE(&ec->elems[0], &ec->elems[1], rb_econv_elem_t, ec->num_trans-1); ec->num_trans--; ec->opts.flags &= ~(ECONV_CRLF_NEWLINE_ENCODER|ECONV_CR_NEWLINE_ENCODER); } } static VALUE econv_description(const char *senc, const char *denc, rb_econv_option_t *opts, VALUE mesg) { int flags = opts ? opts->flags : 0; int has_description = 0; if (NIL_P(mesg)) mesg = rb_str_new(NULL, 0); if (*senc != '\0' || *denc != '\0') { if (*senc == '\0') rb_str_cat2(mesg, denc); else if (*denc == '\0') rb_str_cat2(mesg, senc); else rb_str_catf(mesg, "%s to %s", senc, denc); has_description = 1; } if (flags & (ECONV_UNIVERSAL_NEWLINE_DECODER| ECONV_CRLF_NEWLINE_ENCODER| ECONV_CR_NEWLINE_ENCODER)) { const char *pre = ""; if (has_description) rb_str_cat2(mesg, " with "); if (flags & ECONV_UNIVERSAL_NEWLINE_DECODER) { rb_str_cat2(mesg, pre); pre = ","; rb_str_cat2(mesg, "Universal-newline"); } if (flags & ECONV_CRLF_NEWLINE_ENCODER) { rb_str_cat2(mesg, pre); pre = ","; rb_str_cat2(mesg, "CRLF-newline"); } if (flags & ECONV_CR_NEWLINE_ENCODER) { rb_str_cat2(mesg, pre); pre = ","; rb_str_cat2(mesg, "CR-newline"); } has_description = 1; } if (!has_description) { rb_str_cat2(mesg, "no-conversion"); } return mesg; } VALUE rb_econv_open_exc(const char *senc, const char *denc, rb_econv_option_t *opts) { VALUE mesg, exc; mesg = rb_str_new_cstr("code converter open failed ("); econv_description(senc, denc, opts, mesg); rb_str_cat2(mesg, ")"); exc = rb_exc_new3(rb_eNoConverter, mesg); return exc; } static VALUE make_econv_exception(rb_econv_t *ec) { VALUE mesg, exc; if (ec->last_error.result == econv_invalid_byte_sequence || ec->last_error.result == econv_incomplete_input) { const char *err = (const char *)ec->last_error.error_bytes_start; size_t error_len = ec->last_error.error_bytes_len; VALUE bytes = rb_str_new(err, error_len); VALUE dumped = rb_str_dump(bytes); size_t readagain_len = ec->last_error.readagain_len; VALUE bytes2 = Qnil; VALUE dumped2; int idx; if (ec->last_error.result == econv_incomplete_input) { mesg = rb_sprintf("incomplete %s on %s", StringValueCStr(dumped), ec->last_error.source_encoding); } else if (readagain_len) { bytes2 = rb_str_new(err+error_len, readagain_len); dumped2 = rb_str_dump(bytes2); mesg = rb_sprintf("%s followed by %s on %s", StringValueCStr(dumped), StringValueCStr(dumped2), ec->last_error.source_encoding); } else { mesg = rb_sprintf("%s on %s", StringValueCStr(dumped), ec->last_error.source_encoding); } exc = rb_exc_new3(rb_eInvalidByteSequence, mesg); rb_ivar_set(exc, rb_intern("error_bytes"), bytes); rb_ivar_set(exc, rb_intern("readagain_bytes"), bytes2); rb_ivar_set(exc, rb_intern("incomplete_input"), ec->last_error.result == econv_incomplete_input ? Qtrue : Qfalse); set_encs: rb_ivar_set(exc, rb_intern("source_encoding_name"), rb_str_new2(ec->last_error.source_encoding)); rb_ivar_set(exc, rb_intern("destination_encoding_name"), rb_str_new2(ec->last_error.destination_encoding)); idx = rb_enc_find_index(ec->last_error.source_encoding); if (0 <= idx) rb_ivar_set(exc, rb_intern("source_encoding"), rb_enc_from_encoding(rb_enc_from_index(idx))); idx = rb_enc_find_index(ec->last_error.destination_encoding); if (0 <= idx) rb_ivar_set(exc, rb_intern("destination_encoding"), rb_enc_from_encoding(rb_enc_from_index(idx))); return exc; } if (ec->last_error.result == econv_undefined_conversion) { VALUE bytes = rb_str_new((const char *)ec->last_error.error_bytes_start, ec->last_error.error_bytes_len); VALUE dumped; int idx; dumped = rb_str_dump(bytes); mesg = rb_sprintf("%s from %s to %s", StringValueCStr(dumped), ec->last_error.source_encoding, ec->last_error.destination_encoding); exc = rb_exc_new3(rb_eConversionUndefined, mesg); idx = rb_enc_find_index(ec->last_error.source_encoding); if (0 <= idx) rb_enc_associate_index(bytes, idx); rb_ivar_set(exc, rb_intern("error_char"), bytes); goto set_encs; } return Qnil; } static void more_output_buffer( VALUE destination, unsigned char *(*resize_destination)(VALUE, int, int), int max_output, unsigned char **out_start_ptr, unsigned char **out_pos, unsigned char **out_stop_ptr) { size_t len = (*out_pos - *out_start_ptr); size_t new_len = (len + max_output) * 2; *out_start_ptr = resize_destination(destination, len, new_len); *out_pos = *out_start_ptr + len; *out_stop_ptr = *out_start_ptr + new_len; } static int output_replacement_character(rb_econv_t *ec) { rb_transcoding *tc; const rb_transcoder *tr; rb_encoding *enc; const unsigned char *replacement; const char *repl_enc; int len; int ret; tc = ec->last_tc; if (tc) { tr = tc->transcoder; enc = rb_enc_find(tr->to_encoding); replacement = (const unsigned char *)get_replacement_character(enc, &len, &repl_enc); } else { replacement = (unsigned char *)"?"; len = 1; repl_enc = ""; } ret = rb_econv_insert_output(ec, replacement, len, repl_enc); if (ret == -1) return -1; return 0; } #if 1 static void transcode_loop(const unsigned char **in_pos, unsigned char **out_pos, const unsigned char *in_stop, unsigned char *out_stop, VALUE destination, unsigned char *(*resize_destination)(VALUE, int, int), const char *from_encoding, const char *to_encoding, rb_econv_option_t *ecopts) { rb_econv_t *ec; rb_transcoding *last_tc; rb_econv_result_t ret; unsigned char *out_start = *out_pos; int max_output; VALUE exc; ec = rb_econv_open(from_encoding, to_encoding, ecopts); if (!ec) rb_exc_raise(rb_econv_open_exc(from_encoding, to_encoding, ecopts)); last_tc = ec->last_tc; max_output = last_tc ? last_tc->transcoder->max_output : 1; resume: ret = rb_econv_convert(ec, in_pos, in_stop, out_pos, out_stop, 0); if (ret == econv_invalid_byte_sequence || ret == econv_incomplete_input || ret == econv_undefined_conversion) { exc = make_econv_exception(ec); rb_econv_close(ec); rb_exc_raise(exc); } if (ret == econv_destination_buffer_full) { more_output_buffer(destination, resize_destination, max_output, &out_start, out_pos, &out_stop); goto resume; } rb_econv_close(ec); return; } #else /* sample transcode_loop implementation in byte-by-byte stream style */ static void transcode_loop(const unsigned char **in_pos, unsigned char **out_pos, const unsigned char *in_stop, unsigned char *out_stop, VALUE destination, unsigned char *(*resize_destination)(VALUE, int, int), const char *from_encoding, const char *to_encoding, rb_econv_option_t *ecopts) { rb_econv_t *ec; rb_transcoding *last_tc; rb_econv_result_t ret; unsigned char *out_start = *out_pos; const unsigned char *ptr; int max_output; VALUE exc; ec = rb_econv_open(from_encoding, to_encoding, ecopts); if (!ec) rb_exc_raise(rb_econv_open_exc(from_encoding, to_encoding, ecopts)); last_tc = ec->last_tc; max_output = last_tc ? last_tc->transcoder->max_output : 1; ret = econv_source_buffer_empty; ptr = *in_pos; while (ret != econv_finished) { unsigned char input_byte; const unsigned char *p = &input_byte; if (ret == econv_source_buffer_empty) { if (ptr < in_stop) { input_byte = *ptr; ret = rb_econv_convert(ec, &p, p+1, out_pos, out_stop, ECONV_PARTIAL_INPUT); } else { ret = rb_econv_convert(ec, NULL, NULL, out_pos, out_stop, 0); } } else { ret = rb_econv_convert(ec, NULL, NULL, out_pos, out_stop, ECONV_PARTIAL_INPUT); } if (&input_byte != p) ptr += p - &input_byte; switch (ret) { case econv_invalid_byte_sequence: case econv_incomplete_input: case econv_undefined_conversion: exc = make_econv_exception(ec); rb_econv_close(ec); rb_exc_raise(exc); break; case econv_destination_buffer_full: more_output_buffer(destination, resize_destination, max_output, &out_start, out_pos, &out_stop); break; case econv_source_buffer_empty: break; case econv_finished: break; } } rb_econv_close(ec); *in_pos = in_stop; return; } #endif /* * String-specific code */ static unsigned char * str_transcoding_resize(VALUE destination, int len, int new_len) { rb_str_resize(destination, new_len); return (unsigned char *)RSTRING_PTR(destination); } static int econv_opts(VALUE opt) { VALUE v; int options = 0; v = rb_hash_aref(opt, sym_invalid); if (NIL_P(v)) { } else if (v==sym_ignore) { options |= ECONV_INVALID_IGNORE; } else if (v==sym_replace) { options |= ECONV_INVALID_REPLACE; v = rb_hash_aref(opt, sym_replace); } else { rb_raise(rb_eArgError, "unknown value for invalid character option"); } v = rb_hash_aref(opt, sym_undef); if (NIL_P(v)) { } else if (v==sym_ignore) { options |= ECONV_UNDEF_IGNORE; } else if (v==sym_replace) { options |= ECONV_UNDEF_REPLACE; } else { rb_raise(rb_eArgError, "unknown value for undefined character option"); } return options; } void rb_econv_opts(VALUE hash, rb_econv_option_t *opts) { if (NIL_P(hash)) opts->flags = 0; else opts->flags = econv_opts(hash); } static int str_transcode_enc_args(VALUE str, VALUE arg1, VALUE arg2, const char **sname, rb_encoding **senc, const char **dname, rb_encoding **denc) { rb_encoding *from_enc, *to_enc; const char *from_e, *to_e; int from_encidx, to_encidx; VALUE from_encval, to_encval; if ((to_encidx = rb_to_encoding_index(to_encval = arg1)) < 0) { to_enc = 0; to_encidx = 0; to_e = StringValueCStr(to_encval); } else { to_enc = rb_enc_from_index(to_encidx); to_e = rb_enc_name(to_enc); } if (NIL_P(arg2)) { from_encidx = rb_enc_get_index(str); from_enc = rb_enc_from_index(from_encidx); from_e = rb_enc_name(from_enc); } else if ((from_encidx = rb_to_encoding_index(from_encval = arg2)) < 0) { from_enc = 0; from_e = StringValueCStr(from_encval); } else { from_enc = rb_enc_from_index(from_encidx); from_e = rb_enc_name(from_enc); } *sname = from_e; *senc = from_enc; *dname = to_e; *denc = to_enc; return to_encidx; } static int str_transcode0(int argc, VALUE *argv, VALUE *self, rb_econv_option_t *ecopts_arg) { VALUE dest; VALUE str = *self; long blen, slen; unsigned char *buf, *bp, *sp; const unsigned char *fromp; rb_encoding *from_enc, *to_enc; const char *from_e, *to_e; int to_encidx; rb_econv_option_t ecopts; if (argc < 1 || argc > 2) { rb_raise(rb_eArgError, "wrong number of arguments (%d for 1..2)", argc); } to_encidx = str_transcode_enc_args(str, argv[0], argc==1 ? Qnil : argv[1], &from_e, &from_enc, &to_e, &to_enc); if (ecopts_arg) ecopts = *ecopts_arg; else rb_econv_opts(Qnil, &ecopts); if ((ecopts.flags & (ECONV_UNIVERSAL_NEWLINE_DECODER| ECONV_CRLF_NEWLINE_ENCODER| ECONV_CR_NEWLINE_ENCODER)) == 0) { if (from_enc && from_enc == to_enc) { return -1; } if (from_enc && to_enc && rb_enc_asciicompat(from_enc) && rb_enc_asciicompat(to_enc)) { if (ENC_CODERANGE(str) == ENC_CODERANGE_7BIT) { return to_encidx; } } if (encoding_equal(from_e, to_e)) { return -1; } } else { if (encoding_equal(from_e, to_e)) { from_e = ""; to_e = ""; } } fromp = sp = (unsigned char *)RSTRING_PTR(str); slen = RSTRING_LEN(str); blen = slen + 30; /* len + margin */ dest = rb_str_tmp_new(blen); bp = (unsigned char *)RSTRING_PTR(dest); transcode_loop(&fromp, &bp, (sp+slen), (bp+blen), dest, str_transcoding_resize, from_e, to_e, &ecopts); if (fromp != sp+slen) { rb_raise(rb_eArgError, "not fully converted, %"PRIdPTRDIFF" bytes left", sp+slen-fromp); } buf = (unsigned char *)RSTRING_PTR(dest); *bp = '\0'; rb_str_set_len(dest, bp - buf); /* set encoding */ if (!to_enc) { to_encidx = rb_define_dummy_encoding(to_e); } *self = dest; return to_encidx; } static int str_transcode(int argc, VALUE *argv, VALUE *self) { VALUE opt; rb_econv_option_t ecopts; ecopts.flags = 0; if (0 < argc) { opt = rb_check_convert_type(argv[argc-1], T_HASH, "Hash", "to_hash"); if (!NIL_P(opt)) { argc--; rb_econv_opts(opt, &ecopts); } } return str_transcode0(argc, argv, self, &ecopts); } static inline VALUE str_encode_associate(VALUE str, int encidx) { int cr = 0; rb_enc_associate_index(str, encidx); /* transcoded string never be broken. */ if (rb_enc_asciicompat(rb_enc_from_index(encidx))) { rb_str_coderange_scan_restartable(RSTRING_PTR(str), RSTRING_END(str), 0, &cr); } else { cr = ENC_CODERANGE_VALID; } ENC_CODERANGE_SET(str, cr); return str; } /* * call-seq: * str.encode!(encoding [, options] ) => str * str.encode!(to_encoding, from_encoding [, options] ) => str * * The first form transcodes the contents of str from * str.encoding to +encoding+. * The second form transcodes the contents of str from * from_encoding to to_encoding. * The options Hash gives details for conversion. See String#encode * for details. * Returns the string even if no changes were made. */ static VALUE str_encode_bang(int argc, VALUE *argv, VALUE str) { VALUE newstr = str; int encidx = str_transcode(argc, argv, &newstr); if (encidx < 0) return str; rb_str_shared_replace(str, newstr); return str_encode_associate(str, encidx); } /* * call-seq: * str.encode(encoding [, options] ) => str * str.encode(to_encoding, from_encoding [, options] ) => str * * The first form returns a copy of str transcoded * to encoding +encoding+. * The second form returns a copy of str transcoded * from from_encoding to to_encoding. * The options Hash gives details for conversion. Details * to be added. */ static VALUE str_encode(int argc, VALUE *argv, VALUE str) { VALUE newstr = str; int encidx = str_transcode(argc, argv, &newstr); if (encidx < 0) return rb_str_dup(str); RBASIC(newstr)->klass = rb_obj_class(str); return str_encode_associate(newstr, encidx); } VALUE rb_str_transcode(VALUE str, VALUE to, rb_econv_option_t *ecopts) { int argc = 1; VALUE *argv = &to; VALUE newstr = str; int encidx = str_transcode0(argc, argv, &newstr, ecopts); if (encidx < 0) return rb_str_dup(str); RBASIC(newstr)->klass = rb_obj_class(str); return str_encode_associate(newstr, encidx); } static void econv_free(rb_econv_t *ec) { rb_econv_close(ec); } static VALUE econv_s_allocate(VALUE klass) { return Data_Wrap_Struct(klass, NULL, econv_free, NULL); } static rb_encoding * make_dummy_encoding(const char *name) { rb_encoding *enc; int idx; idx = rb_define_dummy_encoding(name); enc = rb_enc_from_index(idx); return enc; } /* * call-seq: * Encoding::Converter.new(source_encoding, destination_encoding) * Encoding::Converter.new(source_encoding, destination_encoding, flags) * * possible flags: * Encoding::Converter::UNIVERSAL_NEWLINE_DECODER # convert CRLF and CR to LF at last * Encoding::Converter::CRLF_NEWLINE_ENCODER # convert LF to CRLF at first * Encoding::Converter::CR_NEWLINE_ENCODER # convert LF to CR at first * * Encoding::Converter.new creates an instance of Encoding::Converter. * * source_encoding and destination_encoding should be a string or * Encoding object. * * flags should be an integer. * * example: * # UTF-16BE to UTF-8 * ec = Encoding::Converter.new("UTF-16BE", "UTF-8") * * # (1) convert UTF-16BE to UTF-8 * # (2) convert CRLF and CR to LF * ec = Encoding::Converter.new("UTF-16BE", "UTF-8", Encoding::Converter::UNIVERSAL_NEWLINE_DECODER) * * # (1) convert LF to CRLF * # (2) convert UTF-8 to UTF-16BE * ec = Encoding::Converter.new("UTF-8", "UTF-16BE", Encoding::Converter::CRLF_NEWLINE_ENCODER) * */ static VALUE econv_init(int argc, VALUE *argv, VALUE self) { VALUE source_encoding, destination_encoding, flags_v; int sidx, didx; const char *sname, *dname; rb_encoding *senc, *denc; rb_econv_t *ec; rb_econv_option_t ecopts; rb_scan_args(argc, argv, "21", &source_encoding, &destination_encoding, &flags_v); if (flags_v == Qnil) ecopts.flags = 0; else ecopts.flags = NUM2INT(flags_v); senc = NULL; sidx = rb_to_encoding_index(source_encoding); if (0 <= sidx) { senc = rb_enc_from_index(sidx); } else { StringValue(source_encoding); } denc = NULL; didx = rb_to_encoding_index(destination_encoding); if (0 <= didx) { denc = rb_enc_from_index(didx); } else { StringValue(destination_encoding); } sname = senc ? senc->name : StringValueCStr(source_encoding); dname = denc ? denc->name : StringValueCStr(destination_encoding); if (DATA_PTR(self)) { rb_raise(rb_eTypeError, "already initialized"); } ec = rb_econv_open(sname, dname, &ecopts); if (!ec) { rb_exc_raise(rb_econv_open_exc(sname, dname, &ecopts)); } if (*sname && *dname) { /* check "" to "universal_newline" */ if (!senc) senc = make_dummy_encoding(sname); if (!denc) denc = make_dummy_encoding(dname); } ec->source_encoding = senc; ec->destination_encoding = denc; if (ec->last_tc) { ec->source_encoding_name = ec->elems[0].tc->transcoder->from_encoding; ec->destination_encoding_name = ec->last_tc->transcoder->to_encoding; } else { ec->source_encoding_name = ""; ec->destination_encoding_name = ""; } DATA_PTR(self) = ec; return self; } /* * call-seq: * ec.inspect -> string * * Returns a printable version of ec * * ec = Encoding::Converter.new("iso-8859-1", "utf-8") * puts ec.inspect #=> # * */ static VALUE econv_inspect(VALUE self) { const char *cname = rb_obj_classname(self); rb_econv_t *ec = DATA_PTR(self); if (!ec) return rb_sprintf("#<%s: uninitialized>", cname); else { const char *sname = ec->source_encoding_name; const char *dname = ec->destination_encoding_name; VALUE str; str = rb_sprintf("#<%s: ", cname); econv_description(sname, dname, &ec->opts, str); rb_str_cat2(str, ">"); return str; } } #define IS_ECONV(obj) (RDATA(obj)->dfree == (RUBY_DATA_FUNC)econv_free) static rb_econv_t * check_econv(VALUE self) { Check_Type(self, T_DATA); if (!IS_ECONV(self)) { rb_raise(rb_eTypeError, "wrong argument type %s (expected Encoding::Converter)", rb_class2name(CLASS_OF(self))); } if (!DATA_PTR(self)) { rb_raise(rb_eTypeError, "uninitialized encoding converter"); } return DATA_PTR(self); } /* * call-seq: * ec.source_encoding -> encoding * * returns the source encoding as an Encoding object. */ static VALUE econv_source_encoding(VALUE self) { rb_econv_t *ec = check_econv(self); if (!ec->source_encoding) return Qnil; return rb_enc_from_encoding(ec->source_encoding); } /* * call-seq: * ec.destination_encoding -> encoding * * returns the destination encoding as an Encoding object. */ static VALUE econv_destination_encoding(VALUE self) { rb_econv_t *ec = check_econv(self); if (!ec->destination_encoding) return Qnil; return rb_enc_from_encoding(ec->destination_encoding); } static VALUE econv_result_to_symbol(rb_econv_result_t res) { switch (res) { case econv_invalid_byte_sequence: return sym_invalid_byte_sequence; case econv_incomplete_input: return sym_incomplete_input; case econv_undefined_conversion: return sym_undefined_conversion; case econv_destination_buffer_full: return sym_destination_buffer_full; case econv_source_buffer_empty: return sym_source_buffer_empty; case econv_finished: return sym_finished; case econv_output_followed_by_input: return sym_output_followed_by_input; default: return INT2NUM(res); /* should not be reached */ } } /* * call-seq: * ec.primitive_convert(source_buffer, destination_buffer) -> symbol * ec.primitive_convert(source_buffer, destination_buffer, destination_byteoffset) -> symbol * ec.primitive_convert(source_buffer, destination_buffer, destination_byteoffset, destination_bytesize) -> symbol * ec.primitive_convert(source_buffer, destination_buffer, destination_byteoffset, destination_bytesize, flags) -> symbol * * possible flags: * Encoding::Converter::PARTIAL_INPUT # source buffer may be part of larger source * Encoding::Converter::OUTPUT_FOLLOWED_BY_INPUT # stop conversion after output before input * * possible results: * :invalid_byte_sequence * :incomplete_input * :undefined_conversion * :output_followed_by_input * :destination_buffer_full * :source_buffer_empty * :finished * * primitive_convert converts source_buffer into destination_buffer. * * source_buffer should be a string or nil. * nil means a empty string. * * destination_buffer should be a string. * * destination_byteoffset should be an integer or nil. * nil means the end of destination_buffer. * If it is omitted, nil is assumed. * * destination_bytesize should be an integer or nil. * nil means unlimited. * If it is omitted, nil is assumed. * * flags should be an integer or nil. * nil means no flags. * If it is omitted, nil is assumed. * * primitive_convert converts the content of source_buffer from beginning * and store the result into destination_buffer. * * destination_byteoffset and destination_bytesize specify the region which * the converted result is stored. * destination_byteoffset specifies the start position in destination_buffer in bytes. * If destination_byteoffset is nil, * destination_buffer.bytesize is used for appending the result. * destination_bytesize specifies maximum number of bytes. * If destination_bytesize is nil, * destination size is unlimited. * After conversion, destination_buffer is resized to * destination_byteoffset + actually produced number of bytes. * Also destination_buffer's encoding is set to destination_encoding. * * primitive_convert drops the converted part of source_buffer. * the dropped part is converted in destination_buffer or * buffered in Encoding::Converter object. * * primitive_convert stops conversion when one of following condition met. * - invalid byte sequence found in source buffer (:invalid_byte_sequence) * - unexpected end of source buffer (:incomplete_input) * this occur only when PARTIAL_INPUT is not specified. * - character not representable in output encoding (:undefined_conversion) * - after some output is generated, before input is done (:output_followed_by_input) * this occur only when OUTPUT_FOLLOWED_BY_INPUT is specified. * - destination buffer is full (:destination_buffer_full) * this occur only when destination_bytesize is non-nil. * - source buffer is empty (:source_buffer_empty) * this occur only when PARTIAL_INPUT is specified. * - conversion is finished (:finished) * * example: * ec = Encoding::Converter.new("UTF-8", "UTF-16BE") * ret = ec.primitive_convert(src="pi", dst="", 100) * p [ret, src, dst] #=> [:finished, "", "\x00p\x00i"] * * ec = Encoding::Converter.new("UTF-8", "UTF-16BE") * ret = ec.primitive_convert(src="pi", dst="", 1) * p [ret, src, dst] #=> [:destination_buffer_full, "i", "\x00"] * ret = ec.primitive_convert(src, dst="", 1) * p [ret, src, dst] #=> [:destination_buffer_full, "", "p"] * ret = ec.primitive_convert(src, dst="", 1) * p [ret, src, dst] #=> [:destination_buffer_full, "", "\x00"] * ret = ec.primitive_convert(src, dst="", 1) * p [ret, src, dst] #=> [:finished, "", "i"] * */ static VALUE econv_primitive_convert(int argc, VALUE *argv, VALUE self) { VALUE input, output, output_byteoffset_v, output_bytesize_v, flags_v; rb_econv_t *ec = check_econv(self); rb_econv_result_t res; const unsigned char *ip, *is; unsigned char *op, *os; long output_byteoffset, output_bytesize; unsigned long output_byteend; int flags; rb_scan_args(argc, argv, "23", &input, &output, &output_byteoffset_v, &output_bytesize_v, &flags_v); if (NIL_P(output_byteoffset_v)) output_byteoffset = 0; /* dummy */ else output_byteoffset = NUM2LONG(output_byteoffset_v); if (NIL_P(output_bytesize_v)) output_bytesize = 0; /* dummy */ else output_bytesize = NUM2LONG(output_bytesize_v); if (NIL_P(flags_v)) flags = 0; else flags = NUM2INT(flags_v); StringValue(output); if (!NIL_P(input)) StringValue(input); rb_str_modify(output); if (NIL_P(output_bytesize_v)) { output_bytesize = RSTRING_EMBED_LEN_MAX; if (!NIL_P(input) && output_bytesize < RSTRING_LEN(input)) output_bytesize = RSTRING_LEN(input); } retry: if (NIL_P(output_byteoffset_v)) output_byteoffset = RSTRING_LEN(output); if (output_byteoffset < 0) rb_raise(rb_eArgError, "negative output_byteoffset"); if (RSTRING_LEN(output) < output_byteoffset) rb_raise(rb_eArgError, "output_byteoffset too big"); if (output_bytesize < 0) rb_raise(rb_eArgError, "negative output_bytesize"); output_byteend = (unsigned long)output_byteoffset + (unsigned long)output_bytesize; if (output_byteend < (unsigned long)output_byteoffset || LONG_MAX < output_byteend) rb_raise(rb_eArgError, "output_byteoffset+output_bytesize too big"); if (rb_str_capacity(output) < output_byteend) rb_str_resize(output, output_byteend); if (NIL_P(input)) { ip = is = NULL; } else { ip = (const unsigned char *)RSTRING_PTR(input); is = ip + RSTRING_LEN(input); } op = (unsigned char *)RSTRING_PTR(output) + output_byteoffset; os = op + output_bytesize; res = rb_econv_convert(ec, &ip, is, &op, os, flags); rb_str_set_len(output, op-(unsigned char *)RSTRING_PTR(output)); if (!NIL_P(input)) rb_str_drop_bytes(input, ip - (unsigned char *)RSTRING_PTR(input)); if (NIL_P(output_bytesize_v) && res == econv_destination_buffer_full) { if (LONG_MAX / 2 < output_bytesize) rb_raise(rb_eArgError, "too long conversion result"); output_bytesize *= 2; output_byteoffset_v = Qnil; goto retry; } if (ec->destination_encoding) { rb_enc_associate(output, ec->destination_encoding); } return econv_result_to_symbol(res); } /* * call-seq: * ec.convert(source_string) -> destination_string * * convert source_string and return destination_string. * * source_string is assumed as a part of source. * i.e. Encoding::Converter::PARTIAL_INPUT is used internally. * finish method should be used at last. * * ec = Encoding::Converter.new("utf-8", "euc-jp") * puts ec.convert("\u3042").dump #=> "\xA4\xA2" * puts ec.finish.dump #=> "" * * ec = Encoding::Converter.new("euc-jp", "utf-8") * puts ec.convert("\xA4").dump #=> "" * puts ec.convert("\xA2").dump #=> "\xE3\x81\x82" * puts ec.finish.dump #=> "" * * ec = Encoding::Converter.new("utf-8", "iso-2022-jp") * puts ec.convert("\xE3").dump #=> "".force_encoding("ISO-2022-JP") * puts ec.convert("\x81").dump #=> "".force_encoding("ISO-2022-JP") * puts ec.convert("\x82").dump #=> "\e$B$\"".force_encoding("ISO-2022-JP") * puts ec.finish.dump #=> "\e(B".force_encoding("ISO-2022-JP") * * If a conversion error occur, * Encoding::ConversionUndefined or * Encoding::InvalidByteSequence is raised. * */ static VALUE econv_convert(VALUE self, VALUE source_string) { VALUE ret, dst; VALUE av[5]; int ac; rb_econv_t *ec = check_econv(self); StringValue(source_string); dst = rb_str_new(NULL, 0); av[0] = rb_str_dup(source_string); av[1] = dst; av[2] = Qnil; av[3] = Qnil; av[4] = INT2NUM(ECONV_PARTIAL_INPUT); ac = 5; ret = econv_primitive_convert(ac, av, self); if (ret == sym_invalid_byte_sequence || ret == sym_undefined_conversion || ret == sym_incomplete_input) { VALUE exc = make_econv_exception(ec); rb_exc_raise(exc); } if (ret == sym_finished) { rb_raise(rb_eArgError, "converter already finished"); } if (ret != sym_source_buffer_empty) { rb_bug("unexpected result of econv_primitive_convert"); } return dst; } /* * call-seq: * ec.finish -> string * * finishes the converter. * It returns the last part of converted string. * * ec = Encoding::Converter.new("utf-8", "iso-2022-jp") * p ec.convert("\u3042") #=> "\e$B$\"" * p ec.finish #=> "\e(B" */ static VALUE econv_finish(VALUE self) { VALUE ret, dst; VALUE av[5]; int ac; rb_econv_t *ec = check_econv(self); dst = rb_str_new(NULL, 0); av[0] = Qnil; av[1] = dst; av[2] = Qnil; av[3] = Qnil; av[4] = INT2NUM(0); ac = 5; ret = econv_primitive_convert(ac, av, self); if (ret == sym_invalid_byte_sequence || ret == sym_undefined_conversion || ret == sym_incomplete_input) { VALUE exc = make_econv_exception(ec); rb_exc_raise(exc); } if (ret != sym_finished) { rb_bug("unexpected result of econv_primitive_convert"); } return dst; } /* * call-seq: * ec.primitive_errinfo -> array * * primitive_errinfo returns a precious information of the last error result * as a 5-elements array: * * [result, enc1, enc2, error_bytes, readagain_bytes] * * result is the last result of primitive_convert. * * Other elements are only meaningful when result is * :invalid_byte_sequence, :incomplete_input or :undefined_conversion. * * enc1 and enc2 indicates a conversion step as pair of strings. * For example, a converter from EUC-JP to ISO-8859-1 converters * a string as EUC-JP -> UTF-8 -> ISO-8859-1. * So [enc1, enc2] is ["EUC-JP", "UTF-8"] or ["UTF-8", "ISO-8859-1"]. * * error_bytes and readagain_bytes indicates the byte sequences which causes the error. * error_bytes is discarded portion. * readagain_bytes is buffered portion which is read again on next conversion. * * Example: * * # \xff is invalid as EUC-JP. * ec = Encoding::Converter.new("EUC-JP", "Shift_JIS") * ec.primitive_convert(src="\xff", dst="", nil, 10) * p ec.primitive_errinfo * #=> [:invalid_byte_sequence, "EUC-JP", "UTF-8", "\xFF", ""] * * # HIRAGANA LETTER A (\xa4\xa2 in EUC-JP) is not representable in ISO-8859-1. * # Since this error is occur in UTF-8 to ISO-8859-1 conversion, * # error_bytes is HIRAGANA LETTER A in UTF-8 (\xE3\x81\x82). * ec = Encoding::Converter.new("EUC-JP", "ISO-8859-1") * ec.primitive_convert(src="\xa4\xa2", dst="", nil, 10) * p ec.primitive_errinfo * #=> [:undefined_conversion, "UTF-8", "ISO-8859-1", "\xE3\x81\x82", ""] * * # partial character is invalid * ec = Encoding::Converter.new("EUC-JP", "ISO-8859-1") * ec.primitive_convert(src="\xa4", dst="", nil, 10) * p ec.primitive_errinfo * #=> [:incomplete_input, "EUC-JP", "UTF-8", "\xA4", ""] * * # Encoding::Converter::PARTIAL_INPUT prevents invalid errors by * # partial characters. * ec = Encoding::Converter.new("EUC-JP", "ISO-8859-1") * ec.primitive_convert(src="\xa4", dst="", nil, 10, Encoding::Converter::PARTIAL_INPUT) * p ec.primitive_errinfo * #=> [:source_buffer_empty, nil, nil, nil, nil] * * # \xd8\x00\x00@ is invalid as UTF-16BE because * # no low surrogate after high surrogate (\xd8\x00). * # It is detected by 3rd byte (\00) which is part of next character. * # So the high surrogate (\xd8\x00) is discarded and * # the 3rd byte is read again later. * # Since the byte is buffered in ec, it is dropped from src. * ec = Encoding::Converter.new("UTF-16BE", "UTF-8") * ec.primitive_convert(src="\xd8\x00\x00@", dst="", nil, 10) * p ec.primitive_errinfo * #=> [:invalid_byte_sequence, "UTF-16BE", "UTF-8", "\xD8\x00", "\x00"] * p src * #=> "@" * * # Similar to UTF-16BE, \x00\xd8@\x00 is invalid as UTF-16LE. * # The problem is detected by 4th byte. * ec = Encoding::Converter.new("UTF-16LE", "UTF-8") * ec.primitive_convert(src="\x00\xd8@\x00", dst="", nil, 10) * p ec.primitive_errinfo * #=> [:invalid_byte_sequence, "UTF-16LE", "UTF-8", "\x00\xD8", "@\x00"] * p src * #=> "" * */ static VALUE econv_primitive_errinfo(VALUE self) { rb_econv_t *ec = check_econv(self); VALUE ary; ary = rb_ary_new2(5); rb_ary_store(ary, 0, econv_result_to_symbol(ec->last_error.result)); rb_ary_store(ary, 4, Qnil); if (ec->last_error.source_encoding) rb_ary_store(ary, 1, rb_str_new2(ec->last_error.source_encoding)); if (ec->last_error.destination_encoding) rb_ary_store(ary, 2, rb_str_new2(ec->last_error.destination_encoding)); if (ec->last_error.error_bytes_start) { rb_ary_store(ary, 3, rb_str_new((const char *)ec->last_error.error_bytes_start, ec->last_error.error_bytes_len)); rb_ary_store(ary, 4, rb_str_new((const char *)ec->last_error.error_bytes_start + ec->last_error.error_bytes_len, ec->last_error.readagain_len)); } return ary; } /* * call-seq: * ec.insert_output(string) -> nil * * inserts string into the encoding converter. * The string will be converted into the destination encoding and * outputed on later conversions. * * If the destination encoding is stateful, * string is converted according to the state and update the state. * * This method should be used only when a conversion error is occur. * * ec = Encoding::Converter.new("utf-8", "iso-8859-1") * src = "HIRAGANA LETTER A is \u{3042}." * dst = "" * p ec.primitive_convert(src, dst) #=> :undefined_conversion * puts "[#{dst.dump}, #{src.dump}]" #=> ["HIRAGANA LETTER A is ", "."] * ec.insert_output("") * p ec.primitive_convert(src, dst) #=> :finished * puts "[#{dst.dump}, #{src.dump}]" #=> ["HIRAGANA LETTER A is .", ""] * * ec = Encoding::Converter.new("utf-8", "iso-2022-jp") * src = "\u{306F 3041 3068 2661 3002}" # U+2661 is not representable in iso-2022-jp * dst = "" * p ec.primitive_convert(src, dst) #=> :undefined_conversion * puts "[#{dst.dump}, #{src.dump}]" #=> ["\e$B$O$!$H".force_encoding("ISO-2022-JP"), "\xE3\x80\x82"] * ec.insert_output "?" # state change required to output "?". * p ec.primitive_convert(src, dst) #=> :finished * puts "[#{dst.dump}, #{src.dump}]" #=> ["\e$B$O$!$H\e(B?\e$B!#\e(B".force_encoding("ISO-2022-JP"), ""] * */ static VALUE econv_insert_output(VALUE self, VALUE string) { const char *insert_enc; int ret; rb_econv_t *ec = check_econv(self); StringValue(string); insert_enc = rb_econv_encoding_to_insert_output(ec); string = rb_str_transcode(string, rb_enc_from_encoding(rb_enc_find(insert_enc)), 0); ret = rb_econv_insert_output(ec, (const unsigned char *)RSTRING_PTR(string), RSTRING_LEN(string), insert_enc); if (ret == -1) { rb_raise(rb_eArgError, "too big string"); } return Qnil; } /* * call-seq * ec.putback => string * ec.putback(max_numbytes) => string * * put back the bytes which will be converted. * * The bytes are caused by invalid_byte_sequence error. * When invalid_byte_sequence error, some bytes are discarded and * some bytes are buffered to be converted later. * The latter bytes can be put back. * It can be observed by * Encoding::InvalidByteSequence#readagain_bytes and * Encoding::Converter#primitive_errinfo. * * ec = Encoding::Converter.new("utf-16le", "iso-8859-1") * src = "\x00\xd8\x61\x00" * dst = "" * p ec.primitive_convert(src, dst) #=> :invalid_byte_sequence * p ec.primitive_errinfo #=> [:invalid_byte_sequence, "UTF-16LE", "UTF-8", "\x00\xD8", "a\x00"] * p ec.putback #=> "a\x00" * p ec.putback #=> "" # no more bytes to put back * */ static VALUE econv_putback(int argc, VALUE *argv, VALUE self) { rb_econv_t *ec = check_econv(self); int n; int putbackable; VALUE str, max; rb_scan_args(argc, argv, "01", &max); if (NIL_P(max)) n = rb_econv_putbackable(ec); else { n = NUM2INT(max); putbackable = rb_econv_putbackable(ec); if (putbackable < n) n = putbackable; } str = rb_str_new(NULL, n); rb_econv_putback(ec, (unsigned char *)RSTRING_PTR(str), n); if (ec->source_encoding) { rb_enc_associate(str, ec->source_encoding); } return str; } /* * call-seq: * ec.last_error -> exception or nil * * returns an exception object for the last conversion. * It returns nil if the last conversion is not an error. * * "error" means that * Encoding::InvalidByteSequence and Encoding::ConversionUndefined for * Encoding::Converter#convert and * :invalid_byte_sequence, :incomplete_input and :undefined_conversion for * Encoding::Converter#primitive_convert. * * ec = Encoding::Converter.new("utf-8", "iso-8859-1") * p ec.primitive_convert(src="\xf1abcd", dst="") #=> :invalid_byte_sequence * p ec.last_error #=> # * p ec.primitive_convert(src, dst, nil, 1) #=> :destination_buffer_full * p ec.last_error #=> nil * */ static VALUE econv_last_error(VALUE self) { rb_econv_t *ec = check_econv(self); VALUE exc; exc = make_econv_exception(ec); if (NIL_P(exc)) return Qnil; return exc; } void rb_econv_check_error(rb_econv_t *ec) { VALUE exc; exc = make_econv_exception(ec); if (NIL_P(exc)) return; rb_exc_raise(exc); } /* * call-seq: * ecerr.source_encoding_name -> string * * returns the source encoding name as a string. */ static VALUE ecerr_source_encoding_name(VALUE self) { return rb_attr_get(self, rb_intern("source_encoding_name")); } /* * call-seq: * ecerr.source_encoding -> encoding * * returns the source encoding as an encoding object. * * Note that the result may not be equal to the source encoding of * the encoding converter if the conversion has multiple steps. * * ec = Encoding::Converter.new("ISO-8859-1", "EUC-JP") # ISO-8859-1 -> UTF-8 -> EUC-JP * begin * ec.convert("\xa0") # NO-BREAK SPACE, which is available in UTF-8 but not in EUC-JP. * rescue Encoding::ConversionUndefined * p $!.source_encoding #=> # * p $!.destination_encoding #=> # * p $!.source_encoding_name #=> "UTF-8" * p $!.destination_encoding_name #=> "EUC-JP" * end * */ static VALUE ecerr_source_encoding(VALUE self) { return rb_attr_get(self, rb_intern("source_encoding")); } /* * call-seq: * ecerr.destination_encoding_name -> string * * returns the destination encoding name as a string. */ static VALUE ecerr_destination_encoding_name(VALUE self) { return rb_attr_get(self, rb_intern("destination_encoding_name")); } /* * call-seq: * ecerr.destination_encoding -> string * * returns the destination encoding as an encoding object. */ static VALUE ecerr_destination_encoding(VALUE self) { return rb_attr_get(self, rb_intern("destination_encoding")); } /* * call-seq: * ecerr.error_char -> string * * returns the one-character string which cause Encoding::ConversionUndefined. * * ec = Encoding::Converter.new("ISO-8859-1", "EUC-JP") * begin * ec.convert("\xa0") * rescue Encoding::ConversionUndefined * puts $!.error_char.dump #=> "\xC2\xA0" * p $!.error_char.encoding #=> # * end * */ static VALUE ecerr_error_char(VALUE self) { return rb_attr_get(self, rb_intern("error_char")); } /* * call-seq: * ecerr.error_bytes -> string * * returns the discarded bytes when Encoding::InvalidByteSequence occur. * * ec = Encoding::Converter.new("EUC-JP", "ISO-8859-1") * begin * ec.convert("abc\xA1\xFFdef") * rescue Encoding::InvalidByteSequence * p $! #=> # * puts $!.error_bytes.dump #=> "\xA1" * puts $!.readagain_bytes.dump #=> "\xFF" * end */ static VALUE ecerr_error_bytes(VALUE self) { return rb_attr_get(self, rb_intern("error_bytes")); } /* * call-seq: * ecerr.readagain_bytes -> string * * returns the bytes to be read again when Encoding::InvalidByteSequence occur. */ static VALUE ecerr_readagain_bytes(VALUE self) { return rb_attr_get(self, rb_intern("readagain_bytes")); } /* * call-seq: * ecerr.incomplete_input? -> true or false * * returns true if the invalid byte sequence error is caused by * premature end of string. * * ec = Encoding::Converter.new("EUC-JP", "ISO-8859-1") * * begin * ec.convert("abc\xA1z") * rescue Encoding::InvalidByteSequence * p $! #=> # * p $!.incomplete_input? #=> false * end * * begin * ec.convert("abc\xA1") * ec.finish * rescue Encoding::InvalidByteSequence * p $! #=> # * p $!.incomplete_input? #=> true * end */ static VALUE ecerr_incomplete_input(VALUE self) { return rb_attr_get(self, rb_intern("incomplete_input")); } extern void Init_newline(void); void Init_transcode(void) { rb_eConversionUndefined = rb_define_class_under(rb_cEncoding, "ConversionUndefined", rb_eStandardError); rb_eInvalidByteSequence = rb_define_class_under(rb_cEncoding, "InvalidByteSequence", rb_eStandardError); rb_eNoConverter = rb_define_class_under(rb_cEncoding, "NoConverter", rb_eStandardError); transcoder_table = st_init_strcasetable(); sym_invalid = ID2SYM(rb_intern("invalid")); sym_undef = ID2SYM(rb_intern("undef")); sym_ignore = ID2SYM(rb_intern("ignore")); sym_replace = ID2SYM(rb_intern("replace")); sym_invalid_byte_sequence = ID2SYM(rb_intern("invalid_byte_sequence")); sym_undefined_conversion = ID2SYM(rb_intern("undefined_conversion")); sym_destination_buffer_full = ID2SYM(rb_intern("destination_buffer_full")); sym_source_buffer_empty = ID2SYM(rb_intern("source_buffer_empty")); sym_finished = ID2SYM(rb_intern("finished")); sym_output_followed_by_input = ID2SYM(rb_intern("output_followed_by_input")); sym_incomplete_input = ID2SYM(rb_intern("incomplete_input")); rb_define_method(rb_cString, "encode", str_encode, -1); rb_define_method(rb_cString, "encode!", str_encode_bang, -1); rb_cEncodingConverter = rb_define_class_under(rb_cEncoding, "Converter", rb_cData); rb_define_alloc_func(rb_cEncodingConverter, econv_s_allocate); rb_define_method(rb_cEncodingConverter, "initialize", econv_init, -1); rb_define_method(rb_cEncodingConverter, "inspect", econv_inspect, 0); rb_define_method(rb_cEncodingConverter, "source_encoding", econv_source_encoding, 0); rb_define_method(rb_cEncodingConverter, "destination_encoding", econv_destination_encoding, 0); rb_define_method(rb_cEncodingConverter, "primitive_convert", econv_primitive_convert, -1); rb_define_method(rb_cEncodingConverter, "convert", econv_convert, 1); rb_define_method(rb_cEncodingConverter, "finish", econv_finish, 0); rb_define_method(rb_cEncodingConverter, "primitive_errinfo", econv_primitive_errinfo, 0); rb_define_method(rb_cEncodingConverter, "insert_output", econv_insert_output, 1); rb_define_method(rb_cEncodingConverter, "putback", econv_putback, -1); rb_define_method(rb_cEncodingConverter, "last_error", econv_last_error, 0); rb_define_const(rb_cEncodingConverter, "INVALID_MASK", INT2FIX(ECONV_INVALID_MASK)); rb_define_const(rb_cEncodingConverter, "INVALID_IGNORE", INT2FIX(ECONV_INVALID_IGNORE)); rb_define_const(rb_cEncodingConverter, "INVALID_REPLACE", INT2FIX(ECONV_INVALID_REPLACE)); rb_define_const(rb_cEncodingConverter, "UNDEF_MASK", INT2FIX(ECONV_UNDEF_MASK)); rb_define_const(rb_cEncodingConverter, "UNDEF_IGNORE", INT2FIX(ECONV_UNDEF_IGNORE)); rb_define_const(rb_cEncodingConverter, "UNDEF_REPLACE", INT2FIX(ECONV_UNDEF_REPLACE)); rb_define_const(rb_cEncodingConverter, "PARTIAL_INPUT", INT2FIX(ECONV_PARTIAL_INPUT)); rb_define_const(rb_cEncodingConverter, "OUTPUT_FOLLOWED_BY_INPUT", INT2FIX(ECONV_OUTPUT_FOLLOWED_BY_INPUT)); rb_define_const(rb_cEncodingConverter, "UNIVERSAL_NEWLINE_DECODER", INT2FIX(ECONV_UNIVERSAL_NEWLINE_DECODER)); rb_define_const(rb_cEncodingConverter, "CRLF_NEWLINE_ENCODER", INT2FIX(ECONV_CRLF_NEWLINE_ENCODER)); rb_define_const(rb_cEncodingConverter, "CR_NEWLINE_ENCODER", INT2FIX(ECONV_CR_NEWLINE_ENCODER)); rb_define_method(rb_eConversionUndefined, "source_encoding_name", ecerr_source_encoding_name, 0); rb_define_method(rb_eConversionUndefined, "destination_encoding_name", ecerr_destination_encoding_name, 0); rb_define_method(rb_eConversionUndefined, "source_encoding", ecerr_source_encoding, 0); rb_define_method(rb_eConversionUndefined, "destination_encoding", ecerr_destination_encoding, 0); rb_define_method(rb_eConversionUndefined, "error_char", ecerr_error_char, 0); rb_define_method(rb_eInvalidByteSequence, "source_encoding_name", ecerr_source_encoding_name, 0); rb_define_method(rb_eInvalidByteSequence, "destination_encoding_name", ecerr_destination_encoding_name, 0); rb_define_method(rb_eInvalidByteSequence, "source_encoding", ecerr_source_encoding, 0); rb_define_method(rb_eInvalidByteSequence, "destination_encoding", ecerr_destination_encoding, 0); rb_define_method(rb_eInvalidByteSequence, "error_bytes", ecerr_error_bytes, 0); rb_define_method(rb_eInvalidByteSequence, "readagain_bytes", ecerr_readagain_bytes, 0); rb_define_method(rb_eInvalidByteSequence, "incomplete_input?", ecerr_incomplete_input, 0); Init_newline(); }