/********************************************************************** transcode.c - $Author$ created at: Tue Oct 30 16:10:22 JST 2007 Copyright (C) 2007 Martin Duerst **********************************************************************/ #include "ruby/ruby.h" #include "ruby/encoding.h" #define PType (int) #include "transcode_data.h" #include VALUE rb_eConversionUndefined; VALUE rb_eInvalidByteSequence; VALUE rb_cEncodingConverter; static VALUE sym_invalid, sym_undef, sym_ignore, sym_replace; #define INVALID_IGNORE 0x1 #define INVALID_REPLACE 0x2 #define UNDEF_IGNORE 0x10 #define UNDEF_REPLACE 0x20 #define PARTIAL_INPUT 0x100 #define UNIVERSAL_NEWLINE_DECODER 0x200 #define CRLF_NEWLINE_ENCODER 0x400 #define CR_NEWLINE_ENCODER 0x800 #define OUTPUT_FOLLOWED_BY_INPUT 0x1000 /* * Dispatch data and logic */ typedef struct { const char *from; const char *to; const char *lib; /* maybe null. it means that don't load the library. */ const rb_transcoder *transcoder; } transcoder_entry_t; static st_table *transcoder_table; static transcoder_entry_t * make_transcoder_entry(const char *from, const char *to) { st_data_t val; st_table *table2; if (!st_lookup(transcoder_table, (st_data_t)from, &val)) { val = (st_data_t)st_init_strcasetable(); st_add_direct(transcoder_table, (st_data_t)from, val); } table2 = (st_table *)val; if (!st_lookup(table2, (st_data_t)to, &val)) { transcoder_entry_t *entry = ALLOC(transcoder_entry_t); entry->from = from; entry->to = to; entry->lib = NULL; entry->transcoder = NULL; val = (st_data_t)entry; st_add_direct(table2, (st_data_t)to, val); } return (transcoder_entry_t *)val; } static transcoder_entry_t * get_transcoder_entry(const char *from, const char *to) { st_data_t val; st_table *table2; if (!st_lookup(transcoder_table, (st_data_t)from, &val)) { return NULL; } table2 = (st_table *)val; if (!st_lookup(table2, (st_data_t)to, &val)) { return NULL; } return (transcoder_entry_t *)val; } void rb_register_transcoder(const rb_transcoder *tr) { const char *const from_e = tr->from_encoding; const char *const to_e = tr->to_encoding; transcoder_entry_t *entry; entry = make_transcoder_entry(from_e, to_e); if (entry->transcoder) { rb_raise(rb_eArgError, "transcoder from %s to %s has been already registered", from_e, to_e); } entry->transcoder = tr; } static void declare_transcoder(const char *to, const char *from, const char *lib) { transcoder_entry_t *entry; entry = make_transcoder_entry(from, to); entry->lib = lib; } #define MAX_TRANSCODER_LIBNAME_LEN 64 static const char transcoder_lib_prefix[] = "enc/trans/"; void rb_declare_transcoder(const char *enc1, const char *enc2, const char *lib) { if (!lib || strlen(lib) > MAX_TRANSCODER_LIBNAME_LEN) { rb_raise(rb_eArgError, "invalid library name - %s", lib ? lib : "(null)"); } declare_transcoder(enc1, enc2, lib); declare_transcoder(enc2, enc1, lib); } #define encoding_equal(enc1, enc2) (STRCASECMP(enc1, enc2) == 0) typedef struct search_path_queue_tag { struct search_path_queue_tag *next; const char *enc; } search_path_queue_t; typedef struct { st_table *visited; search_path_queue_t *queue; search_path_queue_t **queue_last_ptr; const char *base_enc; } search_path_bfs_t; static int transcode_search_path_i(st_data_t key, st_data_t val, st_data_t arg) { const char *to = (const char *)key; search_path_bfs_t *bfs = (search_path_bfs_t *)arg; search_path_queue_t *q; if (st_lookup(bfs->visited, (st_data_t)to, &val)) { return ST_CONTINUE; } q = ALLOC(search_path_queue_t); q->enc = to; q->next = NULL; *bfs->queue_last_ptr = q; bfs->queue_last_ptr = &q->next; st_add_direct(bfs->visited, (st_data_t)to, (st_data_t)bfs->base_enc); return ST_CONTINUE; } static int transcode_search_path(const char *from, const char *to, void (*callback)(const char *from, const char *to, int depth, void *arg), void *arg) { search_path_bfs_t bfs; search_path_queue_t *q; st_data_t val; st_table *table2; int found; int pathlen; q = ALLOC(search_path_queue_t); q->enc = from; q->next = NULL; bfs.queue_last_ptr = &q->next; bfs.queue = q; bfs.visited = st_init_strcasetable(); st_add_direct(bfs.visited, (st_data_t)from, (st_data_t)NULL); while (bfs.queue) { q = bfs.queue; bfs.queue = q->next; if (!bfs.queue) bfs.queue_last_ptr = &bfs.queue; if (!st_lookup(transcoder_table, (st_data_t)q->enc, &val)) { xfree(q); continue; } table2 = (st_table *)val; if (st_lookup(table2, (st_data_t)to, &val)) { st_add_direct(bfs.visited, (st_data_t)to, (st_data_t)q->enc); xfree(q); found = 1; goto cleanup; } bfs.base_enc = q->enc; st_foreach(table2, transcode_search_path_i, (st_data_t)&bfs); bfs.base_enc = NULL; xfree(q); } found = 0; cleanup: while (bfs.queue) { q = bfs.queue; bfs.queue = q->next; xfree(q); } if (found) { const char *enc = to; int depth; pathlen = 0; while (1) { st_lookup(bfs.visited, (st_data_t)enc, &val); if (!val) break; pathlen++; enc = (const char *)val; } depth = pathlen; enc = to; while (1) { st_lookup(bfs.visited, (st_data_t)enc, &val); if (!val) break; callback((const char *)val, enc, --depth, arg); enc = (const char *)val; } } st_free_table(bfs.visited); if (found) return pathlen; else return -1; } static const rb_transcoder * load_transcoder_entry(transcoder_entry_t *entry) { if (entry->transcoder) return entry->transcoder; if (entry->lib) { const char *lib = entry->lib; int len = strlen(lib); char path[sizeof(transcoder_lib_prefix) + MAX_TRANSCODER_LIBNAME_LEN]; entry->lib = NULL; if (len > MAX_TRANSCODER_LIBNAME_LEN) return NULL; memcpy(path, transcoder_lib_prefix, sizeof(transcoder_lib_prefix) - 1); memcpy(path + sizeof(transcoder_lib_prefix) - 1, lib, len + 1); if (!rb_require(path)) return NULL; } if (entry->transcoder) return entry->transcoder; return NULL; } static const char* get_replacement_character(rb_encoding *enc, int *len_ret) { static rb_encoding *utf16be_encoding, *utf16le_encoding; static rb_encoding *utf32be_encoding, *utf32le_encoding; if (!utf16be_encoding) { utf16be_encoding = rb_enc_find("UTF-16BE"); utf16le_encoding = rb_enc_find("UTF-16LE"); utf32be_encoding = rb_enc_find("UTF-32BE"); utf32le_encoding = rb_enc_find("UTF-32LE"); } if (rb_utf8_encoding() == enc) { *len_ret = 3; return "\xEF\xBF\xBD"; } else if (utf16be_encoding == enc) { *len_ret = 2; return "\xFF\xFD"; } else if (utf16le_encoding == enc) { *len_ret = 2; return "\xFD\xFF"; } else if (utf32be_encoding == enc) { *len_ret = 4; return "\x00\x00\xFF\xFD"; } else if (utf32le_encoding == enc) { *len_ret = 4; return "\xFD\xFF\x00\x00"; } else { *len_ret = 1; return "?"; } } /* * Transcoding engine logic */ static const unsigned char * transcode_char_start(rb_transcoding *tc, const unsigned char *in_start, const unsigned char *inchar_start, const unsigned char *in_p, size_t *char_len_ptr) { const unsigned char *ptr; if (inchar_start - in_start < tc->recognized_len) { MEMCPY(TRANSCODING_READBUF(tc) + tc->recognized_len, inchar_start, unsigned char, in_p - inchar_start); ptr = TRANSCODING_READBUF(tc); } else { ptr = inchar_start - tc->recognized_len; } *char_len_ptr = tc->recognized_len + (in_p - inchar_start); return ptr; } static rb_econv_result_t transcode_restartable0(const unsigned char **in_pos, unsigned char **out_pos, const unsigned char *in_stop, unsigned char *out_stop, rb_transcoding *tc, const int opt) { const rb_transcoder *tr = tc->transcoder; int unitlen = tr->input_unit_length; int readagain_len = 0; const unsigned char *inchar_start; const unsigned char *in_p; unsigned char *out_p; unsigned char empty_buf; unsigned char *empty_ptr = &empty_buf; if (!in_pos) { in_pos = (const unsigned char **)&empty_ptr; in_stop = empty_ptr; } if (!out_pos) { out_pos = &empty_ptr; out_stop = empty_ptr; } in_p = inchar_start = *in_pos; out_p = *out_pos; #define SUSPEND(ret, num) \ do { \ tc->resume_position = (num); \ if (0 < in_p - inchar_start) \ MEMMOVE(TRANSCODING_READBUF(tc)+tc->recognized_len, \ inchar_start, unsigned char, in_p - inchar_start); \ *in_pos = in_p; \ *out_pos = out_p; \ tc->recognized_len += in_p - inchar_start; \ if (readagain_len) { \ tc->recognized_len -= readagain_len; \ tc->readagain_len = readagain_len; \ } \ return ret; \ resume_label ## num:; \ } while (0) #define SUSPEND_OBUF(num) \ do { \ while (out_stop - out_p < 1) { SUSPEND(transcode_destination_buffer_full, num); } \ } while (0) #define SUSPEND_OUTPUT_FOLLOWED_BY_INPUT(num) \ if ((opt & OUTPUT_FOLLOWED_BY_INPUT) && *out_pos != out_p) { \ SUSPEND(transcode_output_followed_by_input, num); \ } #define next_table (tc->next_table) #define next_info (tc->next_info) #define next_byte (tc->next_byte) #define writebuf_len (tc->writebuf_len) #define writebuf_off (tc->writebuf_off) switch (tc->resume_position) { case 0: break; case 1: goto resume_label1; case 2: goto resume_label2; case 3: goto resume_label3; case 4: goto resume_label4; case 5: goto resume_label5; case 6: goto resume_label6; case 7: goto resume_label7; case 8: goto resume_label8; case 9: goto resume_label9; case 10: goto resume_label10; case 11: goto resume_label11; case 12: goto resume_label12; case 13: goto resume_label13; case 14: goto resume_label14; case 15: goto resume_label15; case 16: goto resume_label16; case 17: goto resume_label17; case 18: goto resume_label18; case 19: goto resume_label19; case 20: goto resume_label20; case 21: goto resume_label21; case 22: goto resume_label22; case 23: goto resume_label23; case 24: goto resume_label24; case 25: goto resume_label25; case 26: goto resume_label26; } while (1) { SUSPEND_OUTPUT_FOLLOWED_BY_INPUT(24); if (in_stop <= in_p) { if (!(opt & PARTIAL_INPUT)) break; SUSPEND(transcode_source_buffer_empty, 7); continue; } tc->recognized_len = 0; inchar_start = in_p; next_table = tr->conv_tree_start; next_byte = (unsigned char)*in_p++; follow_byte: if (next_byte < next_table->base[0] || next_table->base[1] < next_byte) next_info = INVALID; else { unsigned int next_offset = next_table->base[2+next_byte-next_table->base[0]]; next_info = (VALUE)next_table->info[next_offset]; } follow_info: switch (next_info & 0x1F) { case NOMAP: /* xxx: copy last byte only? */ SUSPEND_OBUF(3); *out_p++ = next_byte; continue; case 0x00: case 0x04: case 0x08: case 0x0C: case 0x10: case 0x14: case 0x18: case 0x1C: SUSPEND_OUTPUT_FOLLOWED_BY_INPUT(25); while (in_p >= in_stop) { if (!(opt & PARTIAL_INPUT)) goto invalid; SUSPEND(transcode_source_buffer_empty, 5); } next_byte = (unsigned char)*in_p++; next_table = (const BYTE_LOOKUP *)next_info; goto follow_byte; case ZERObt: /* drop input */ continue; case ONEbt: SUSPEND_OBUF(9); *out_p++ = getBT1(next_info); continue; case TWObt: SUSPEND_OBUF(10); *out_p++ = getBT1(next_info); SUSPEND_OBUF(21); *out_p++ = getBT2(next_info); continue; case THREEbt: SUSPEND_OBUF(11); *out_p++ = getBT1(next_info); SUSPEND_OBUF(15); *out_p++ = getBT2(next_info); SUSPEND_OBUF(16); *out_p++ = getBT3(next_info); continue; case FOURbt: SUSPEND_OBUF(12); *out_p++ = getBT0(next_info); SUSPEND_OBUF(17); *out_p++ = getBT1(next_info); SUSPEND_OBUF(18); *out_p++ = getBT2(next_info); SUSPEND_OBUF(19); *out_p++ = getBT3(next_info); continue; case FUNii: next_info = (VALUE)(*tr->func_ii)(tc, next_info); goto follow_info; case FUNsi: { const unsigned char *char_start; size_t char_len; char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len); next_info = (VALUE)(*tr->func_si)(tc, char_start, (size_t)char_len); goto follow_info; } case FUNio: SUSPEND_OBUF(13); if (tr->max_output <= out_stop - out_p) out_p += (VALUE)(*tr->func_io)(tc, next_info, out_p); else { writebuf_len = (VALUE)(*tr->func_io)(tc, next_info, TRANSCODING_WRITEBUF(tc)); writebuf_off = 0; while (writebuf_off < writebuf_len) { SUSPEND_OBUF(20); *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++]; } } break; case FUNso: { const unsigned char *char_start; size_t char_len; SUSPEND_OBUF(14); if (tr->max_output <= out_stop - out_p) { char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len); out_p += (VALUE)(*tr->func_so)(tc, char_start, (size_t)char_len, out_p); } else { char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len); writebuf_len = (VALUE)(*tr->func_so)(tc, char_start, (size_t)char_len, TRANSCODING_WRITEBUF(tc)); writebuf_off = 0; while (writebuf_off < writebuf_len) { SUSPEND_OBUF(22); *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++]; } } break; } case INVALID: if (tc->recognized_len + (in_p - inchar_start) <= unitlen) { if (tc->recognized_len + (in_p - inchar_start) < unitlen) SUSPEND_OUTPUT_FOLLOWED_BY_INPUT(26); while ((opt & PARTIAL_INPUT) && tc->recognized_len + (in_stop - inchar_start) < unitlen) { in_p = in_stop; SUSPEND(transcode_source_buffer_empty, 8); } if (tc->recognized_len + (in_stop - inchar_start) <= unitlen) { in_p = in_stop; } else { in_p = inchar_start + (unitlen - tc->recognized_len); } } else { int invalid_len; /* including the last byte which causes invalid */ int discard_len; invalid_len = tc->recognized_len + (in_p - inchar_start); discard_len = ((invalid_len - 1) / unitlen) * unitlen; readagain_len = invalid_len - discard_len; } goto invalid; case UNDEF: goto undef; } continue; invalid: SUSPEND(transcode_invalid_byte_sequence, 1); continue; undef: SUSPEND(transcode_undefined_conversion, 2); continue; } /* cleanup */ if (tr->finish_func) { SUSPEND_OBUF(4); if (tr->max_output <= out_stop - out_p) { out_p += tr->finish_func(tc, out_p); } else { writebuf_len = tr->finish_func(tc, TRANSCODING_WRITEBUF(tc)); writebuf_off = 0; while (writebuf_off < writebuf_len) { SUSPEND_OBUF(23); *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++]; } } } while (1) SUSPEND(transcode_finished, 6); #undef SUSPEND #undef next_table #undef next_info #undef next_byte #undef writebuf_len #undef writebuf_off } static rb_econv_result_t transcode_restartable(const unsigned char **in_pos, unsigned char **out_pos, const unsigned char *in_stop, unsigned char *out_stop, rb_transcoding *tc, const int opt) { if (tc->readagain_len) { unsigned char *readagain_buf = ALLOCA_N(unsigned char, tc->readagain_len); const unsigned char *readagain_pos = readagain_buf; const unsigned char *readagain_stop = readagain_buf + tc->readagain_len; rb_econv_result_t res; MEMCPY(readagain_buf, TRANSCODING_READBUF(tc) + tc->recognized_len, unsigned char, tc->readagain_len); tc->readagain_len = 0; res = transcode_restartable0(&readagain_pos, out_pos, readagain_stop, out_stop, tc, opt|PARTIAL_INPUT); if (res != transcode_source_buffer_empty) { MEMCPY(TRANSCODING_READBUF(tc) + tc->recognized_len + tc->readagain_len, readagain_pos, unsigned char, readagain_stop - readagain_pos); tc->readagain_len += readagain_stop - readagain_pos; return res; } } return transcode_restartable0(in_pos, out_pos, in_stop, out_stop, tc, opt); } static rb_transcoding * rb_transcoding_open_by_transcoder(const rb_transcoder *tr, int flags) { rb_transcoding *tc; tc = ALLOC(rb_transcoding); tc->transcoder = tr; tc->flags = flags; memset(tc->stateful, 0, sizeof(tc->stateful)); tc->resume_position = 0; tc->recognized_len = 0; tc->readagain_len = 0; tc->writebuf_len = 0; tc->writebuf_off = 0; if (sizeof(tc->readbuf.ary) < tr->max_input) { tc->readbuf.ptr = xmalloc(tr->max_input); } if (sizeof(tc->writebuf.ary) < tr->max_output) { tc->writebuf.ptr = xmalloc(tr->max_output); } return tc; } static rb_econv_result_t rb_transcoding_convert(rb_transcoding *tc, const unsigned char **input_ptr, const unsigned char *input_stop, unsigned char **output_ptr, unsigned char *output_stop, int flags) { return transcode_restartable( input_ptr, output_ptr, input_stop, output_stop, tc, flags); } static void rb_transcoding_close(rb_transcoding *tc) { const rb_transcoder *tr = tc->transcoder; if (sizeof(tc->readbuf.ary) < tr->max_input) xfree(tc->readbuf.ptr); if (sizeof(tc->writebuf.ary) < tr->max_output) xfree(tc->writebuf.ptr); xfree(tc); } static rb_econv_t * rb_trans_open_by_transcoder_entries(int n, transcoder_entry_t **entries) { rb_econv_t *ts; int i; for (i = 0; i < n; i++) { const rb_transcoder *tr; tr = load_transcoder_entry(entries[i]); if (!tr) return NULL; } ts = ALLOC(rb_econv_t); ts->num_trans = n; ts->elems = ALLOC_N(rb_econv_elem_t, ts->num_trans); ts->num_finished = 0; ts->last_tc = NULL; for (i = 0; i < ts->num_trans; i++) { const rb_transcoder *tr = load_transcoder_entry(entries[i]); ts->elems[i].from = tr->from_encoding; ts->elems[i].to = tr->to_encoding; ts->elems[i].tc = rb_transcoding_open_by_transcoder(tr, 0); ts->elems[i].out_buf_start = NULL; ts->elems[i].out_data_start = NULL; ts->elems[i].out_data_end = NULL; ts->elems[i].out_buf_end = NULL; ts->elems[i].last_result = transcode_source_buffer_empty; } ts->last_tc = ts->elems[ts->num_trans-1].tc; for (i = 0; i < ts->num_trans-1; i++) { int bufsize = 4096; unsigned char *p; p = xmalloc(bufsize); ts->elems[i].out_buf_start = p; ts->elems[i].out_buf_end = p + bufsize; ts->elems[i].out_data_start = p; ts->elems[i].out_data_end = p; } return ts; } static void trans_open_i(const char *from, const char *to, int depth, void *arg) { transcoder_entry_t ***entries_ptr = arg; transcoder_entry_t **entries; if (!*entries_ptr) { entries = ALLOC_N(transcoder_entry_t *, depth+1+2); *entries_ptr = entries; } else { entries = *entries_ptr; } entries[depth] = get_transcoder_entry(from, to); } static rb_econv_t * rb_econv_open(const char *from, const char *to, int flags) { transcoder_entry_t **entries = NULL; int num_trans; static rb_econv_t *ts; num_trans = transcode_search_path(from, to, trans_open_i, (void *)&entries); if (num_trans < 0 || !entries) return NULL; if (flags & (CRLF_NEWLINE_ENCODER|CR_NEWLINE_ENCODER)) { const char *name = (flags & CRLF_NEWLINE_ENCODER) ? "crlf_newline" : "cr_newline"; transcoder_entry_t *e = get_transcoder_entry("", name); if (!e) return NULL; MEMMOVE(entries+1, entries, transcoder_entry_t *, num_trans); entries[0] = e; num_trans++; } if (flags & UNIVERSAL_NEWLINE_DECODER) { transcoder_entry_t *e = get_transcoder_entry("universal_newline", ""); if (!e) return NULL; entries[num_trans++] = e; } ts = rb_trans_open_by_transcoder_entries(num_trans, entries); if (flags & UNIVERSAL_NEWLINE_DECODER) { ts->last_tc = ts->elems[ts->num_trans-2].tc; } return ts; } static int trans_sweep(rb_econv_t *ts, const unsigned char **input_ptr, const unsigned char *input_stop, unsigned char **output_ptr, unsigned char *output_stop, int flags, int start) { int try; int i, f; const unsigned char **ipp, *is, *iold; unsigned char **opp, *os, *oold; rb_econv_result_t res; try = 1; while (try) { try = 0; for (i = start; i < ts->num_trans; i++) { rb_econv_elem_t *te = &ts->elems[i]; if (i == 0) { ipp = input_ptr; is = input_stop; } else { rb_econv_elem_t *prev_te = &ts->elems[i-1]; ipp = (const unsigned char **)&prev_te->out_data_start; is = prev_te->out_data_end; } if (!te->out_buf_start) { opp = output_ptr; os = output_stop; } else { if (te->out_buf_start != te->out_data_start) { int len = te->out_data_end - te->out_data_start; int off = te->out_data_start - te->out_buf_start; MEMMOVE(te->out_buf_start, te->out_data_start, unsigned char, len); te->out_data_start = te->out_buf_start; te->out_data_end -= off; } opp = &te->out_data_end; os = te->out_buf_end; } f = flags; if (ts->num_finished != i) f |= PARTIAL_INPUT; if (i == 0 && (flags & OUTPUT_FOLLOWED_BY_INPUT)) { start = 1; flags &= ~OUTPUT_FOLLOWED_BY_INPUT; } if (i != 0) f &= ~OUTPUT_FOLLOWED_BY_INPUT; iold = *ipp; oold = *opp; te->last_result = res = rb_transcoding_convert(te->tc, ipp, is, opp, os, f); if (iold != *ipp || oold != *opp) try = 1; switch (res) { case transcode_invalid_byte_sequence: case transcode_undefined_conversion: case transcode_output_followed_by_input: return i; case transcode_destination_buffer_full: case transcode_source_buffer_empty: break; case transcode_finished: ts->num_finished = i+1; break; } } } return -1; } static rb_econv_result_t rb_trans_conv(rb_econv_t *ts, const unsigned char **input_ptr, const unsigned char *input_stop, unsigned char **output_ptr, unsigned char *output_stop, int flags) { int i; int needreport_index; int sweep_start; unsigned char empty_buf; unsigned char *empty_ptr = &empty_buf; if (!input_ptr) { input_ptr = (const unsigned char **)&empty_ptr; input_stop = empty_ptr; } if (!output_ptr) { output_ptr = &empty_ptr; output_stop = empty_ptr; } if (ts->elems[0].last_result == transcode_output_followed_by_input) ts->elems[0].last_result = transcode_source_buffer_empty; needreport_index = -1; for (i = ts->num_trans-1; 0 <= i; i--) { switch (ts->elems[i].last_result) { case transcode_invalid_byte_sequence: case transcode_undefined_conversion: case transcode_output_followed_by_input: case transcode_finished: sweep_start = i+1; needreport_index = i; goto found_needreport; case transcode_destination_buffer_full: case transcode_source_buffer_empty: break; default: rb_bug("unexpected transcode last result"); } } /* /^[io]+$/ is confirmed. but actually /^i*o*$/. */ if (ts->elems[ts->num_trans-1].last_result == transcode_destination_buffer_full && (flags & OUTPUT_FOLLOWED_BY_INPUT)) { rb_econv_result_t res; res = rb_trans_conv(ts, NULL, NULL, output_ptr, output_stop, (flags & ~OUTPUT_FOLLOWED_BY_INPUT)|PARTIAL_INPUT); if (res == transcode_source_buffer_empty) return transcode_output_followed_by_input; return res; } sweep_start = 0; found_needreport: do { needreport_index = trans_sweep(ts, input_ptr, input_stop, output_ptr, output_stop, flags, sweep_start); sweep_start = needreport_index + 1; } while (needreport_index != -1 && needreport_index != ts->num_trans-1); for (i = ts->num_trans-1; 0 <= i; i--) { if (ts->elems[i].last_result != transcode_source_buffer_empty) { rb_econv_result_t res = ts->elems[i].last_result; if (res == transcode_invalid_byte_sequence || res == transcode_undefined_conversion || res == transcode_output_followed_by_input) { ts->elems[i].last_result = transcode_source_buffer_empty; } return res; } } return transcode_source_buffer_empty; } static rb_econv_result_t rb_econv_conv(rb_econv_t *ts, const unsigned char **input_ptr, const unsigned char *input_stop, unsigned char **output_ptr, unsigned char *output_stop, int flags) { rb_econv_result_t res; if ((flags & OUTPUT_FOLLOWED_BY_INPUT) || ts->num_trans == 1) return rb_trans_conv(ts, input_ptr, input_stop, output_ptr, output_stop, flags); flags |= OUTPUT_FOLLOWED_BY_INPUT; do { res = rb_trans_conv(ts, input_ptr, input_stop, output_ptr, output_stop, flags); } while (res == transcode_output_followed_by_input); return res; } static void rb_econv_close(rb_econv_t *ts) { int i; for (i = 0; i < ts->num_trans; i++) { rb_transcoding_close(ts->elems[i].tc); if (ts->elems[i].out_buf_start) xfree(ts->elems[i].out_buf_start); } xfree(ts->elems); xfree(ts); } static void more_output_buffer( VALUE destination, unsigned char *(*resize_destination)(VALUE, int, int), int max_output, unsigned char **out_start_ptr, unsigned char **out_pos, unsigned char **out_stop_ptr) { size_t len = (*out_pos - *out_start_ptr); size_t new_len = (len + max_output) * 2; *out_start_ptr = resize_destination(destination, len, new_len); *out_pos = *out_start_ptr + len; *out_stop_ptr = *out_start_ptr + new_len; } static void output_replacement_character( VALUE destination, unsigned char *(*resize_destination)(VALUE, int, int), rb_transcoding *tc, unsigned char **out_start_ptr, unsigned char **out_pos, unsigned char **out_stop_ptr) { const rb_transcoder *tr; int max_output; rb_encoding *enc; const char *replacement; int len; tr = tc->transcoder; max_output = tr->max_output; enc = rb_enc_find(tr->to_encoding); /* * Assumption for stateful encoding: * * - The replacement character can be output on resetted state and doesn't * change the state. * - it is acceptable that extra state changing sequence if the replacement * character contains a state changing sequence. * * Currently the replacement character for stateful encoding such as * ISO-2022-JP is "?" and it has no state changing sequence. * So the extra state changing sequence don't occur. * * Thease assumption may be removed in future. * It needs to scan the replacement character to check * state changing sequences in the replacement character. */ if (tr->resetstate_func) { if (*out_stop_ptr - *out_pos < max_output) more_output_buffer(destination, resize_destination, max_output, out_start_ptr, out_pos, out_stop_ptr); *out_pos += tr->resetstate_func(tc, *out_pos); } if (*out_stop_ptr - *out_pos < max_output) more_output_buffer(destination, resize_destination, max_output, out_start_ptr, out_pos, out_stop_ptr); replacement = get_replacement_character(enc, &len); memcpy(*out_pos, replacement, len); *out_pos += len; return; } #if 1 static void transcode_loop(const unsigned char **in_pos, unsigned char **out_pos, const unsigned char *in_stop, unsigned char *out_stop, VALUE destination, unsigned char *(*resize_destination)(VALUE, int, int), const char *from_encoding, const char *to_encoding, const int opt) { rb_econv_t *ts; rb_transcoding *last_tc; rb_econv_result_t ret; unsigned char *out_start = *out_pos; int max_output; ts = rb_econv_open(from_encoding, to_encoding, 0); if (!ts) rb_raise(rb_eArgError, "transcoding not supported (from %s to %s)", from_encoding, to_encoding); last_tc = ts->last_tc; max_output = last_tc->transcoder->max_output; resume: ret = rb_econv_conv(ts, in_pos, in_stop, out_pos, out_stop, opt); if (ret == transcode_invalid_byte_sequence) { /* deal with invalid byte sequence */ /* todo: add more alternative behaviors */ if (opt&INVALID_IGNORE) { goto resume; } else if (opt&INVALID_REPLACE) { output_replacement_character(destination, resize_destination, last_tc, &out_start, out_pos, &out_stop); goto resume; } rb_econv_close(ts); rb_raise(rb_eInvalidByteSequence, "invalid byte sequence"); } if (ret == transcode_undefined_conversion) { /* valid character in from encoding * but no related character(s) in to encoding */ /* todo: add more alternative behaviors */ if (opt&UNDEF_IGNORE) { goto resume; } else if (opt&UNDEF_REPLACE) { output_replacement_character(destination, resize_destination, last_tc, &out_start, out_pos, &out_stop); goto resume; } rb_econv_close(ts); rb_raise(rb_eConversionUndefined, "conversion undefined for byte sequence (maybe invalid byte sequence)"); } if (ret == transcode_destination_buffer_full) { more_output_buffer(destination, resize_destination, max_output, &out_start, out_pos, &out_stop); goto resume; } rb_econv_close(ts); return; } #else /* sample transcode_loop implementation in byte-by-byte stream style */ static void transcode_loop(const unsigned char **in_pos, unsigned char **out_pos, const unsigned char *in_stop, unsigned char *out_stop, VALUE destination, unsigned char *(*resize_destination)(VALUE, int, int), const char *from_encoding, const char *to_encoding, const int opt) { rb_econv_t *ts; rb_transcoding *last_tc; rb_econv_result_t ret; unsigned char *out_start = *out_pos; const unsigned char *ptr; int max_output; ts = rb_econv_open(from_encoding, to_encoding, 0); if (!ts) rb_raise(rb_eArgError, "transcoding not supported (from %s to %s)", from_encoding, to_encoding); last_tc = ts->last_tc; max_output = ts->elems[ts->num_trans-1].tc->transcoder->max_output; ret = transcode_source_buffer_empty; ptr = *in_pos; while (ret != transcode_finished) { unsigned char input_byte; const unsigned char *p = &input_byte; if (ret == transcode_source_buffer_empty) { if (ptr < in_stop) { input_byte = *ptr; ret = rb_econv_conv(ts, &p, p+1, out_pos, out_stop, PARTIAL_INPUT); } else { ret = rb_econv_conv(ts, NULL, NULL, out_pos, out_stop, 0); } } else { ret = rb_econv_conv(ts, NULL, NULL, out_pos, out_stop, PARTIAL_INPUT); } if (&input_byte != p) ptr += p - &input_byte; switch (ret) { case transcode_invalid_byte_sequence: /* deal with invalid byte sequence */ /* todo: add more alternative behaviors */ if (opt&INVALID_IGNORE) { break; } else if (opt&INVALID_REPLACE) { output_replacement_character(destination, resize_destination, last_tc, &out_start, out_pos, &out_stop); break; } rb_econv_close(ts); rb_raise(rb_eInvalidByteSequence, "invalid byte sequence"); break; case transcode_undefined_conversion: /* valid character in from encoding * but no related character(s) in to encoding */ /* todo: add more alternative behaviors */ if (opt&UNDEF_IGNORE) { break; } else if (opt&UNDEF_REPLACE) { output_replacement_character(destination, resize_destination, last_tc, &out_start, out_pos, &out_stop); break; } rb_econv_close(ts); rb_raise(rb_eConversionUndefined, "conversion undefined for byte sequence (maybe invalid byte sequence)"); break; case transcode_destination_buffer_full: more_output_buffer(destination, resize_destination, max_output, &out_start, out_pos, &out_stop); break; case transcode_source_buffer_empty: break; case transcode_finished: break; } } rb_econv_close(ts); *in_pos = in_stop; return; } #endif /* * String-specific code */ static unsigned char * str_transcoding_resize(VALUE destination, int len, int new_len) { rb_str_resize(destination, new_len); return (unsigned char *)RSTRING_PTR(destination); } static int str_transcode(int argc, VALUE *argv, VALUE *self) { VALUE dest; VALUE str = *self; long blen, slen; unsigned char *buf, *bp, *sp; const unsigned char *fromp; rb_encoding *from_enc, *to_enc; const char *from_e, *to_e; int from_encidx, to_encidx; VALUE from_encval, to_encval; VALUE opt; int options = 0; opt = rb_check_convert_type(argv[argc-1], T_HASH, "Hash", "to_hash"); if (!NIL_P(opt)) { VALUE v; argc--; v = rb_hash_aref(opt, sym_invalid); if (NIL_P(v)) { } else if (v==sym_ignore) { options |= INVALID_IGNORE; } else if (v==sym_replace) { options |= INVALID_REPLACE; v = rb_hash_aref(opt, sym_replace); } else { rb_raise(rb_eArgError, "unknown value for invalid character option"); } v = rb_hash_aref(opt, sym_undef); if (NIL_P(v)) { } else if (v==sym_ignore) { options |= UNDEF_IGNORE; } else if (v==sym_replace) { options |= UNDEF_REPLACE; } else { rb_raise(rb_eArgError, "unknown value for undefined character option"); } } if (argc < 1 || argc > 2) { rb_raise(rb_eArgError, "wrong number of arguments (%d for 1..2)", argc); } if ((to_encidx = rb_to_encoding_index(to_encval = argv[0])) < 0) { to_enc = 0; to_encidx = 0; to_e = StringValueCStr(to_encval); } else { to_enc = rb_enc_from_index(to_encidx); to_e = rb_enc_name(to_enc); } if (argc==1) { from_encidx = rb_enc_get_index(str); from_enc = rb_enc_from_index(from_encidx); from_e = rb_enc_name(from_enc); } else if ((from_encidx = rb_to_encoding_index(from_encval = argv[1])) < 0) { from_enc = 0; from_e = StringValueCStr(from_encval); } else { from_enc = rb_enc_from_index(from_encidx); from_e = rb_enc_name(from_enc); } if (from_enc && from_enc == to_enc) { return -1; } if (from_enc && to_enc && rb_enc_asciicompat(from_enc) && rb_enc_asciicompat(to_enc)) { if (ENC_CODERANGE(str) == ENC_CODERANGE_7BIT) { return to_encidx; } } if (encoding_equal(from_e, to_e)) { return -1; } fromp = sp = (unsigned char *)RSTRING_PTR(str); slen = RSTRING_LEN(str); blen = slen + 30; /* len + margin */ dest = rb_str_tmp_new(blen); bp = (unsigned char *)RSTRING_PTR(dest); transcode_loop(&fromp, &bp, (sp+slen), (bp+blen), dest, str_transcoding_resize, from_e, to_e, options); if (fromp != sp+slen) { rb_raise(rb_eArgError, "not fully converted, %"PRIdPTRDIFF" bytes left", sp+slen-fromp); } buf = (unsigned char *)RSTRING_PTR(dest); *bp = '\0'; rb_str_set_len(dest, bp - buf); /* set encoding */ if (!to_enc) { to_encidx = rb_define_dummy_encoding(to_e); } *self = dest; return to_encidx; } static inline VALUE str_encode_associate(VALUE str, int encidx) { int cr = 0; rb_enc_associate_index(str, encidx); /* transcoded string never be broken. */ if (rb_enc_asciicompat(rb_enc_from_index(encidx))) { rb_str_coderange_scan_restartable(RSTRING_PTR(str), RSTRING_END(str), 0, &cr); } else { cr = ENC_CODERANGE_VALID; } ENC_CODERANGE_SET(str, cr); return str; } /* * call-seq: * str.encode!(encoding [, options] ) => str * str.encode!(to_encoding, from_encoding [, options] ) => str * * The first form transcodes the contents of str from * str.encoding to +encoding+. * The second form transcodes the contents of str from * from_encoding to to_encoding. * The options Hash gives details for conversion. See String#encode * for details. * Returns the string even if no changes were made. */ static VALUE str_encode_bang(int argc, VALUE *argv, VALUE str) { VALUE newstr = str; int encidx = str_transcode(argc, argv, &newstr); if (encidx < 0) return str; rb_str_shared_replace(str, newstr); return str_encode_associate(str, encidx); } /* * call-seq: * str.encode(encoding [, options] ) => str * str.encode(to_encoding, from_encoding [, options] ) => str * * The first form returns a copy of str transcoded * to encoding +encoding+. * The second form returns a copy of str transcoded * from from_encoding to to_encoding. * The options Hash gives details for conversion. Details * to be added. */ static VALUE str_encode(int argc, VALUE *argv, VALUE str) { VALUE newstr = str; int encidx = str_transcode(argc, argv, &newstr); if (encidx < 0) return rb_str_dup(str); RBASIC(newstr)->klass = rb_obj_class(str); return str_encode_associate(newstr, encidx); } VALUE rb_str_transcode(VALUE str, VALUE to) { return str_encode(1, &to, str); } static void econv_free(rb_econv_t *ts) { rb_econv_close(ts); } static VALUE econv_s_allocate(VALUE klass) { return Data_Wrap_Struct(klass, NULL, econv_free, NULL); } /* * call-seq: * Encoding::Converter.new(source_encoding, destination_encoding) * Encoding::Converter.new(source_encoding, destination_encoding, flags) * * possible flags: * Encoding::Converter::UNIVERSAL_NEWLINE_DECODER # convert CRLF and CR to LF at last * Encoding::Converter::CRLF_NEWLINE_ENCODER # convert LF to CRLF at first * Encoding::Converter::CR_NEWLINE_ENCODER # convert LF to CR at first * * Encoding::Converter.new creates an instance of Encoding::Converter. * * source_encoding and destination_encoding should be a string. * flags should be an integer. * * example: * # UTF-16BE to UTF-8 * ec = Encoding::Converter.new("UTF-16BE", "UTF-8") * * # (1) convert UTF-16BE to UTF-8 * # (2) convert CRLF and CR to LF * ec = Encoding::Converter.new("UTF-16BE", "UTF-8", Encoding::Converter::UNIVERSAL_NEWLINE_DECODER) * * # (1) convert LF to CRLF * # (2) convert UTF-8 to UTF-16BE * ec = Encoding::Converter.new("UTF-8", "UTF-16BE", Encoding::Converter::CRLF_NEWLINE_ENCODER) * */ static VALUE econv_init(int argc, VALUE *argv, VALUE self) { VALUE from_encoding, to_encoding, flags_v; const char *from_e, *to_e; rb_econv_t *ts; int flags; rb_scan_args(argc, argv, "21", &from_encoding, &to_encoding, &flags_v); StringValue(from_encoding); StringValue(to_encoding); if (flags_v == Qnil) flags = 0; else flags = NUM2INT(flags_v); from_e = RSTRING_PTR(from_encoding); to_e = RSTRING_PTR(to_encoding); if (DATA_PTR(self)) { rb_raise(rb_eTypeError, "already initialized"); } ts = rb_econv_open(from_e, to_e, flags); if (!ts) { rb_raise(rb_eArgError, "encoding convewrter not supported (from %s to %s)", from_e, to_e); } DATA_PTR(self) = ts; return self; } static VALUE econv_inspect(VALUE self) { const char *cname = rb_obj_classname(self); rb_econv_t *ts = DATA_PTR(self); if (!ts) return rb_sprintf("#<%s: uninitialized>", cname); else return rb_sprintf("#<%s: %s to %s>", cname, ts->elems[0].from, ts->last_tc->transcoder->to_encoding); } #define IS_ECONV(obj) (RDATA(obj)->dfree == (RUBY_DATA_FUNC)econv_free) static rb_econv_t * check_econv(VALUE self) { Check_Type(self, T_DATA); if (!IS_ECONV(self)) { rb_raise(rb_eTypeError, "wrong argument type %s (expected Encoding::Converter)", rb_class2name(CLASS_OF(self))); } if (!DATA_PTR(self)) { rb_raise(rb_eTypeError, "uninitialized encoding converter"); } return DATA_PTR(self); } /* * call-seq: * primitive_convert(source_buffer, destination_buffer, destination_byteoffset, destination_bytesize) -> symbol * primitive_convert(source_buffer, destination_buffer, destination_byteoffset, destination_bytesize, flags) -> symbol * * possible flags: * Encoding::Converter::PARTIAL_INPUT # source buffer may be part of larger source * Encoding::Converter::OUTPUT_FOLLOWED_BY_INPUT # stop conversion after output before input * * possible results: * :invalid_byte_sequence * :undefined_conversion * :output_followed_by_input * :destination_buffer_full * :source_buffer_empty * :finished * * primitive_convert converts source_buffer into destination_buffer. * * source_buffer and destination_buffer should be a string. * destination_byteoffset should be an integer or nil. * destination_bytesize and flags should be an integer. * * primitive_convert convert the content of source_buffer from beginning * and store the result into destination_buffer. * * destination_byteoffset and destination_bytesize specify the region which * the converted result is stored. * destination_byteoffset specifies the start position in destination_buffer in bytes. * If destination_byteoffset is nil, * destination_buffer.bytesize is used for appending the result. * destination_bytesize specifies maximum number of bytes. * After conversion, destination_buffer is resized to * destination_byteoffset + actually converted number of bytes. * * primitive_convert drops the first part of source_buffer. * the dropped part is converted in destination_buffer or * buffered in Encoding::Converter object. * * primitive_convert stops conversion when one of following condition met. * - invalid byte sequence found in source buffer (:invalid_byte_sequence) * - character not representable in output encoding (:undefined_conversion) * - after some output is generated, before input is done (:output_followed_by_input) * this occur only when OUTPUT_FOLLOWED_BY_INPUT is specified. * - destination buffer is full (:destination_buffer_full) * - source buffer is empty (:source_buffer_empty) * this occur only when PARTIAL_INPUT is specified. * - conversion is finished (:finished) * * example: * ec = Encoding::Converter.new("UTF-8", "UTF-16BE") * ret = ec.primitive_convert(src="pi", dst="", 100) * p [ret, src, dst] #=> [:finished, "", "\x00p\x00i"] * * ec = Encoding::Converter.new("UTF-8", "UTF-16BE") * ret = ec.primitive_convert(src="pi", dst="", 1) * p [ret, src, dst] #=> [:destination_buffer_full, "i", "\x00"] * ret = ec.primitive_convert(src, dst="", 1) * p [ret, src, dst] #=> [:destination_buffer_full, "", "p"] * ret = ec.primitive_convert(src, dst="", 1) * p [ret, src, dst] #=> [:destination_buffer_full, "", "\x00"] * ret = ec.primitive_convert(src, dst="", 1) * p [ret, src, dst] #=> [:finished, "", "i"] * */ static VALUE econv_primitive_convert(int argc, VALUE *argv, VALUE self) { VALUE input, output, output_byteoffset_v, output_bytesize_v, flags_v; rb_econv_t *ts = check_econv(self); rb_econv_result_t res; const unsigned char *ip, *is; unsigned char *op, *os; long output_byteoffset, output_bytesize; unsigned long output_byteend; int flags; rb_scan_args(argc, argv, "41", &input, &output, &output_byteoffset_v, &output_bytesize_v, &flags_v); if (output_byteoffset_v == Qnil) output_byteoffset = 0; else output_byteoffset = NUM2LONG(output_byteoffset_v); output_bytesize = NUM2LONG(output_bytesize_v); if (flags_v == Qnil) flags = 0; else flags = NUM2INT(flags_v); StringValue(output); StringValue(input); rb_str_modify(output); if (output_byteoffset_v == Qnil) output_byteoffset = RSTRING_LEN(output); if (output_byteoffset < 0) rb_raise(rb_eArgError, "negative output_byteoffset"); if (RSTRING_LEN(output) < output_byteoffset) rb_raise(rb_eArgError, "output_byteoffset too big"); if (output_bytesize < 0) rb_raise(rb_eArgError, "negative output_bytesize"); output_byteend = (unsigned long)output_byteoffset + (unsigned long)output_bytesize; if (output_byteend < (unsigned long)output_byteoffset || LONG_MAX < output_byteend) rb_raise(rb_eArgError, "output_byteoffset+output_bytesize too big"); if (rb_str_capacity(output) < output_byteend) rb_str_resize(output, output_byteend); ip = (const unsigned char *)RSTRING_PTR(input); is = ip + RSTRING_LEN(input); op = (unsigned char *)RSTRING_PTR(output) + output_byteoffset; os = op + output_bytesize; res = rb_econv_conv(ts, &ip, is, &op, os, flags); rb_str_set_len(output, op-(unsigned char *)RSTRING_PTR(output)); rb_str_drop_bytes(input, ip - (unsigned char *)RSTRING_PTR(input)); switch (res) { case transcode_invalid_byte_sequence: return ID2SYM(rb_intern("invalid_byte_sequence")); case transcode_undefined_conversion: return ID2SYM(rb_intern("undefined_conversion")); case transcode_destination_buffer_full: return ID2SYM(rb_intern("destination_buffer_full")); case transcode_source_buffer_empty: return ID2SYM(rb_intern("source_buffer_empty")); case transcode_finished: return ID2SYM(rb_intern("finished")); case transcode_output_followed_by_input: return ID2SYM(rb_intern("output_followed_by_input")); default: return INT2NUM(res); /* should not be reached */ } } /* * call-seq: * max_output -> int * * returns the maximum length of output unit in bytes. */ static VALUE econv_max_output(VALUE self) { rb_econv_t *ts = check_econv(self); int n; n = ts->elems[ts->num_trans-1].tc->transcoder->max_output; return INT2FIX(n); } void Init_transcode(void) { rb_eConversionUndefined = rb_define_class_under(rb_cEncoding, "ConversionUndefined", rb_eStandardError); rb_eInvalidByteSequence = rb_define_class_under(rb_cEncoding, "InvalidByteSequence", rb_eStandardError); transcoder_table = st_init_strcasetable(); sym_invalid = ID2SYM(rb_intern("invalid")); sym_undef = ID2SYM(rb_intern("undef")); sym_ignore = ID2SYM(rb_intern("ignore")); sym_replace = ID2SYM(rb_intern("replace")); rb_define_method(rb_cString, "encode", str_encode, -1); rb_define_method(rb_cString, "encode!", str_encode_bang, -1); rb_cEncodingConverter = rb_define_class_under(rb_cEncoding, "Converter", rb_cData); rb_define_alloc_func(rb_cEncodingConverter, econv_s_allocate); rb_define_method(rb_cEncodingConverter, "initialize", econv_init, -1); rb_define_method(rb_cEncodingConverter, "inspect", econv_inspect, 0); rb_define_method(rb_cEncodingConverter, "primitive_convert", econv_primitive_convert, -1); rb_define_method(rb_cEncodingConverter, "max_output", econv_max_output, 0); rb_define_const(rb_cEncodingConverter, "PARTIAL_INPUT", INT2FIX(PARTIAL_INPUT)); rb_define_const(rb_cEncodingConverter, "OUTPUT_FOLLOWED_BY_INPUT", INT2FIX(OUTPUT_FOLLOWED_BY_INPUT)); rb_define_const(rb_cEncodingConverter, "UNIVERSAL_NEWLINE_DECODER", INT2FIX(UNIVERSAL_NEWLINE_DECODER)); rb_define_const(rb_cEncodingConverter, "CRLF_NEWLINE_ENCODER", INT2FIX(CRLF_NEWLINE_ENCODER)); rb_define_const(rb_cEncodingConverter, "CR_NEWLINE_ENCODER", INT2FIX(CR_NEWLINE_ENCODER)); }