compile: translate iseq in-place

running "ruby -rpp -e 'pp GC.stat'", a reduction in
malloc usage is shown:

before:
	:malloc_increase=>118784,
	:oldmalloc_increase=>1178736,

after:
	:malloc_increase=>99832,
	:oldmalloc_increase=>1031976,

For "ruby -e exit", valgrind reports over 300K reduction in
overall allocations (and unnecessary memory copies).

before:
  total heap usage: 49,622 allocs, 20,492 frees, 8,697,493 bytes allocated
after:
  total heap usage: 48,935 allocs, 19,805 frees, 8,373,773 bytes allocated

(numbers from x86-64)

v2 changes based on ko1 recommendations [ruby-core:64883]:
- squashed in-place direct thread translation to avoid alloc+copy
- renamed rb_iseq_untranslate_threaded_code to rb_iseq_original_iseq,
  cache new iseq->iseq_original field.

* compile.c (rb_iseq_translate_threaded_code): modify in-place w/o copy
  (rb_vm_addr2insn): new function for debug
  (rb_iseq_original_iseq): ditto
  (iseq_set_sequence): assign iseq_encoded directly
  [Feature #10185]

* vm_core (rb_iseq_t): move original ->iseq to bottom

* iseq.c (iseq_free, iseq_free): adjust for new layout
  (rb_iseq_disasm): use original iseq for dump
  (iseq_data_to_ary): ditto
  (rb_iseq_line_trace_each): ditto
  (rb_iseq_build_for_ruby2cext): use iseq_encoded directly

* vm_dump.c (rb_vmdebug_debug_print_pre): use original iseq

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@47508 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

6 files changed, 90 insertions, 30 deletions

diff --git a/ChangeLog b/ChangeLog
index 3885e8598e..ac0f6662bb 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,22 @@
+Wed Sep 10 15:07:35 2014  Eric Wong  <e@80x24.org>
+
+	* compile.c (rb_iseq_translate_threaded_code):
+	  modify in-place w/o copy
+	  (rb_vm_addr2insn): new function for debug
+	  (rb_iseq_original_iseq): ditto
+	  (iseq_set_sequence): assign iseq_encoded directly
+	  [Feature #10185]
+
+	* vm_core (rb_iseq_t): move original ->iseq to bottom
+
+	* iseq.c (iseq_free, iseq_free): adjust for new layout
+	  (rb_iseq_disasm): use original iseq for dump
+	  (iseq_data_to_ary): ditto
+	  (rb_iseq_line_trace_each): ditto
+	  (rb_iseq_build_for_ruby2cext): use iseq_encoded directly
+
+	* vm_dump.c (rb_vmdebug_debug_print_pre): use original iseq
+
 Wed Sep 10 15:00:11 2014  Eric Wong  <e@80x24.org>
 
 	* time.c (time_mark): remove NULL check
diff --git a/compile.c b/compile.c
index 434d385aa8..5b778e0ccf 100644
--- a/compile.c
+++ b/compile.c
@@ -569,21 +569,56 @@ rb_iseq_translate_threaded_code(rb_iseq_t *iseq)
     const void * const *table = rb_vm_get_insns_address_table();
     unsigned int i;
 
-    iseq->iseq_encoded = ALLOC_N(VALUE, iseq->iseq_size);
-    MEMCPY(iseq->iseq_encoded, iseq->iseq, VALUE, iseq->iseq_size);
-
     for (i = 0; i < iseq->iseq_size; /* */ ) {
 	int insn = (int)iseq->iseq_encoded[i];
 	int len = insn_len(insn);
 	iseq->iseq_encoded[i] = (VALUE)table[insn];
 	i += len;
     }
-#else
-    iseq->iseq_encoded = iseq->iseq;
 #endif
     return COMPILE_OK;
 }
 
+#if OPT_DIRECT_THREADED_CODE || OPT_CALL_THREADED_CODE
+static int
+rb_vm_addr2insn(const void *addr) /* cold path */
+{
+    int insn;
+    const void * const *table = rb_vm_get_insns_address_table();
+
+    for (insn = 0; insn < VM_INSTRUCTION_SIZE; insn++) {
+	if (table[insn] == addr)
+	    return insn;
+    }
+    rb_bug("rb_vm_addr2insn: invalid insn address: %p", addr);
+}
+#endif
+
+VALUE *
+rb_iseq_original_iseq(rb_iseq_t *iseq) /* cold path */
+{
+    if (iseq->iseq) return iseq->iseq;
+
+    iseq->iseq = ALLOC_N(VALUE, iseq->iseq_size);
+
+    MEMCPY(iseq->iseq, iseq->iseq_encoded, VALUE, iseq->iseq_size);
+
+#if OPT_DIRECT_THREADED_CODE || OPT_CALL_THREADED_CODE
+    {
+	unsigned int i;
+
+	for (i = 0; i < iseq->iseq_size; /* */ ) {
+	    const void *addr = (const void *)iseq->iseq[i];
+	    int insn = (VALUE)rb_vm_addr2insn(addr);
+
+	    iseq->iseq[i] = insn;
+	    i += insn_len(insn);
+	}
+    }
+#endif
+    return iseq->iseq;
+}
+
 /*********************************************/
 /* definition of data structure for compiler */
 /*********************************************/
@@ -1645,7 +1680,7 @@ iseq_set_sequence(rb_iseq_t *iseq, LINK_ANCHOR *anchor)
     }
 #endif
 
-    iseq->iseq = (void *)generated_iseq;
+    iseq->iseq_encoded = (void *)generated_iseq;
     iseq->iseq_size = pos;
     iseq->stack_max = stack_max;
 
diff --git a/iseq.c b/iseq.c
index dfc90cc3de..b6bdc261b4 100644
--- a/iseq.c
+++ b/iseq.c
@@ -75,11 +75,7 @@ iseq_free(void *ptr)
 					  RSTRING_PTR(iseq->location.path));
 	    }
 
-	    if (iseq->iseq != iseq->iseq_encoded) {
-		RUBY_FREE_UNLESS_NULL(iseq->iseq_encoded);
-	    }
-
-	    RUBY_FREE_UNLESS_NULL(iseq->iseq);
+	    RUBY_FREE_UNLESS_NULL(iseq->iseq_encoded);
 	    RUBY_FREE_UNLESS_NULL(iseq->line_info_table);
 	    RUBY_FREE_UNLESS_NULL(iseq->local_table);
 	    RUBY_FREE_UNLESS_NULL(iseq->is_entries);
@@ -88,6 +84,7 @@ iseq_free(void *ptr)
 	    RUBY_FREE_UNLESS_NULL(iseq->arg_opt_table);
 	    RUBY_FREE_UNLESS_NULL(iseq->arg_keyword_table);
 	    compile_data_free(iseq->compile_data);
+	    RUBY_FREE_UNLESS_NULL(iseq->iseq);
 	}
 	ruby_xfree(ptr);
     }
@@ -134,10 +131,6 @@ iseq_memsize(const void *ptr)
     if (ptr) {
 	iseq = ptr;
 	if (!iseq->orig) {
-	    if (iseq->iseq != iseq->iseq_encoded) {
-		size += iseq->iseq_size * sizeof(VALUE);
-	    }
-
 	    size += iseq->iseq_size * sizeof(VALUE);
 	    size += iseq->line_info_size * sizeof(struct iseq_line_info_entry);
 	    size += iseq->local_table_size * sizeof(ID);
@@ -158,6 +151,9 @@ iseq_memsize(const void *ptr)
 		}
 		size += sizeof(struct iseq_compile_data);
 	    }
+	    if (iseq->iseq) {
+		size += iseq->iseq_size * sizeof(VALUE);
+	    }
 	}
     }
 
@@ -1392,7 +1388,6 @@ rb_iseq_disasm(VALUE self)
 
     rb_secure(1);
 
-    iseq = iseqdat->iseq;
     size = iseqdat->iseq_size;
 
     rb_str_cat2(str, "== disasm: ");
@@ -1472,6 +1467,7 @@ rb_iseq_disasm(VALUE self)
     }
 
     /* show each line */
+    iseq = rb_iseq_original_iseq(iseqdat);
     for (n = 0; n < size;) {
 	n += rb_iseq_disasm_insn(str, iseq, n, iseqdat, child);
     }
@@ -1658,7 +1654,7 @@ iseq_data_to_ary(rb_iseq_t *iseq)
     size_t ti;
     unsigned int pos;
     unsigned int line = 0;
-    VALUE *seq;
+    VALUE *seq, *iseq_original;
 
     VALUE val = rb_ary_new();
     VALUE type; /* Symbol */
@@ -1759,7 +1755,9 @@ iseq_data_to_ary(rb_iseq_t *iseq)
     }
 
     /* body */
-    for (seq = iseq->iseq; seq < iseq->iseq + iseq->iseq_size; ) {
+    iseq_original = rb_iseq_original_iseq(iseq);
+
+    for (seq = iseq_original; seq < iseq_original + iseq->iseq_size; ) {
 	VALUE insn = *seq++;
 	int j, len = insn_len(insn);
 	VALUE *nseq = seq + len - 1;
@@ -1769,7 +1767,7 @@ iseq_data_to_ary(rb_iseq_t *iseq)
 	for (j=0; j<len-1; j++, seq++) {
 	    switch (insn_op_type(insn, j)) {
 	      case TS_OFFSET: {
-		unsigned long idx = nseq - iseq->iseq + *seq;
+		unsigned long idx = nseq - iseq_original + *seq;
 		rb_ary_push(ary, register_label(labels_table, idx));
 		break;
 	      }
@@ -1828,7 +1826,7 @@ iseq_data_to_ary(rb_iseq_t *iseq)
 
 		    for (i=0; i<RARRAY_LEN(val); i+=2) {
 			VALUE pos = FIX2INT(rb_ary_entry(val, i+1));
-			unsigned long idx = nseq - iseq->iseq + pos;
+			unsigned long idx = nseq - iseq_original + pos;
 
 			rb_ary_store(val, i+1,
 				     register_label(labels_table, idx));
@@ -2100,11 +2098,11 @@ rb_iseq_build_for_ruby2cext(
     RB_OBJ_WRITE(iseq->self, &iseq->mark_ary, 0);
     iseq->self = iseqval;
 
-    iseq->iseq = ALLOC_N(VALUE, iseq->iseq_size);
+    iseq->iseq_encoded = ALLOC_N(VALUE, iseq->iseq_size);
 
     for (i=0; i<iseq->iseq_size; i+=2) {
-	iseq->iseq[i] = BIN(opt_call_c_function);
-	iseq->iseq[i+1] = (VALUE)func;
+	iseq->iseq_encoded[i] = BIN(opt_call_c_function);
+	iseq->iseq_encoded[i+1] = (VALUE)func;
     }
 
     rb_iseq_translate_threaded_code(iseq);
@@ -2148,13 +2146,15 @@ rb_iseq_line_trace_each(VALUE iseqval, int (*func)(int line, rb_event_flag_t *ev
     size_t insn;
     rb_iseq_t *iseq;
     int cont = 1;
+    VALUE *iseq_original;
     GetISeqPtr(iseqval, iseq);
 
+    iseq_original = rb_iseq_original_iseq(iseq);
     for (pos = 0; cont && pos < iseq->iseq_size; pos += insn_len(insn)) {
-	insn = iseq->iseq[pos];
+	insn = iseq_original[pos];
 
 	if (insn == BIN(trace)) {
-	    rb_event_flag_t current_events = (VALUE)iseq->iseq[pos+1];
+	    rb_event_flag_t current_events = (VALUE)iseq_original[pos+1];
 
 	    if (current_events & RUBY_EVENT_LINE) {
 		rb_event_flag_t events = current_events & RUBY_EVENT_SPECIFIED_LINE;
@@ -2165,7 +2165,7 @@ rb_iseq_line_trace_each(VALUE iseqval, int (*func)(int line, rb_event_flag_t *ev
 		    /* printf("line: %d\n", line); */
 		    cont = (*func)(line, &events, data);
 		    if (current_events != events) {
-			iseq->iseq[pos+1] = iseq->iseq_encoded[pos+1] =
+			iseq_original[pos+1] = iseq->iseq_encoded[pos+1] =
 			  (VALUE)(current_events | (events & RUBY_EVENT_SPECIFIED_LINE));
 		    }
 		}
diff --git a/iseq.h b/iseq.h
index da2b678c1b..bf3a714e9d 100644
--- a/iseq.h
+++ b/iseq.h
@@ -17,6 +17,7 @@ RUBY_SYMBOL_EXPORT_BEGIN
 /* compile.c */
 VALUE rb_iseq_compile_node(VALUE self, NODE *node);
 int rb_iseq_translate_threaded_code(rb_iseq_t *iseq);
+VALUE *rb_iseq_original_iseq(rb_iseq_t *iseq);
 VALUE rb_iseq_build_from_ary(rb_iseq_t *iseq, VALUE locals, VALUE args,
 			     VALUE exception, VALUE body);
 
diff --git a/vm_core.h b/vm_core.h
index 939fd87d9d..dfbc8c7f76 100644
--- a/vm_core.h
+++ b/vm_core.h
@@ -212,8 +212,7 @@ struct rb_iseq_struct {
 
     rb_iseq_location_t location;
 
-    VALUE *iseq;         /* iseq (insn number and operands) */
-    VALUE *iseq_encoded; /* encoded iseq */
+    VALUE *iseq_encoded; /* encoded iseq (insn addr and operands) */
     unsigned int iseq_size;
     unsigned int line_info_size;
 
@@ -309,6 +308,10 @@ struct rb_iseq_struct {
 
     /* used at compile time */
     struct iseq_compile_data *compile_data;
+
+    /* original iseq, before encoding
+     * used for debug/dump (TODO: union with compile_data) */
+    VALUE *iseq;
 };
 
 enum ruby_special_exceptions {
diff --git a/vm_dump.c b/vm_dump.c
index bb2b2415c4..a65542b0f8 100644
--- a/vm_dump.c
+++ b/vm_dump.c
@@ -13,6 +13,7 @@
 #include "addr2line.h"
 #include "vm_core.h"
 #include "internal.h"
+#include "iseq.h"
 
 /* see vm_insnhelper.h for the values */
 #ifndef VMDEBUG
@@ -359,7 +360,6 @@ rb_vmdebug_debug_print_pre(rb_thread_t *th, rb_control_frame_t *cfp,VALUE *_pc)
     rb_iseq_t *iseq = cfp->iseq;
 
     if (iseq != 0) {
-	VALUE *seq = iseq->iseq;
 	ptrdiff_t pc = _pc - iseq->iseq_encoded;
 	int i;
 
@@ -371,7 +371,9 @@ rb_vmdebug_debug_print_pre(rb_thread_t *th, rb_control_frame_t *cfp,VALUE *_pc)
 
 	/* printf("%3"PRIdPTRDIFF" ", VM_CFP_CNT(th, cfp)); */
 	if (pc >= 0) {
-	    rb_iseq_disasm_insn(0, seq, (size_t)pc, iseq, 0);
+	    const VALUE *iseq_original = rb_iseq_original_iseq(iseq);
+
+	    rb_iseq_disasm_insn(0, iseq_original, (size_t)pc, iseq, 0);
 	}
     }