variable.c: fix multiple autoload with identical file

We need to ensure autoload declarations pointing to the same
feature (aka "file") can wait on each other to avoid deadlock
situations.

So, reorganize autoload data structures to maintain a
feature => autoload_data_i mapping, and have module constant
tables point to the new autoload_const struct instead of
directly to autoload_data_i.  This allows multiple
autoload_const structs to refer to the SAME autoload_data_i
struct, and with it, the on-stack autoload_state.waitq.

The end result is different constants can share the same waitq
(tied to the feature name), and not deadlock each other during
loading.

Thanks to Eugene Kenny for the bug report and reproducible test case.

Reported-by: Eugene Kenny <elkenny@gmail.com>

* variable.c (autoload_featuremap): new global
  (struct autoload_const): new per-const struct
  (struct autoload_state): reference autoload_const instead of autoload_data_i
  (struct autoload_data_i): remove per-const
  (autoload_i_mark): delete from autoload_featuremap if unreferenced
  (autoload_c_mark): new dmark callback
  (autoload_c_free): new dfree callback
  (autoload_c_memsize): new memsize callback
  (autoload_const_type): new data type
  (get_autoload_data): set autoload_const as well
  (rb_autoload_str): use new data structures
  (autoload_delete): cleanup from autoload_featuremap
  (check_autoload_required): adjust for new internals
  (rb_autoloading_value): ditto
  (struct autoload_const_set_args): remove, redundant with autoload_const
  (const_tbl_update): adjust for new internals
  (autoload_const_set): ditto
  (autoload_require): ditto
  (autoload_reset): ditto
  (rb_autoload_load): ditto
  (rb_const_set): ditto
  (current_autoload_data): ditto
  (set_const_visibility): ditto
* test/ruby/test_autoload.rb (test_autoload_same_file): new test
  [ruby-core:86935] [Bug #14742]

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@63387 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

1 files changed, 158 insertions, 77 deletions

diff --git a/variable.c b/variable.c
index 6de7b476e3..6590567bf9 100644
--- a/variable.c
+++ b/variable.c
@@ -25,6 +25,7 @@
 
 static struct rb_id_table *rb_global_tbl;
 static ID autoload, classpath, tmp_classpath, classid;
+static VALUE autoload_featuremap; /* feature => autoload_i */
 
 static void check_before_mod_set(VALUE, ID, VALUE, const char *);
 static void setup_const_entry(rb_const_entry_t *, VALUE, VALUE, rb_const_flag_t);
@@ -1842,31 +1843,42 @@ autoload_data(VALUE mod, ID id)
     return (VALUE)val;
 }
 
+struct autoload_const {
+    struct list_node cnode; /* <=> autoload_data_i.constants */
+    VALUE mod;
+    VALUE ad; /* autoload_data_i */
+    VALUE value;
+    ID id;
+    int safe_level;
+    rb_const_flag_t flag;
+};
+
 /* always on stack, no need to mark */
 struct autoload_state {
-    struct autoload_data_i *ele;
-    VALUE mod;
+    struct autoload_const *ac;
     VALUE result;
-    ID id;
     VALUE thread;
     struct list_node waitq;
 };
 
 struct autoload_data_i {
     VALUE feature;
-    int safe_level;
-    rb_const_flag_t flag;
-    VALUE value;
     struct autoload_state *state; /* points to on-stack struct */
     rb_serial_t fork_gen;
+    struct list_head constants; /* <=> autoload_const.cnode */
 };
 
 static void
 autoload_i_mark(void *ptr)
 {
     struct autoload_data_i *p = ptr;
+
     rb_gc_mark(p->feature);
-    rb_gc_mark(p->value);
+
+    /* allow GC to free us if no modules refer to this via autoload_const.ad */
+    if (list_empty(&p->constants)) {
+        rb_hash_delete(autoload_featuremap, p->feature);
+    }
 }
 
 static size_t
@@ -1881,16 +1893,48 @@ static const rb_data_type_t autoload_data_i_type = {
     0, 0, RUBY_TYPED_FREE_IMMEDIATELY
 };
 
+static void
+autoload_c_mark(void *ptr)
+{
+    struct autoload_const *ac = ptr;
+
+    rb_gc_mark(ac->mod);
+    rb_gc_mark(ac->ad);
+    rb_gc_mark(ac->value);
+}
+
+static void
+autoload_c_free(void *ptr)
+{
+    struct autoload_const *ac = ptr;
+    list_del(&ac->cnode);
+}
+
+static size_t
+autoload_c_memsize(const void *ptr)
+{
+    return sizeof(struct autoload_const);
+}
+
+static const rb_data_type_t autoload_const_type = {
+    "autoload_const",
+    {autoload_c_mark, autoload_c_free, autoload_c_memsize,},
+    0, 0, RUBY_TYPED_FREE_IMMEDIATELY
+};
+
 static struct autoload_data_i *
-get_autoload_data(VALUE av)
+get_autoload_data(VALUE acv, struct autoload_const **acp)
 {
-    struct autoload_data_i *ele = rb_check_typeddata(av, &autoload_data_i_type);
+    struct autoload_const *ac = rb_check_typeddata(acv, &autoload_const_type);
+    struct autoload_data_i *ele;
 
+    ele = rb_check_typeddata(ac->ad, &autoload_data_i_type);
     /* do not reach across stack for ->state after forking: */
     if (ele && ele->state && ele->fork_gen != GET_VM()->fork_gen) {
         ele->state = 0;
         ele->fork_gen = 0;
     }
+    if (acp) *acp = ac;
     return ele;
 }
 
@@ -1940,17 +1984,42 @@ rb_autoload_str(VALUE mod, ID id, VALUE file)
 	DATA_PTR(av) = tbl = st_init_numtable();
     }
 
-    ad = TypedData_Make_Struct(0, struct autoload_data_i, &autoload_data_i_type, ele);
     if (OBJ_TAINTED(file)) {
 	file = rb_str_dup(file);
 	FL_UNSET(file, FL_TAINT);
     }
-    ele->feature = rb_fstring(file);
-    ele->safe_level = rb_safe_level();
-    ele->value = Qundef;
-    ele->state = 0;
-    ele->flag = CONST_PUBLIC;
-    st_insert(tbl, (st_data_t)id, (st_data_t)ad);
+    file = rb_fstring(file);
+    if (!autoload_featuremap) {
+        autoload_featuremap = rb_hash_new();
+        rb_obj_hide(autoload_featuremap);
+        rb_gc_register_mark_object(autoload_featuremap);
+    }
+    ad = rb_hash_aref(autoload_featuremap, file);
+    if (NIL_P(ad)) {
+        ad = TypedData_Make_Struct(0, struct autoload_data_i,
+                                    &autoload_data_i_type, ele);
+        ele->feature = file;
+        ele->state = 0;
+        list_head_init(&ele->constants);
+        rb_hash_aset(autoload_featuremap, file, ad);
+    }
+    else {
+        ele = rb_check_typeddata(ad, &autoload_data_i_type);
+    }
+    {
+        VALUE acv;
+        struct autoload_const *ac;
+        acv = TypedData_Make_Struct(0, struct autoload_const,
+                                    &autoload_const_type, ac);
+        ac->mod = mod;
+        ac->id = id;
+        ac->value = Qundef;
+        ac->safe_level = rb_safe_level();
+        ac->flag = CONST_PUBLIC;
+        ac->ad = ad;
+        list_add_tail(&ele->constants, &ac->cnode);
+        st_insert(tbl, (st_data_t)id, (st_data_t)acv);
+    }
 }
 
 static void
@@ -1960,8 +2029,19 @@ autoload_delete(VALUE mod, ID id)
 
     if (st_lookup(RCLASS_IV_TBL(mod), (st_data_t)autoload, &val)) {
 	struct st_table *tbl = check_autoload_table((VALUE)val);
+	struct autoload_data_i *ele;
+	struct autoload_const *ac;
 
 	st_delete(tbl, &n, &load);
+	ele = get_autoload_data((VALUE)load, &ac);
+	VM_ASSERT(!list_empty(&ele->constants));
+
+	/* list_del_init to make list_del in autoload_c_free idempotent: */
+	list_del_init(&ac->cnode);
+
+	if (list_empty(&ele->constants)) {
+	    rb_hash_delete(autoload_featuremap, ele->feature);
+	}
 
 	if (tbl->num_entries == 0) {
 	    n = autoload;
@@ -1987,12 +2067,13 @@ reset_safe(VALUE safe)
 static VALUE
 check_autoload_required(VALUE mod, ID id, const char **loadingpath)
 {
-    VALUE file, load;
+    VALUE file;
+    VALUE load = autoload_data(mod, id);
     struct autoload_data_i *ele;
     const char *loading;
     int safe;
 
-    if (!(load = autoload_data(mod, id)) || !(ele = get_autoload_data(load))) {
+    if (!load || !(ele = get_autoload_data(load, 0))) {
 	return 0;
     }
     file = ele->feature;
@@ -2027,19 +2108,21 @@ check_autoload_required(VALUE mod, ID id, const char **loadingpath)
 MJIT_FUNC_EXPORTED int
 rb_autoloading_value(VALUE mod, ID id, VALUE* value, rb_const_flag_t *flag)
 {
-    VALUE load;
+    VALUE load = autoload_data(mod, id);
     struct autoload_data_i *ele;
+    struct autoload_const *ac;
 
-    if (!(load = autoload_data(mod, id)) || !(ele = get_autoload_data(load))) {
-	return 0;
+    if (!load || !(ele = get_autoload_data(load, &ac))) {
+        return 0;
     }
+
     if (ele->state && ele->state->thread == rb_thread_current()) {
-	if (ele->value != Qundef) {
+	if (ac->value != Qundef) {
 	    if (value) {
-		*value = ele->value;
+		*value = ac->value;
 	    }
 	    if (flag) {
-		*flag = ele->flag;
+		*flag = ac->flag;
 	    }
 	    return 1;
 	}
@@ -2058,23 +2141,16 @@ autoload_defined_p(VALUE mod, ID id)
     return !rb_autoloading_value(mod, id, NULL, NULL);
 }
 
-struct autoload_const_set_args {
-    VALUE mod;
-    ID id;
-    VALUE value;
-    rb_const_flag_t flag;
-};
-
-static void const_tbl_update(struct autoload_const_set_args *);
+static void const_tbl_update(struct autoload_const *);
 
 static VALUE
 autoload_const_set(VALUE arg)
 {
-    struct autoload_const_set_args* args = (struct autoload_const_set_args *)arg;
-    VALUE klass = args->mod;
-    ID id = args->id;
-    check_before_mod_set(klass, id, args->value, "constant");
-    const_tbl_update(args);
+    struct autoload_const *ac = (struct autoload_const *)arg;
+    VALUE klass = ac->mod;
+    ID id = ac->id;
+    check_before_mod_set(klass, id, ac->value, "constant");
+    const_tbl_update(ac);
     return 0;			/* ignored */
 }
 
@@ -2082,10 +2158,13 @@ static VALUE
 autoload_require(VALUE arg)
 {
     struct autoload_state *state = (struct autoload_state *)arg;
+    struct autoload_const *ac = state->ac;
+    struct autoload_data_i *ele;
 
+    ele = rb_check_typeddata(ac->ad, &autoload_data_i_type);
     /* this may release GVL and switch threads: */
     state->result = rb_funcall(rb_vm_top_self(), rb_intern("require"), 1,
-			       state->ele->feature);
+			       ele->feature);
 
     return state->result;
 }
@@ -2095,26 +2174,27 @@ autoload_reset(VALUE arg)
 {
     struct autoload_state *state = (struct autoload_state *)arg;
     int need_wakeups = 0;
+    struct autoload_const *ac = state->ac;
+    struct autoload_data_i *ele;
 
-    if (state->ele->state == state) {
+    ele = rb_check_typeddata(ac->ad, &autoload_data_i_type);
+    if (ele->state == state) {
         need_wakeups = 1;
-        state->ele->state = 0;
-        state->ele->fork_gen = 0;
+        ele->state = 0;
+        ele->fork_gen = 0;
     }
 
     /* At the last, move a value defined in autoload to constant table */
-    if (RTEST(state->result) && state->ele->value != Qundef) {
-	int safe_backup;
-	struct autoload_const_set_args args;
-
-	args.mod = state->mod;
-	args.id = state->id;
-	args.value = state->ele->value;
-	args.flag = state->ele->flag;
-	safe_backup = rb_safe_level();
-	rb_set_safe_level_force(state->ele->safe_level);
-	rb_ensure(autoload_const_set, (VALUE)&args,
-	          reset_safe, (VALUE)safe_backup);
+    if (RTEST(state->result)) {
+        struct autoload_const *next;
+        int safe_backup = rb_safe_level();
+
+        list_for_each_safe(&ele->constants, ac, next, cnode) {
+            if (ac->value != Qundef) {
+                rb_ensure(autoload_const_set, (VALUE)ac,
+                          reset_safe, (VALUE)safe_backup);
+            }
+        }
     }
 
     /* wakeup any waiters we had */
@@ -2172,6 +2252,7 @@ rb_autoload_load(VALUE mod, ID id)
     VALUE load, result;
     const char *loading = 0, *src;
     struct autoload_data_i *ele;
+    struct autoload_const *ac;
     struct autoload_state state;
 
     if (!autoload_defined_p(mod, id)) return Qfalse;
@@ -2181,13 +2262,10 @@ rb_autoload_load(VALUE mod, ID id)
     if (src && loading && strcmp(src, loading) == 0) return Qfalse;
 
     /* set ele->state for a marker of autoloading thread */
-    if (!(ele = get_autoload_data(load))) {
+    if (!(ele = get_autoload_data(load, &ac))) {
 	return Qfalse;
     }
-
-    state.ele = ele;
-    state.mod = mod;
-    state.id = id;
+    state.ac = ac;
     state.thread = rb_thread_current();
     if (!ele->state) {
 	ele->state = &state;
@@ -2229,7 +2307,7 @@ rb_autoload_p(VALUE mod, ID id)
     }
     load = check_autoload_required(mod, id, 0);
     if (!load) return Qnil;
-    return (ele = get_autoload_data(load)) ? ele->feature : Qnil;
+    return (ele = get_autoload_data(load, 0)) ? ele->feature : Qnil;
 }
 
 MJIT_FUNC_EXPORTED void
@@ -2610,12 +2688,12 @@ rb_const_set(VALUE klass, ID id, VALUE val)
 	setup_const_entry(ce, klass, val, CONST_PUBLIC);
     }
     else {
-	struct autoload_const_set_args args;
-	args.mod = klass;
-	args.id = id;
-	args.value = val;
-	args.flag = CONST_PUBLIC;
-	const_tbl_update(&args);
+	struct autoload_const ac;
+	ac.mod = klass;
+	ac.id = id;
+	ac.value = val;
+	ac.flag = CONST_PUBLIC;
+	const_tbl_update(&ac);
     }
     /*
      * Resolve and cache class name immediately to resolve ambiguity
@@ -2647,12 +2725,12 @@ rb_const_set(VALUE klass, ID id, VALUE val)
 }
 
 static struct autoload_data_i *
-current_autoload_data(VALUE mod, ID id)
+current_autoload_data(VALUE mod, ID id, struct autoload_const **acp)
 {
     struct autoload_data_i *ele;
     VALUE load = autoload_data(mod, id);
     if (!load) return 0;
-    ele = get_autoload_data(load);
+    ele = get_autoload_data(load, acp);
     if (!ele) return 0;
     /* for autoloading thread, keep the defined value to autoloading storage */
     if (ele->state && (ele->state->thread == rb_thread_current())) {
@@ -2662,25 +2740,25 @@ current_autoload_data(VALUE mod, ID id)
 }
 
 static void
-const_tbl_update(struct autoload_const_set_args *args)
+const_tbl_update(struct autoload_const *ac)
 {
     VALUE value;
-    VALUE klass = args->mod;
-    VALUE val = args->value;
-    ID id = args->id;
+    VALUE klass = ac->mod;
+    VALUE val = ac->value;
+    ID id = ac->id;
     struct rb_id_table *tbl = RCLASS_CONST_TBL(klass);
-    rb_const_flag_t visibility = args->flag;
+    rb_const_flag_t visibility = ac->flag;
     rb_const_entry_t *ce;
 
     if (rb_id_table_lookup(tbl, id, &value)) {
 	ce = (rb_const_entry_t *)value;
 	if (ce->value == Qundef) {
-	    struct autoload_data_i *ele = current_autoload_data(klass, id);
+	    struct autoload_data_i *ele = current_autoload_data(klass, id, &ac);
 
 	    if (ele) {
 		rb_clear_constant_cache();
 
-		ele->value = val; /* autoload_i is non-WB-protected */
+		ac->value = val; /* autoload_i is non-WB-protected */
 		return;
 	    }
 	    /* otherwise, allow to override */
@@ -2753,6 +2831,7 @@ set_const_visibility(VALUE mod, int argc, const VALUE *argv,
     }
 
     for (i = 0; i < argc; i++) {
+	struct autoload_const *ac;
 	VALUE val = argv[i];
 	id = rb_check_id(&val);
 	if (!id) {
@@ -2767,10 +2846,12 @@ set_const_visibility(VALUE mod, int argc, const VALUE *argv,
 	    ce->flag &= ~mask;
 	    ce->flag |= flag;
 	    if (ce->value == Qundef) {
-		struct autoload_data_i *ele = current_autoload_data(mod, id);
+		struct autoload_data_i *ele;
+
+		ele = current_autoload_data(mod, id, &ac);
 		if (ele) {
-		    ele->flag &= ~mask;
-		    ele->flag |= flag;
+		    ac->flag &= ~mask;
+		    ac->flag |= flag;
 		}
 	    }
 	}