[Bug #18501] Fire write barrier after hash has been written

Before this change the write barrier was executed before the key and
value were actually reachable via the Hash.  This could cause
inconsistencies in object coloration which would lead to accidental
collection of dup'd keys.

Example:

1. Object O is grey, Object P is white.
2. Write barrier fires O -> P
3. Write barrier does nothing
4. Malloc happens, which starts GC
5. GC colors O black
6. P is written in to O (now we have O -> P reference)
7. P is now accidentally treated as garbage

1 files changed, 15 insertions, 5 deletions

diff --git a/hash.c b/hash.c
index f032ef642a..06b4652cde 100644
--- a/hash.c
+++ b/hash.c
@@ -1673,6 +1673,8 @@ struct update_arg {
     st_data_t arg;
     st_update_callback_func *func;
     VALUE hash;
+    VALUE key;
+    VALUE value;
 };
 
 typedef int (*tbl_update_func)(st_data_t *, st_data_t *, st_data_t, int);
@@ -1705,11 +1707,11 @@ tbl_update_modify(st_data_t *key, st_data_t *val, st_data_t arg, int existing)
       default:
         break;
       case ST_CONTINUE:
-        if (!existing || *key != old_key || *val != old_value)
+        if (!existing || *key != old_key || *val != old_value) {
             rb_hash_modify(hash);
-        /* write barrier */
-        RB_OBJ_WRITTEN(hash, Qundef, *key);
-        RB_OBJ_WRITTEN(hash, Qundef, *val);
+            p->key = *key;
+            p->value = *val;
+        }
         break;
       case ST_DELETE:
         if (existing)
@@ -1727,9 +1729,17 @@ tbl_update(VALUE hash, VALUE key, tbl_update_func func, st_data_t optional_arg)
         .arg = optional_arg,
         .func = func,
         .hash = hash,
+        .key  = key,
+        .value = (VALUE)optional_arg,
     };
 
-    return rb_hash_stlike_update(hash, key, tbl_update_modify, (st_data_t)&arg);
+    int ret = rb_hash_stlike_update(hash, key, tbl_update_modify, (st_data_t)&arg);
+
+    /* write barrier */
+    RB_OBJ_WRITTEN(hash, Qundef, arg.key);
+    RB_OBJ_WRITTEN(hash, Qundef, arg.value);
+
+    return ret;
 }
 
 #define UPDATE_CALLBACK(iter_lev, func) ((iter_lev) > 0 ? func##_noinsert : func##_insert)