merge revision(s) 4c019f5a626523e99e2827ed917802e3097c380d,c3584dfacce4d0f2058d8403de6fdce4fd4d686b: [Backport #16676]

	check ar_table after `#hash` call

	ar_table can be converted to st_table just after `ar_do_hash()`
	function which calls `#hash` method. We need to check
	the representation to detect this mutation.
	[Bug #16676]

	check ar_table first.

	RHASH_AR_TABLE_SIZE() has assertion that it is a ar_talbe.
	The last commit breaks this assumption so check ar_table first.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/branches/ruby_2_6@67858 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

1 files changed, 26 insertions, 3 deletions

diff --git a/hash.c b/hash.c
index 3bba88adc7..e8be71bec3 100644
--- a/hash.c
+++ b/hash.c
@@ -607,10 +607,14 @@ ar_try_convert_table(VALUE hash)
 {
     st_table *new_tab;
     ar_table_entry *entry;
-    const unsigned size = RHASH_AR_TABLE_SIZE(hash);
+    unsigned size;
     st_index_t i;
 
-    if (!RHASH_AR_TABLE_P(hash) || size < RHASH_AR_TABLE_MAX_SIZE) {
+    if (!RHASH_AR_TABLE_P(hash)) return;
+
+    size = RHASH_AR_TABLE_SIZE(hash);
+
+    if (size < RHASH_AR_TABLE_MAX_SIZE) {
         return;
     }
 
@@ -824,6 +828,11 @@ ar_update(VALUE hash, st_data_t key,
     st_data_t value = 0, old_key;
     st_hash_t hash_value = do_hash(key);
 
+    if (UNLIKELY(!RHASH_AR_TABLE_P(hash))) {
+        /* `#hash` changes ar_table -> st_table */
+        return -1;
+    }
+
     if (RHASH_AR_TABLE_SIZE(hash) > 0) {
         bin = find_entry(hash, hash_value, key);
         existing = (bin != RHASH_AR_TABLE_MAX_BOUND) ? TRUE : FALSE;
@@ -872,6 +881,11 @@ ar_insert(VALUE hash, st_data_t key, st_data_t value)
     unsigned bin = RHASH_AR_TABLE_BOUND(hash);
     st_hash_t hash_value = do_hash(key);
 
+    if (UNLIKELY(!RHASH_AR_TABLE_P(hash))) {
+        /* `#hash` changes ar_table -> st_table */
+        return -1;
+    }
+
     hash_ar_table(hash); /* prepare ltbl */
 
     bin = find_entry(hash, hash_value, key);
@@ -900,7 +914,12 @@ static int
 ar_lookup(VALUE hash, st_data_t key, st_data_t *value)
 {
     st_hash_t hash_value = do_hash(key);
-    unsigned bin = find_entry(hash, hash_value, key);
+    unsigned bin;
+    if (UNLIKELY(!RHASH_AR_TABLE_P(hash))) {
+        /* `#hash` changes ar_table -> st_table */
+        return st_lookup(RHASH_ST_TABLE(hash), key, value);
+    }
+    bin = find_entry(hash, hash_value, key);
 
     if (bin == RHASH_AR_TABLE_MAX_BOUND) {
         return 0;
@@ -920,6 +939,10 @@ ar_delete(VALUE hash, st_data_t *key, st_data_t *value)
     unsigned bin;
     st_hash_t hash_value = do_hash(*key);
 
+    if (UNLIKELY(!RHASH_AR_TABLE_P(hash))) {
+        /* `#hash` changes ar_table -> st_table */
+        return st_delete(RHASH_ST_TABLE(hash), key, value);
+    }
 
     bin = find_entry(hash, hash_value, *key);