merge revision(s) 05553cf22d43dd78b8f30cc4591230b5c000c538: [Backport #20517]

	[Bug #20517] Make a multibyte character one token at meta escape

3 files changed, 56 insertions, 2 deletions

diff --git a/parse.y b/parse.y
index 03efa9aa9d..63624cb408 100644
--- a/parse.y
+++ b/parse.y
@@ -7428,7 +7428,11 @@ read_escape(struct parser_params *p, int flags, rb_encoding **encp)
 	    }
 	    return read_escape(p, flags|ESCAPE_META, encp) | 0x80;
 	}
-	else if (c == -1 || !ISASCII(c)) goto eof;
+        else if (c == -1) goto eof;
+        else if (!ISASCII(c)) {
+            tokskip_mbchar(p);
+            goto eof;
+        }
 	else {
 	    int c2 = escaped_control_code(c);
 	    if (c2) {
diff --git a/test/ripper/test_lexer.rb b/test/ripper/test_lexer.rb
index 8e8a616627..bbb0b31ae3 100644
--- a/test/ripper/test_lexer.rb
+++ b/test/ripper/test_lexer.rb
@@ -264,4 +264,54 @@ world"
     CODE
     assert_equal(code, Ripper.tokenize(code).join(""), bug)
   end
+
+  def test_invalid_escape_ctrl_mbchar
+    code = %["\\C-\u{3042}"]
+    expected = [
+      [[1, 0], :on_tstring_beg, '"', state(:EXPR_BEG)],
+      [[1, 1], :on_tstring_content, "\\C-\u{3042}", state(:EXPR_BEG)],
+      [[1, 7], :on_tstring_end, '"', state(:EXPR_END)],
+    ]
+
+    assert_lexer(expected, code)
+  end
+
+  def test_invalid_escape_meta_mbchar
+    code = %["\\M-\u{3042}"]
+    expected = [
+      [[1, 0], :on_tstring_beg, '"', state(:EXPR_BEG)],
+      [[1, 1], :on_tstring_content, "\\M-\u{3042}", state(:EXPR_BEG)],
+      [[1, 7], :on_tstring_end, '"', state(:EXPR_END)],
+    ]
+
+    assert_lexer(expected, code)
+  end
+
+  def test_invalid_escape_meta_ctrl_mbchar
+    code = %["\\M-\\C-\u{3042}"]
+    expected = [
+      [[1, 0], :on_tstring_beg, '"', state(:EXPR_BEG)],
+      [[1, 1], :on_tstring_content, "\\M-\\C-\u{3042}", state(:EXPR_BEG)],
+      [[1, 10], :on_tstring_end, '"', state(:EXPR_END)],
+    ]
+
+    assert_lexer(expected, code)
+  end
+
+  def test_invalid_escape_ctrl_meta_mbchar
+    code = %["\\C-\\M-\u{3042}"]
+    expected = [
+      [[1, 0], :on_tstring_beg, '"', state(:EXPR_BEG)],
+      [[1, 1], :on_tstring_content, "\\C-\\M-\u{3042}", state(:EXPR_BEG)],
+      [[1, 10], :on_tstring_end, '"', state(:EXPR_END)],
+    ]
+
+    assert_lexer(expected, code)
+  end
+
+  def assert_lexer(expected, code)
+    assert_equal(code, Ripper.tokenize(code).join(""))
+    assert_equal(expected, result = Ripper.lex(code),
+                 proc {expected.zip(result) {|e, r| break diff(e, r) unless e == r}})
+  end
 end
diff --git a/version.h b/version.h
index 2218b98c74..ca15d33238 100644
--- a/version.h
+++ b/version.h
@@ -11,7 +11,7 @@
 # define RUBY_VERSION_MINOR RUBY_API_VERSION_MINOR
 #define RUBY_VERSION_TEENY 4
 #define RUBY_RELEASE_DATE RUBY_RELEASE_YEAR_STR"-"RUBY_RELEASE_MONTH_STR"-"RUBY_RELEASE_DAY_STR
-#define RUBY_PATCHLEVEL 198
+#define RUBY_PATCHLEVEL 199
 
 #include "ruby/version.h"
 #include "ruby/internal/abi.h"