summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authoraycabta <aycabta@gmail.com>2021-01-03 03:25:47 +0900
committeraycabta <aycabta@gmail.com>2021-01-05 18:06:34 +0900
commit5a1866caff1ec7d924c4101ea05e9d460b6ef3ae (patch)
tree6289a03b84007f4f7e5088e70bc83add918d2ecf
parent505e01fe12007382aa2cd4b7231698597f563b42 (diff)
[ruby/irb] Use Ripper::Lexer#scan to take broken tokens
ref. https://github.com/ruby/reline/pull/242 https://github.com/ruby/irb/commit/54f90cb6c9
-rw-r--r--lib/irb/ruby-lex.rb12
-rw-r--r--test/irb/test_ruby_lex.rb19
2 files changed, 30 insertions, 1 deletions
diff --git a/lib/irb/ruby-lex.rb b/lib/irb/ruby-lex.rb
index 5c07ef4..72135c8 100644
--- a/lib/irb/ruby-lex.rb
+++ b/lib/irb/ruby-lex.rb
@@ -66,6 +66,7 @@ class RubyLex
unprocessed_tokens = []
line_num_offset = 0
tokens.each do |t|
+ next if t[1] == :on_parse_error || t[1] == :compile_error
partial_tokens << t
unprocessed_tokens << t
if t[2].include?("\n")
@@ -110,7 +111,12 @@ class RubyLex
verbose, $VERBOSE = $VERBOSE, nil
tokens = nil
self.class.compile_with_errors_suppressed(code) do |inner_code, line_no|
- tokens = Ripper.lex(inner_code, '-', line_no)
+ lexer = Ripper::Lexer.new(inner_code, '-', line_no)
+ if lexer.respond_to?(:scan) # Ruby 2.7+
+ tokens = lexer.scan
+ else
+ tokens = lexer.parse
+ end
end
$VERBOSE = verbose
tokens
@@ -122,6 +128,7 @@ class RubyLex
prev_spaces = md.nil? ? 0 : md[1].count(' ')
line_count = 0
@tokens.each_with_index do |t, i|
+ next if t[1] == :on_parse_error || t[1] == :compile_error
if t[2].include?("\n")
line_count += t[2].count("\n")
if line_count >= line_index
@@ -350,6 +357,7 @@ class RubyLex
indent = 0
in_oneliner_def = nil
tokens.each_with_index { |t, index|
+ next if t[1] == :on_parse_error || t[1] == :compile_error
# detecting one-liner method definition
if in_oneliner_def.nil?
if t[3].allbits?(Ripper::EXPR_ENDFN)
@@ -435,6 +443,7 @@ class RubyLex
open_brace_on_line = 0
in_oneliner_def = nil
@tokens.each_with_index do |t, index|
+ next if t[1] == :on_parse_error || t[1] == :compile_error
# detecting one-liner method definition
if in_oneliner_def.nil?
if t[3].allbits?(Ripper::EXPR_ENDFN)
@@ -504,6 +513,7 @@ class RubyLex
open_brace_on_line = 0
in_oneliner_def = nil
@tokens.each_with_index do |t, index|
+ next if t[1] == :on_parse_error || t[1] == :compile_error
# detecting one-liner method definition
if in_oneliner_def.nil?
if t[3].allbits?(Ripper::EXPR_ENDFN)
diff --git a/test/irb/test_ruby_lex.rb b/test/irb/test_ruby_lex.rb
index 69fa1f3..0c7c9f2 100644
--- a/test/irb/test_ruby_lex.rb
+++ b/test/irb/test_ruby_lex.rb
@@ -367,6 +367,25 @@ module TestIRB
end
end
+ def test_broken_heredoc
+ if Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.7.0')
+ skip 'This test needs Ripper::Lexer#scan to take broken tokens'
+ end
+ input_with_correct_indents = [
+ Row.new(%q(def foo), nil, 2, 1),
+ Row.new(%q( <<~Q), nil, 2, 1),
+ Row.new(%q( Qend), nil, 2, 1),
+ ]
+
+ lines = []
+ input_with_correct_indents.each do |row|
+ lines << row.content
+ assert_indenting(lines, row.current_line_spaces, false)
+ assert_indenting(lines, row.new_line_spaces, true)
+ assert_nesting_level(lines, row.nesting_level)
+ end
+ end
+
PromptRow = Struct.new(:prompt, :content)
class MockIO_DynamicPrompt