summaryrefslogtreecommitdiff
path: root/test/irb/test_ruby_lex.rb
diff options
context:
space:
mode:
Diffstat (limited to 'test/irb/test_ruby_lex.rb')
-rw-r--r--test/irb/test_ruby_lex.rb242
1 files changed, 0 insertions, 242 deletions
diff --git a/test/irb/test_ruby_lex.rb b/test/irb/test_ruby_lex.rb
deleted file mode 100644
index 4e406a8ce0..0000000000
--- a/test/irb/test_ruby_lex.rb
+++ /dev/null
@@ -1,242 +0,0 @@
-# frozen_string_literal: true
-require "irb"
-
-require_relative "helper"
-
-module TestIRB
- class RubyLexTest < TestCase
- def setup
- save_encodings
- end
-
- def teardown
- restore_encodings
- end
-
- def test_interpolate_token_with_heredoc_and_unclosed_embexpr
- code = <<~'EOC'
- ①+<<A-②
- #{③*<<B/④
- #{⑤&<<C|⑥
- EOC
- ripper_tokens = Ripper.tokenize(code)
- rubylex_tokens = IRB::RubyLex.ripper_lex_without_warning(code)
- # Assert no missing part
- assert_equal(code, rubylex_tokens.map(&:tok).join)
- # Assert ripper tokens are not removed
- ripper_tokens.each do |tok|
- assert(rubylex_tokens.any? { |t| t.tok == tok && t.tok != :on_ignored_by_ripper })
- end
- # Assert interpolated token position
- rubylex_tokens.each do |t|
- row, col = t.pos
- assert_equal t.tok, code.lines[row - 1].byteslice(col, t.tok.bytesize)
- end
- end
-
- def test_local_variables_dependent_code
- lines = ["a /1#/ do", "2"]
- assert_indent_level(lines, 1)
- assert_code_block_open(lines, true)
- assert_indent_level(lines, 0, local_variables: ['a'])
- assert_code_block_open(lines, false, local_variables: ['a'])
- end
-
- def test_literal_ends_with_space
- assert_code_block_open(['% a'], true)
- assert_code_block_open(['% a '], false)
- end
-
- def test_literal_ends_with_newline
- assert_code_block_open(['%'], true)
- assert_code_block_open(['%', ''], false)
- end
-
- def test_should_continue
- assert_should_continue(['a'], false)
- assert_should_continue(['/a/'], false)
- assert_should_continue(['a;'], false)
- assert_should_continue(['<<A', 'A'], false)
- assert_should_continue(['a...'], false)
- assert_should_continue(['a\\'], true)
- assert_should_continue(['a.'], true)
- assert_should_continue(['a+'], true)
- assert_should_continue(['a; #comment', '', '=begin', 'embdoc', '=end', ''], false)
- assert_should_continue(['a+ #comment', '', '=begin', 'embdoc', '=end', ''], true)
- end
-
- def test_code_block_open_with_should_continue
- # syntax ok
- assert_code_block_open(['a'], false) # continue: false
- assert_code_block_open(['a\\'], true) # continue: true
-
- # recoverable syntax error code is not terminated
- assert_code_block_open(['a+'], true)
-
- # unrecoverable syntax error code is terminated
- assert_code_block_open(['.; a+'], false)
-
- # other syntax error that failed to determine if it is recoverable or not
- assert_code_block_open(['@; a'], false)
- assert_code_block_open(['@; a+'], true)
- assert_code_block_open(['@; (a'], true)
- end
-
- def test_broken_percent_literal
- tokens = IRB::RubyLex.ripper_lex_without_warning('%wwww')
- pos_to_index = {}
- tokens.each_with_index { |t, i|
- assert_nil(pos_to_index[t.pos], "There is already another token in the position of #{t.inspect}.")
- pos_to_index[t.pos] = i
- }
- end
-
- def test_broken_percent_literal_in_method
- tokens = IRB::RubyLex.ripper_lex_without_warning(<<~EOC.chomp)
- def foo
- %wwww
- end
- EOC
- pos_to_index = {}
- tokens.each_with_index { |t, i|
- assert_nil(pos_to_index[t.pos], "There is already another token in the position of #{t.inspect}.")
- pos_to_index[t.pos] = i
- }
- end
-
- def test_unterminated_code
- ['do', '<<A'].each do |code|
- tokens = IRB::RubyLex.ripper_lex_without_warning(code)
- assert_equal(code, tokens.map(&:tok).join, "Cannot reconstruct code from tokens")
- error_tokens = tokens.map(&:event).grep(/error/)
- assert_empty(error_tokens, 'Error tokens must be ignored if there is corresponding non-error token')
- end
- end
-
- def test_unterminated_heredoc_string_literal
- ['<<A;<<B', "<<A;<<B\n", "%W[\#{<<A;<<B", "%W[\#{<<A;<<B\n"].each do |code|
- tokens = IRB::RubyLex.ripper_lex_without_warning(code)
- string_literal = IRB::NestingParser.open_tokens(tokens).last
- assert_equal('<<A', string_literal&.tok)
- end
- end
-
- def test_indent_level_with_heredoc_and_embdoc
- reference_code = <<~EOC.chomp
- if true
- hello
- p(
- )
- EOC
- code_with_heredoc = <<~EOC.chomp
- if true
- <<~A
- A
- p(
- )
- EOC
- code_with_embdoc = <<~EOC.chomp
- if true
- =begin
- =end
- p(
- )
- EOC
- expected = 1
- assert_indent_level(reference_code.lines, expected)
- assert_indent_level(code_with_heredoc.lines, expected)
- assert_indent_level(code_with_embdoc.lines, expected)
- end
-
- def test_assignment_expression
- ruby_lex = IRB::RubyLex.new
-
- [
- "foo = bar",
- "@foo = bar",
- "$foo = bar",
- "@@foo = bar",
- "::Foo = bar",
- "a::Foo = bar",
- "Foo = bar",
- "foo.bar = 1",
- "foo[1] = bar",
- "foo += bar",
- "foo -= bar",
- "foo ||= bar",
- "foo &&= bar",
- "foo, bar = 1, 2",
- "foo.bar=(1)",
- "foo; foo = bar",
- "foo; foo = bar; ;\n ;",
- "foo\nfoo = bar",
- ].each do |exp|
- assert(
- ruby_lex.assignment_expression?(exp, local_variables: []),
- "#{exp.inspect}: should be an assignment expression"
- )
- end
-
- [
- "foo",
- "foo.bar",
- "foo[0]",
- "foo = bar; foo",
- "foo = bar\nfoo",
- ].each do |exp|
- refute(
- ruby_lex.assignment_expression?(exp, local_variables: []),
- "#{exp.inspect}: should not be an assignment expression"
- )
- end
- end
-
- def test_assignment_expression_with_local_variable
- ruby_lex = IRB::RubyLex.new
- code = "a /1;x=1#/"
- refute(ruby_lex.assignment_expression?(code, local_variables: []), "#{code}: should not be an assignment expression")
- assert(ruby_lex.assignment_expression?(code, local_variables: [:a]), "#{code}: should be an assignment expression")
- refute(ruby_lex.assignment_expression?("", local_variables: [:a]), "empty code should not be an assignment expression")
- end
-
- def test_initialising_the_old_top_level_ruby_lex
- assert_in_out_err(["--disable-gems", "-W:deprecated"], <<~RUBY, [], /warning: constant ::RubyLex is deprecated/)
- require "irb"
- ::RubyLex.new(nil)
- RUBY
- end
-
- private
-
- def assert_indent_level(lines, expected, local_variables: [])
- indent_level, _continue, _code_block_open = check_state(lines, local_variables: local_variables)
- error_message = "Calculated the wrong number of indent level for:\n #{lines.join("\n")}"
- assert_equal(expected, indent_level, error_message)
- end
-
- def assert_should_continue(lines, expected, local_variables: [])
- _indent_level, continue, _code_block_open = check_state(lines, local_variables: local_variables)
- error_message = "Wrong result of should_continue for:\n #{lines.join("\n")}"
- assert_equal(expected, continue, error_message)
- end
-
- def assert_code_block_open(lines, expected, local_variables: [])
- if RUBY_ENGINE == 'truffleruby'
- omit "Remove me after https://github.com/ruby/prism/issues/2129 is addressed and adopted in TruffleRuby"
- end
-
- _indent_level, _continue, code_block_open = check_state(lines, local_variables: local_variables)
- error_message = "Wrong result of code_block_open for:\n #{lines.join("\n")}"
- assert_equal(expected, code_block_open, error_message)
- end
-
- def check_state(lines, local_variables: [])
- code = lines.map { |l| "#{l}\n" }.join # code should end with "\n"
- ruby_lex = IRB::RubyLex.new
- tokens, opens, terminated = ruby_lex.check_code_state(code, local_variables: local_variables)
- indent_level = ruby_lex.calc_indent_level(opens)
- continue = ruby_lex.should_continue?(tokens)
- [indent_level, continue, !terminated]
- end
- end
-end
generated by cgit v1.2.3 (git 2.25.1) at 2026-06-05 10:22:55 +0000