ripper: fix `#tok` on some error events [Bug 17345]

sorting alias target by event arity, and setup suitable `Elem` for error.

2 files changed, 35 insertions, 9 deletions

diff --git a/ext/ripper/lib/ripper/lexer.rb b/ext/ripper/lib/ripper/lexer.rb
index 6a9a9cb39a..cff38c617d 100644
--- a/ext/ripper/lib/ripper/lexer.rb
+++ b/ext/ripper/lib/ripper/lexer.rb
@@ -190,13 +190,18 @@ class Ripper
       e
     end
 
-    def on_error(mesg, tok = token())
-      @errors.push Elem.new([lineno(), column()], __callee__, tok, state(), mesg)
+    def on_error1(mesg)
+      @errors.push Elem.new([lineno(), column()], __callee__, token(), state(), mesg)
+    end
+
+    def on_error2(mesg, elem)
+      @errors.push Elem.new(elem.pos, __callee__, elem.tok, elem.state, mesg)
     end
     PARSER_EVENTS.grep(/_error\z/) do |e|
-      alias_method "on_#{e}", :on_error
+      arity = PARSER_EVENT_TABLE.fetch(e)
+      alias_method "on_#{e}", "on_error#{arity}"
     end
-    alias compile_error on_error
+    alias compile_error on_error1
 
     (SCANNER_EVENTS.map {|event|:"on_#{event}"} - private_instance_methods(false)).each do |event|
       alias_method event, :_push_token
diff --git a/test/ripper/test_lexer.rb b/test/ripper/test_lexer.rb
index 62ea98d1f8..910048b28c 100644
--- a/test/ripper/test_lexer.rb
+++ b/test/ripper/test_lexer.rb
@@ -147,11 +147,11 @@ class TestRipper::Lexer < Test::Unit::TestCase
   end
 
   BAD_CODE = {
-    parse_error: ['def req(true) end', %r[unexpected `true']],
-    assign_error: ['begin; nil = 1; end', %r[assign to nil]],
-    alias_error: ['begin; alias $x $1; end', %r[number variables]],
-    class_name_error: ['class bad; end', %r[class/module name]],
-    param_error: ['def req(@a) end', %r[formal argument.*instance]],
+    parse_error: ['def req(true) end', %r[unexpected `true'], 'true'],
+    assign_error: ['begin; nil = 1; end', %r[assign to nil], 'nil'],
+    alias_error: ['begin; alias $x $1; end', %r[number variables], '$1'],
+    class_name_error: ['class bad; end', %r[class/module name], 'bad'],
+    param_error: ['def req(@a) end', %r[formal argument.*instance], '@a'],
   }
 
   def test_raise_errors_keyword
@@ -183,4 +183,25 @@ class TestRipper::Lexer < Test::Unit::TestCase
       end
     end
   end
+
+  def test_lexer_scan_with_syntax_error
+    all_assertions do |all|
+      BAD_CODE.each do |err, (code, message, token)|
+        all.for(err) do
+          lexer = Ripper::Lexer.new(code)
+          elems = lexer.scan
+          assert_predicate lexer, :error?
+          error = lexer.errors.first
+          assert_match message, error.message
+          i = elems.find_index{|e| e == error}
+          assert_operator 0...elems.size, :include?, i
+          elem = elems[i + 1]
+          assert_not_equal error.event, elem.event
+          assert_equal error.pos, elem.pos
+          assert_equal error.tok, elem.tok
+          assert_equal error.state, elem.state
+        end
+      end
+    end
+  end
 end