From 5ceff480c2c1696e9893a1e5e09e39f8425c6c5a Mon Sep 17 00:00:00 2001
From: Nobuyoshi Nakada <nobu@ruby-lang.org>
Date: Wed, 29 May 2019 16:02:09 +0900
Subject: ripper: Ripper::Lexer#scan

* ext/ripper/lib/ripper/lexer.rb (Ripper::Lexer#scan): parses the
  code and returns the result elements including errors.
  [EXPERIMENTAL]
---
 ext/ripper/lib/ripper/lexer.rb | 33 +++++++++++++++++++++++++++++----
 1 file changed, 29 insertions(+), 4 deletions(-)

(limited to 'ext')

diff --git a/ext/ripper/lib/ripper/lexer.rb b/ext/ripper/lib/ripper/lexer.rb
index a5ba5b6890..d1d257582d 100644
--- a/ext/ripper/lib/ripper/lexer.rb
+++ b/ext/ripper/lib/ripper/lexer.rb
@@ -60,13 +60,13 @@ class Ripper
       def nobits?(i) to_int.nobits?(i) end
     end
 
-    Elem = Struct.new(:pos, :event, :tok, :state) do
-      def initialize(pos, event, tok, state)
-        super(pos, event, tok, State.new(state))
+    Elem = Struct.new(:pos, :event, :tok, :state, :message) do
+      def initialize(pos, event, tok, state, message = nil)
+        super(pos, event, tok, State.new(state), message)
       end
 
       def inspect
-        "#<#{self.class}: #{event}@#{pos[0]}:#{pos[1]}:#{state}: #{tok.inspect}>"
+        "#<#{self.class}: #{event}@#{pos[0]}:#{pos[1]}:#{state}: #{tok.inspect}#{": " if message}#{message}>"
       end
 
       def pretty_print(q)
@@ -76,10 +76,23 @@ class Ripper
           q.breakable
           q.text("token: ")
           tok.pretty_print(q)
+          if message
+            q.breakable
+            q.text("message: ")
+            q.text(message)
+          end
         }
       end
+
+      def to_a
+        a = super
+        a.pop unless a.last
+        a
+      end
     end
 
+    attr_reader :errors
+
     def tokenize
       parse().sort_by(&:pos).map(&:tok)
     end
@@ -88,7 +101,13 @@ class Ripper
       parse().sort_by(&:pos).map(&:to_a)
     end
 
+    # parse the code and returns elements including errors.
+    def scan
+      (parse() + errors).sort_by {|e| [*e.pos, (e.message ? -1 : 0)]}
+    end
+
     def parse
+      @errors = []
       @buf = []
       @stack = []
       super
@@ -144,6 +163,12 @@ class Ripper
       @buf.push Elem.new([lineno(), column()], __callee__, tok, state())
     end
 
+    def on_error(mesg)
+      @errors.push Elem.new([lineno(), column()], __callee__, token(), state(), mesg)
+    end
+    alias on_parse_error on_error
+    alias compile_error on_error
+
     (SCANNER_EVENTS.map {|event|:"on_#{event}"} - private_instance_methods(false)).each do |event|
       alias_method event, :_push_token
     end
-- 
cgit v1.2.3