2 files changed, 26 insertions, 17 deletions

diff --git a/ext/ripper/lib/ripper/filter.rb b/ext/ripper/lib/ripper/filter.rb
index 455197f690..9955d30550 100644
--- a/ext/ripper/lib/ripper/filter.rb
+++ b/ext/ripper/lib/ripper/filter.rb
@@ -25,6 +25,7 @@ class Ripper
       @__lexer = Lexer.new(src, filename, lineno)
       @__line = nil
       @__col = nil
+      @__state = nil
     end
 
     # The file name of the input.
@@ -46,13 +47,20 @@ class Ripper
       @__col
     end
 
+    # The scanner's state of the current token.
+    # This value is the bitwise OR of zero or more of the +Ripper::EXPR_*+ constants.
+    def state
+      @__state
+    end
+
     # Starts the parser.
     # +init+ is a data accumulator and is passed to the next event handler (as
     # of Enumerable#inject).
     def parse(init = nil)
       data = init
-      @__lexer.lex.each do |pos, event, tok|
+      @__lexer.lex.each do |pos, event, tok, state|
         @__line, @__col = *pos
+        @__state = state
         data = if respond_to?(event, true)
                then __send__(event, tok, data)
                else on_default(event, tok, data)
diff --git a/ext/ripper/lib/ripper/lexer.rb b/ext/ripper/lib/ripper/lexer.rb
index 300c8eef51..4e82a1275c 100644
--- a/ext/ripper/lib/ripper/lexer.rb
+++ b/ext/ripper/lib/ripper/lexer.rb
@@ -23,29 +23,30 @@ class Ripper
   end
 
   # Tokenizes the Ruby program and returns an array of an array,
-  # which is formatted like <code>[[lineno, column], type, token]</code>.
+  # which is formatted like
+  # <code>[[lineno, column], type, token, state]</code>.
   #
   #   require 'ripper'
   #   require 'pp'
   #
   #   pp Ripper.lex("def m(a) nil end")
-  #     #=> [[[1,  0], :on_kw,     "def"],
-  #          [[1,  3], :on_sp,     " "  ],
-  #          [[1,  4], :on_ident,  "m"  ],
-  #          [[1,  5], :on_lparen, "("  ],
-  #          [[1,  6], :on_ident,  "a"  ],
-  #          [[1,  7], :on_rparen, ")"  ],
-  #          [[1,  8], :on_sp,     " "  ],
-  #          [[1,  9], :on_kw,     "nil"],
-  #          [[1, 12], :on_sp,     " "  ],
-  #          [[1, 13], :on_kw,     "end"]]
+  #   #=> [[[1,  0], :on_kw,     "def", Ripper::EXPR_FNAME                   ],
+  #        [[1,  3], :on_sp,     " ",   Ripper::EXPR_FNAME                   ],
+  #        [[1,  4], :on_ident,  "m",   Ripper::EXPR_ENDFN                   ],
+  #        [[1,  5], :on_lparen, "(",   Ripper::EXPR_LABEL | Ripper::EXPR_BEG],
+  #        [[1,  6], :on_ident,  "a",   Ripper::EXPR_ARG                     ],
+  #        [[1,  7], :on_rparen, ")",   Ripper::EXPR_ENDFN                   ],
+  #        [[1,  8], :on_sp,     " ",   Ripper::EXPR_BEG                     ],
+  #        [[1,  9], :on_kw,     "nil", Ripper::EXPR_END                     ],
+  #        [[1, 12], :on_sp,     " ",   Ripper::EXPR_END                     ],
+  #        [[1, 13], :on_kw,     "end", Ripper::EXPR_END                     ]]
   #
   def Ripper.lex(src, filename = '-', lineno = 1)
     Lexer.new(src, filename, lineno).lex
   end
 
   class Lexer < ::Ripper   #:nodoc: internal use only
-    Elem = Struct.new(:pos, :event, :tok)
+    Elem = Struct.new(:pos, :event, :tok, :state)
 
     def tokenize
       parse().sort_by(&:pos).map(&:tok)
@@ -77,7 +78,7 @@ class Ripper
               e.event = :on_ignored_sp
               next
             end
-            ignored_sp << [i, Elem.new(e.pos.dup, :on_ignored_sp, tok[0, n])]
+            ignored_sp << [i, Elem.new(e.pos.dup, :on_ignored_sp, tok[0, n], e.state)]
             e.pos[1] += n
           end
         end
@@ -93,16 +94,16 @@ class Ripper
       buf = []
       @buf << buf
       @buf = buf
-      @buf.push Elem.new([lineno(), column()], __callee__, tok)
+      @buf.push Elem.new([lineno(), column()], __callee__, tok, state())
     end
 
     def on_heredoc_end(tok)
-      @buf.push Elem.new([lineno(), column()], __callee__, tok)
+      @buf.push Elem.new([lineno(), column()], __callee__, tok, state())
       @buf = @stack.pop
     end
 
     def _push_token(tok)
-      @buf.push Elem.new([lineno(), column()], __callee__, tok)
+      @buf.push Elem.new([lineno(), column()], __callee__, tok, state())
     end
 
     (SCANNER_EVENTS.map {|event|:"on_#{event}"} - private_instance_methods(false)).each do |event|