ripper: add states of scanner

* parse.y (ripper_state): add states of scanner to tokens from
  Ripper.lex and Ripper::Filter#on_*.  based on the patch by
  aycabta (Code Ahss) at [ruby-core:81789].  [Feature #13686]

* ext/ripper/tools/preproc.rb (prelude, usercode): generate EXPR_*
  constants from enums.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@59896 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

1 files changed, 17 insertions, 3 deletions

diff --git a/ext/ripper/tools/preproc.rb b/ext/ripper/tools/preproc.rb
index 16449ec81c..8b68579164 100755
--- a/ext/ripper/tools/preproc.rb
+++ b/ext/ripper/tools/preproc.rb
@@ -40,6 +40,7 @@ def main
 end
 
 def prelude(f, out)
+  @exprs = {}
   while line = f.gets
     case line
     when %r</\*%%%\*/>
@@ -56,6 +57,16 @@ def prelude(f, out)
     when /\A%type/
       out << line.sub(/<\w+>/, '<val>')
     else
+      if (/^enum lex_state_(?:bits|e) \{/ =~ line)..(/^\}/ =~ line)
+        case line
+        when /^\s*(EXPR_\w+),\s+\/\*(.+)\*\//
+          @exprs[$1.chomp("_bit")] = $2.strip
+        when /^\s*(EXPR_\w+)\s+=\s+(.+)$/
+          name = $1
+          val = $2.chomp(",")
+          @exprs[name] = "equals to " + (val.start_with?("(") ? "<tt>#{val}</tt>" : "+#{val}+")
+        end
+      end
       out << line
     end
   end
@@ -84,9 +95,12 @@ def grammar(f, out)
 end
 
 def usercode(f, out)
-  while line = f.gets
-    out << line
-  end
+  require 'erb'
+  compiler = ERB::Compiler.new('%-')
+  compiler.put_cmd = compiler.insert_cmd = "out.<<"
+  lineno = f.lineno
+  src, = compiler.compile(f.read)
+  eval(src, binding, f.path, lineno)
 end
 
 main