From 7df1e45bb6b4326da5c799dcf58d38f0a14362af Mon Sep 17 00:00:00 2001 From: nobu Date: Thu, 14 Sep 2017 10:53:47 +0000 Subject: ripper: add states of scanner * parse.y (ripper_state): add states of scanner to tokens from Ripper.lex and Ripper::Filter#on_*. based on the patch by aycabta (Code Ahss) at [ruby-core:81789]. [Feature #13686] * ext/ripper/tools/preproc.rb (prelude, usercode): generate EXPR_* constants from enums. git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@59896 b2dd03c8-39d4-4d8f-98ff-823fe69b080e --- ext/ripper/lib/ripper/filter.rb | 10 +++++++++- ext/ripper/lib/ripper/lexer.rb | 33 +++++++++++++++++---------------- ext/ripper/tools/preproc.rb | 20 +++++++++++++++++--- 3 files changed, 43 insertions(+), 20 deletions(-) (limited to 'ext') diff --git a/ext/ripper/lib/ripper/filter.rb b/ext/ripper/lib/ripper/filter.rb index 455197f690..9955d30550 100644 --- a/ext/ripper/lib/ripper/filter.rb +++ b/ext/ripper/lib/ripper/filter.rb @@ -25,6 +25,7 @@ class Ripper @__lexer = Lexer.new(src, filename, lineno) @__line = nil @__col = nil + @__state = nil end # The file name of the input. @@ -46,13 +47,20 @@ class Ripper @__col end + # The scanner's state of the current token. + # This value is the bitwise OR of zero or more of the +Ripper::EXPR_*+ constants. + def state + @__state + end + # Starts the parser. # +init+ is a data accumulator and is passed to the next event handler (as # of Enumerable#inject). def parse(init = nil) data = init - @__lexer.lex.each do |pos, event, tok| + @__lexer.lex.each do |pos, event, tok, state| @__line, @__col = *pos + @__state = state data = if respond_to?(event, true) then __send__(event, tok, data) else on_default(event, tok, data) diff --git a/ext/ripper/lib/ripper/lexer.rb b/ext/ripper/lib/ripper/lexer.rb index 300c8eef51..4e82a1275c 100644 --- a/ext/ripper/lib/ripper/lexer.rb +++ b/ext/ripper/lib/ripper/lexer.rb @@ -23,29 +23,30 @@ class Ripper end # Tokenizes the Ruby program and returns an array of an array, - # which is formatted like [[lineno, column], type, token]. + # which is formatted like + # [[lineno, column], type, token, state]. # # require 'ripper' # require 'pp' # # pp Ripper.lex("def m(a) nil end") - # #=> [[[1, 0], :on_kw, "def"], - # [[1, 3], :on_sp, " " ], - # [[1, 4], :on_ident, "m" ], - # [[1, 5], :on_lparen, "(" ], - # [[1, 6], :on_ident, "a" ], - # [[1, 7], :on_rparen, ")" ], - # [[1, 8], :on_sp, " " ], - # [[1, 9], :on_kw, "nil"], - # [[1, 12], :on_sp, " " ], - # [[1, 13], :on_kw, "end"]] + # #=> [[[1, 0], :on_kw, "def", Ripper::EXPR_FNAME ], + # [[1, 3], :on_sp, " ", Ripper::EXPR_FNAME ], + # [[1, 4], :on_ident, "m", Ripper::EXPR_ENDFN ], + # [[1, 5], :on_lparen, "(", Ripper::EXPR_LABEL | Ripper::EXPR_BEG], + # [[1, 6], :on_ident, "a", Ripper::EXPR_ARG ], + # [[1, 7], :on_rparen, ")", Ripper::EXPR_ENDFN ], + # [[1, 8], :on_sp, " ", Ripper::EXPR_BEG ], + # [[1, 9], :on_kw, "nil", Ripper::EXPR_END ], + # [[1, 12], :on_sp, " ", Ripper::EXPR_END ], + # [[1, 13], :on_kw, "end", Ripper::EXPR_END ]] # def Ripper.lex(src, filename = '-', lineno = 1) Lexer.new(src, filename, lineno).lex end class Lexer < ::Ripper #:nodoc: internal use only - Elem = Struct.new(:pos, :event, :tok) + Elem = Struct.new(:pos, :event, :tok, :state) def tokenize parse().sort_by(&:pos).map(&:tok) @@ -77,7 +78,7 @@ class Ripper e.event = :on_ignored_sp next end - ignored_sp << [i, Elem.new(e.pos.dup, :on_ignored_sp, tok[0, n])] + ignored_sp << [i, Elem.new(e.pos.dup, :on_ignored_sp, tok[0, n], e.state)] e.pos[1] += n end end @@ -93,16 +94,16 @@ class Ripper buf = [] @buf << buf @buf = buf - @buf.push Elem.new([lineno(), column()], __callee__, tok) + @buf.push Elem.new([lineno(), column()], __callee__, tok, state()) end def on_heredoc_end(tok) - @buf.push Elem.new([lineno(), column()], __callee__, tok) + @buf.push Elem.new([lineno(), column()], __callee__, tok, state()) @buf = @stack.pop end def _push_token(tok) - @buf.push Elem.new([lineno(), column()], __callee__, tok) + @buf.push Elem.new([lineno(), column()], __callee__, tok, state()) end (SCANNER_EVENTS.map {|event|:"on_#{event}"} - private_instance_methods(false)).each do |event| diff --git a/ext/ripper/tools/preproc.rb b/ext/ripper/tools/preproc.rb index 16449ec81c..8b68579164 100755 --- a/ext/ripper/tools/preproc.rb +++ b/ext/ripper/tools/preproc.rb @@ -40,6 +40,7 @@ def main end def prelude(f, out) + @exprs = {} while line = f.gets case line when %r @@ -56,6 +57,16 @@ def prelude(f, out) when /\A%type/ out << line.sub(/<\w+>/, '') else + if (/^enum lex_state_(?:bits|e) \{/ =~ line)..(/^\}/ =~ line) + case line + when /^\s*(EXPR_\w+),\s+\/\*(.+)\*\// + @exprs[$1.chomp("_bit")] = $2.strip + when /^\s*(EXPR_\w+)\s+=\s+(.+)$/ + name = $1 + val = $2.chomp(",") + @exprs[name] = "equals to " + (val.start_with?("(") ? "#{val}" : "+#{val}+") + end + end out << line end end @@ -84,9 +95,12 @@ def grammar(f, out) end def usercode(f, out) - while line = f.gets - out << line - end + require 'erb' + compiler = ERB::Compiler.new('%-') + compiler.put_cmd = compiler.insert_cmd = "out.<<" + lineno = f.lineno + src, = compiler.compile(f.read) + eval(src, binding, f.path, lineno) end main -- cgit v1.2.3