summaryrefslogtreecommitdiff
path: root/ext/ripper/lib
diff options
context:
space:
mode:
Diffstat (limited to 'ext/ripper/lib')
-rw-r--r--ext/ripper/lib/ripper/lexer.rb38
-rw-r--r--ext/ripper/lib/ripper/sexp.rb43
2 files changed, 75 insertions, 6 deletions
diff --git a/ext/ripper/lib/ripper/lexer.rb b/ext/ripper/lib/ripper/lexer.rb
index 586d0807a1..e0460e8b02 100644
--- a/ext/ripper/lib/ripper/lexer.rb
+++ b/ext/ripper/lib/ripper/lexer.rb
@@ -44,28 +44,56 @@ class Ripper
end
class Lexer < ::Ripper #:nodoc: internal use only
+ Elem = Struct.new(:pos, :event, :tok)
+
def tokenize
- lex().map {|pos, event, tok| tok }
+ parse().sort_by(&:pos).map(&:tok)
end
def lex
- parse().sort_by {|pos, event, tok| pos }
+ parse().sort_by(&:pos).map(&:to_a)
end
def parse
@buf = []
+ @stack = []
super
+ @buf.flatten!
@buf
end
private
+ def on_heredoc_dedent(v, w)
+ @buf.each do |e|
+ if e.event == :on_tstring_content
+ if (n = dedent_string(e.tok, w)) > 0
+ e.pos[1] += n
+ end
+ end
+ end
+ v
+ end
+
+ def on_heredoc_beg(tok)
+ @stack.push @buf
+ buf = []
+ @buf << buf
+ @buf = buf
+ @buf.push Elem.new([lineno(), column()], __callee__, tok)
+ end
+
+ def on_heredoc_end(tok)
+ @buf.push Elem.new([lineno(), column()], __callee__, tok)
+ @buf = @stack.pop
+ end
+
def _push_token(tok)
- @buf.push [[lineno(), column()], __callee__, tok]
+ @buf.push Elem.new([lineno(), column()], __callee__, tok)
end
- SCANNER_EVENTS.each do |event|
- alias_method "on_#{event}", :_push_token
+ (SCANNER_EVENTS.map {|event|:"on_#{event}"} - private_instance_methods(false)).each do |event|
+ alias_method event, :_push_token
end
end
diff --git a/ext/ripper/lib/ripper/sexp.rb b/ext/ripper/lib/ripper/sexp.rb
index 55b942a1c0..d9f445cfed 100644
--- a/ext/ripper/lib/ripper/sexp.rb
+++ b/ext/ripper/lib/ripper/sexp.rb
@@ -62,7 +62,35 @@ class Ripper
class SexpBuilder < ::Ripper #:nodoc:
private
- PARSER_EVENTS.each do |event|
+ def dedent_element(e, width)
+ if (n = dedent_string(e[1], width)) > 0
+ e[2][1] += n
+ end
+ e
+ end
+
+ def on_heredoc_dedent(val, width)
+ sub = proc do |cont|
+ cont.map! do |e|
+ if Array === e
+ case e[0]
+ when :@tstring_content
+ e = dedent_element(e, width)
+ when /_add\z/
+ e[1] = sub[e[1]]
+ end
+ elsif String === e
+ dedent_string(e, width)
+ end
+ e
+ end
+ end
+ sub[val]
+ val
+ end
+
+ events = private_instance_methods(false).grep(/\Aon_/) {$'.to_sym}
+ (PARSER_EVENTS - events).each do |event|
module_eval(<<-End, __FILE__, __LINE__ + 1)
def on_#{event}(*args)
args.unshift :#{event}
@@ -83,6 +111,19 @@ class Ripper
class SexpBuilderPP < SexpBuilder #:nodoc:
private
+ def on_heredoc_dedent(val, width)
+ val.map! do |e|
+ next e if Symbol === e and /_content\z/ =~ e
+ if Array === e and e[0] == :@tstring_content
+ e = dedent_element(e, width)
+ elsif String === e
+ dedent_string(e, width)
+ end
+ e
+ end
+ val
+ end
+
def _dispatch_event_new
[]
end