diff options
Diffstat (limited to 'ext/ripper/tools')
-rw-r--r-- | ext/ripper/tools/dsl.rb | 173 | ||||
-rw-r--r-- | ext/ripper/tools/generate.rb | 51 | ||||
-rw-r--r-- | ext/ripper/tools/preproc.rb | 115 |
3 files changed, 236 insertions, 103 deletions
diff --git a/ext/ripper/tools/dsl.rb b/ext/ripper/tools/dsl.rb index 49ff51711f..38f859dd97 100644 --- a/ext/ripper/tools/dsl.rb +++ b/ext/ripper/tools/dsl.rb @@ -1,83 +1,177 @@ +# frozen_string_literal: true + # Simple DSL implementation for Ripper code generation # -# input: /*% ripper: stmts_add(stmts_new, void_stmt) %*/ +# input: /*% ripper: stmts_add!(stmts_new!, void_stmt!) %*/ # output: # VALUE v1, v2; # v1 = dispatch0(stmts_new); # v2 = dispatch0(void_stmt); # $$ = dispatch2(stmts_add, v1, v2); - -$dollar = "$$" -alias $$ $dollar +# +# - The code must be a single line. +# +# - The code is basically Ruby code, even if it appears like in C and +# the result will be processed as C. e.g., comments need to be in +# Ruby style. class DSL - def initialize(code, options) + TAG_PATTERN = /(?><[a-zA-Z0-9_]+>)/.source + NAME_PATTERN = /(?>\$|\d+|[a-zA-Z_][a-zA-Z0-9_]*|\[[a-zA-Z_.][-a-zA-Z0-9_.]*\])(?>(?:\.|->)[a-zA-Z_][a-zA-Z0-9_]*)*/.source + NOT_REF_PATTERN = /(?>\#.*|[^\"$@]*|"(?>\\.|[^\"])*")/.source + + def self.line?(line, lineno = nil, indent: nil) + if %r<(?<space>\s*)/\*% *ripper(?:\[(?<option>.*?)\])?: *(?<code>.*?) *%\*/> =~ line + new(code, comma_split(option), lineno, indent: indent || space) + end + end + + def self.comma_split(str) + str or return [] + str.scan(/(([^(,)]+|\((?:,|\g<0>)*\))+)/).map(&:first) + end + + using Module.new { + refine Array do + def to_s + if empty? + "rb_ary_new()" + else + "rb_ary_new_from_args(#{size}, #{map(&:to_s).join(', ')})" + end + end + end + } + + class Var + class Table < Hash + def initialize(&block) + super() {|tbl, arg| + tbl.fetch(arg, &block) + } + end + + def fetch(arg, &block) + super { + self[arg] = Var.new(self, arg, &block) + } + end + + def add(&block) + v = new_var + self[v] = Var.new(self, v, &block) + end + + def defined?(name) + name = name.to_s + any? {|_, v| v.var == name} + end + + def new_var + "v#{size+1}" + end + end + + attr_reader :var, :value + + PRETTY_PRINT_INSTANCE_VARIABLES = instance_methods(false).freeze + + def pretty_print_instance_variables + PRETTY_PRINT_INSTANCE_VARIABLES + end + + alias to_s var + + def initialize(table, arg, &block) + @var = table.new_var + @value = yield arg + @table = table + end + + # Indexing. + # + # $:1 -> v1=get_value($:1) + # $:1[0] -> rb_ary_entry(v1, 0) + # $:1[0..1] -> [rb_ary_entry(v1, 0), rb_ary_entry(v1, 1)] + # *$:1[0..1] -> rb_ary_entry(v1, 0), rb_ary_entry(v1, 1) + # + # Splat needs `[range]` because `Var` does not have the length info. + def [](idx) + if ::Range === idx + idx.map {|i| self[i]} + else + @table.fetch("#@var[#{idx}]") {"rb_ary_entry(#{@var}, #{idx})"} + end + end + end + + def initialize(code, options, lineno = nil, indent: "\t\t\t") + @lineno = lineno + @indent = indent @events = {} @error = options.include?("error") - @brace = options.include?("brace") if options.include?("final") @final = "p->result" else - @final = (options.grep(/\A\$(?:\$|\d+)\z/)[0] || "$$") + @final = (options.grep(/\A\$#{NAME_PATTERN}\z/o)[0] || "p->s_lvalue") end - @vars = 0 - # create $1 == "$1", $2 == "$2", ... - s = (1..20).map {|n| "$#{n}"} - re = Array.new(s.size, "([^\0]+)") - /#{re.join("\0")}/ =~ s.join("\0") + bind = dsl_binding + @var_table = Var::Table.new {|arg| "get_value(#{arg})"} + code = code.gsub(%r[\G#{NOT_REF_PATTERN}\K(\$|\$:|@)#{TAG_PATTERN}?#{NAME_PATTERN}]o) { + if (arg = $&) == "$:$" + '"p->s_lvalue"' + elsif arg.start_with?("$:") + "(#{@var_table[arg]}=@var_table[#{arg.dump}])" + else + arg.dump + end + } + @last_value = bind.eval(code) + rescue SyntaxError + $stderr.puts "error on line #{@lineno}" if @lineno + raise + end + def dsl_binding(p = "p") # struct parser_params *p - p = p = "p" - - @code = "" - @last_value = eval(code) + binding end attr_reader :events undef lambda undef hash - undef class + undef :class def generate - s = "#@code#@final=#@last_value;" - s = "{VALUE #{ (1..@vars).map {|v| "v#{ v }" }.join(",") };#{ s }}" if @vars > 0 + s = "#@final=#@last_value;" s << "ripper_error(p);" if @error - s = "{#{ s }}" if @brace - "\t\t\t#{s}" - end - - def new_var - "v#{ @vars += 1 }" - end - - def opt_event(event, default, addend) - add_event(event, [default, addend], true) + unless @var_table.empty? + vars = @var_table.map {|_, v| "#{v.var}=#{v.value}"}.join(", ") + s = "VALUE #{ vars }; #{ s }" + end + "#{@indent}{#{s}}" end - def add_event(event, args, qundef_check = false) + def add_event(event, args) event = event.to_s.sub(/!\z/, "") @events[event] = args.size vars = [] args.each do |arg| - vars << v = new_var - @code << "#{ v }=#{ arg };" + arg = @var_table.add {arg} unless Var === arg + vars << arg end - v = new_var - d = "dispatch#{ args.size }(#{ [event, *vars].join(",") })" - d = "#{ vars.last }==Qundef ? #{ vars.first } : #{ d }" if qundef_check - @code << "#{ v }=#{ d };" - v + @var_table.add {"dispatch#{ args.size }(#{ [event, *vars].join(",") })"} end def method_missing(event, *args) if event.to_s =~ /!\z/ add_event(event, args) - elsif args.empty? and /\Aid[A-Z_]/ =~ event.to_s + elsif args.empty? and (/\Aid[A-Z_]/ =~ event or @var_table.defined?(event)) event else - "#{ event }(#{ args.join(", ") })" + "#{ event }(#{ args.map(&:to_s).join(", ") })" end end @@ -85,4 +179,3 @@ class DSL name end end - diff --git a/ext/ripper/tools/generate.rb b/ext/ripper/tools/generate.rb index 883e6ef2df..57ecac0b39 100644 --- a/ext/ripper/tools/generate.rb +++ b/ext/ripper/tools/generate.rb @@ -11,7 +11,7 @@ def main parser = @parser = OptionParser.new parser.banner = "Usage: #{File.basename($0)} --mode=MODE [--ids1src=PATH] [--ids2src=PATH] [--output=PATH]" - parser.on('--mode=MODE', 'check, eventids1, or eventids2table.') {|m| + parser.on('--mode=MODE', 'check, eventids1_h, eventids1, or eventids2table.') {|m| mode = m } parser.on('--ids1src=PATH', 'A source file of event-IDs 1 (parse.y).') {|path| @@ -45,6 +45,9 @@ def main abort "event crash: #{common.join(' ')}" end exit 0 + when 'eventids1_h' + usage 'no --ids1src' unless ids1src + result = generate_eventids1_h(read_ids1(ids1src)) when 'eventids1' usage 'no --ids1src' unless ids1src result = generate_eventids1(read_ids1(ids1src)) @@ -67,28 +70,44 @@ def usage(msg) exit false end -def generate_eventids1(ids) +def generate_eventids1_h(ids) buf = "".dup - buf << %Q[static struct {\n] + buf << %Q[#ifndef RIPPER_EVENTIDS1\n] + buf << %Q[#define RIPPER_EVENTIDS1\n] + buf << %Q[\n] + buf << %Q[#define RIPPER_ID(n) ripper_parser_ids.id_ ## n\n] + buf << %Q[void ripper_init_eventids1(void);\n] + buf << %Q[void ripper_init_eventids1_table(VALUE self);\n] + buf << %Q[\n] + buf << %Q[struct ripper_parser_ids {\n] ids.each do |id, arity| buf << %Q[ ID id_#{id};\n] end - buf << %Q[} ripper_parser_ids;\n] + buf << %Q[};\n] buf << %Q[\n] - ids.each do |id, arity| - buf << %Q[#define ripper_id_#{id} ripper_parser_ids.id_#{id}\n] - end + buf << %Q[#endif /* RIPPER_EVENTIDS1 */\n] + buf << %Q[\n] +end + +def generate_eventids1(ids) + buf = "".dup + buf << %Q[#include "ruby/ruby.h"\n] + buf << %Q[#include "eventids1.h"\n] + buf << %Q[\n] + buf << %Q[struct ripper_parser_ids ripper_parser_ids;\n] buf << %Q[\n] - buf << %Q[static void\n] + buf << %Q[void\n] buf << %Q[ripper_init_eventids1(void)\n] buf << %Q[{\n] - buf << %Q[#define set_id1(name) ripper_id_##name = rb_intern_const("on_"#name)\n] + buf << %Q[#define set_id1(name) RIPPER_ID(name) = rb_intern_const("on_"#name)\n] ids.each do |id, arity| buf << %Q[ set_id1(#{id});\n] end buf << %Q[}\n] buf << %Q[\n] - buf << %Q[static void\n] + buf << %Q[#define intern_sym(name) ID2SYM(rb_intern_const(name))\n] + buf << %Q[\n] + buf << %Q[void\n] buf << %Q[ripper_init_eventids1_table(VALUE self)\n] buf << %Q[{\n] buf << %Q[ VALUE h = rb_hash_new();\n] @@ -102,7 +121,11 @@ end def generate_eventids2_table(ids) buf = "".dup - buf << %Q[static void\n] + buf << %Q[#include "ruby/ruby.h"\n] + buf << %Q[\n] + buf << %Q[#define intern_sym(name) ID2SYM(rb_intern_const(name))\n] + buf << %Q[\n] + buf << %Q[void\n] buf << %Q[ripper_init_eventids2_table(VALUE self)\n] buf << %Q[{\n] buf << %Q[ VALUE h = rb_hash_new();\n] @@ -111,6 +134,8 @@ def generate_eventids2_table(ids) buf << %Q[ rb_hash_aset(h, intern_sym("#{id}"), INT2FIX(1));\n] end buf << %Q[}\n] + buf << %Q[\n] + buf << %Q[#define RIPPER_EVENTIDS2_TABLE_SIZE #{ids.size}\n] buf end @@ -146,9 +171,7 @@ def read_ids1_with_locations(path) line.scan(/\bdispatch(\d)\((\w+)/) do |arity, event| (h[event] ||= []).push [f.lineno, arity.to_i] end - if line =~ %r</\*% *ripper(?:\[(.*?)\])?: *(.*?) *%\*/> - gen = DSL.new($2, ($1 || "").split(",")) - gen.generate + if gen = DSL.line?(line, f.lineno) gen.events.each do |event, arity| (h[event] ||= []).push [f.lineno, arity.to_i] end diff --git a/ext/ripper/tools/preproc.rb b/ext/ripper/tools/preproc.rb index b838a78db7..5e8a6e0cb5 100644 --- a/ext/ripper/tools/preproc.rb +++ b/ext/ripper/tools/preproc.rb @@ -5,11 +5,15 @@ require 'optparse' def main output = nil + template = nil parser = OptionParser.new - parser.banner = "Usage: #{File.basename($0)} [--output=PATH] <parse.y>" + parser.banner = "Usage: #{File.basename($0)} [--output=PATH] [--template=PATH] <parse.y>" parser.on('--output=PATH', 'An output file.') {|path| output = path } + parser.on('--template=PATH', 'An template file.') {|path| + template = path + } parser.on('--help', 'Prints this message and quit.') { puts parser.help exit true @@ -17,50 +21,56 @@ def main begin parser.parse! rescue OptionParser::ParseError => err - $stderr.puts err.message - $stderr.puts parser.help - exit false - end - unless ARGV.size == 1 - abort "wrong number of arguments (#{ARGV.size} for 1)" + warn err.message + abort parser.help end out = "".dup - File.open(ARGV[0]) {|f| - prelude f, out - grammar f, out - usercode f, out - } - if output - File.open(output, 'w') {|f| - f.write out + if ARGV[0] == "-" + unless ARGV.size == 2 + abort "wrong number of arguments (#{ARGV.size} for 2)" + end + process STDIN, out, ARGV[1], template + else + unless ARGV.size == 1 + abort "wrong number of arguments (#{ARGV.size} for 1)" + end + File.open(ARGV[0]) {|f| + process f, out, ARGV[0], template } + end + if output + File.write(output, out) else print out end end -def prelude(f, out) - @exprs = {} - lex_state_def = false +def process(f, out, path, template) + prelude f, out + grammar f, out + usercode f, out, path, template +end + +require_relative 'dsl' + +def generate_line(f, out) while line = f.gets - case line - when /\A%%/ + case + when gen = DSL.line?(line, f.lineno) + out << gen.generate << "\n" + when line.start_with?("%%") out << "%%\n" - return - when /\A%token/ - out << line.sub(/<\w+>/, '<val>') - when /\A%type/ - out << line.sub(/<\w+>/, '<val>') - when /^enum lex_state_(?:bits|e) \{/ - lex_state_def = true - out << line - when /^\}/ - lex_state_def = false - out << line + break else - out << line + out << yield(line) end - if lex_state_def + end +end + +def prelude(f, out) + @exprs = {} + generate_line(f, out) do |line| + if (/^enum lex_state_(?:bits|e) \{/ =~ line)..(/^\}/ =~ line) case line when /^\s*(EXPR_\w+),\s+\/\*(.+)\*\// @exprs[$1.chomp("_bit")] = $2.strip @@ -70,38 +80,45 @@ def prelude(f, out) @exprs[name] = "equals to " + (val.start_with?("(") ? "<tt>#{val}</tt>" : "+#{val}+") end end + line end end -require_relative "dsl" - def grammar(f, out) - while line = f.gets + generate_line(f, out) do |line| case line - when %r</\*% *ripper(?:\[(.*?)\])?: *(.*?) *%\*/> - out << DSL.new($2, ($1 || "").split(",")).generate << "\n" when %r</\*%%%\*/> - out << "#if 0\n" + "#if 0\n" when %r</\*%> - out << "#endif\n" + "#endif\n" when %r<%\*/> - out << "\n" - when /\A%%/ - out << "%%\n" - return + "\n" else - out << line + line end end end -def usercode(f, out) +def usercode(f, out, path, template) require 'erb' + lineno = nil + src = nil compiler = ERB::Compiler.new('%-') compiler.put_cmd = compiler.insert_cmd = "out.<<" - lineno = f.lineno - src, = compiler.compile(f.read) - eval(src, binding, f.path, lineno) + + if template + File.open(template) do |f| + out.clear + lineno = f.lineno + src, = compiler.compile(f.read) + path = template + end + else + lineno = f.lineno + src, = compiler.compile(f.read) + end + + eval(src, binding, path, lineno) end main |