summaryrefslogtreecommitdiff
path: root/ext/ripper/tools
diff options
context:
space:
mode:
Diffstat (limited to 'ext/ripper/tools')
-rw-r--r--ext/ripper/tools/dsl.rb181
-rw-r--r--[-rwxr-xr-x]ext/ripper/tools/generate-param-macros.rb0
-rw-r--r--[-rwxr-xr-x]ext/ripper/tools/generate.rb54
-rw-r--r--[-rwxr-xr-x]ext/ripper/tools/preproc.rb132
-rw-r--r--[-rwxr-xr-x]ext/ripper/tools/strip.rb0
5 files changed, 299 insertions, 68 deletions
diff --git a/ext/ripper/tools/dsl.rb b/ext/ripper/tools/dsl.rb
new file mode 100644
index 0000000000..38f859dd97
--- /dev/null
+++ b/ext/ripper/tools/dsl.rb
@@ -0,0 +1,181 @@
+# frozen_string_literal: true
+
+# Simple DSL implementation for Ripper code generation
+#
+# input: /*% ripper: stmts_add!(stmts_new!, void_stmt!) %*/
+# output:
+# VALUE v1, v2;
+# v1 = dispatch0(stmts_new);
+# v2 = dispatch0(void_stmt);
+# $$ = dispatch2(stmts_add, v1, v2);
+#
+# - The code must be a single line.
+#
+# - The code is basically Ruby code, even if it appears like in C and
+# the result will be processed as C. e.g., comments need to be in
+# Ruby style.
+
+class DSL
+ TAG_PATTERN = /(?><[a-zA-Z0-9_]+>)/.source
+ NAME_PATTERN = /(?>\$|\d+|[a-zA-Z_][a-zA-Z0-9_]*|\[[a-zA-Z_.][-a-zA-Z0-9_.]*\])(?>(?:\.|->)[a-zA-Z_][a-zA-Z0-9_]*)*/.source
+ NOT_REF_PATTERN = /(?>\#.*|[^\"$@]*|"(?>\\.|[^\"])*")/.source
+
+ def self.line?(line, lineno = nil, indent: nil)
+ if %r<(?<space>\s*)/\*% *ripper(?:\[(?<option>.*?)\])?: *(?<code>.*?) *%\*/> =~ line
+ new(code, comma_split(option), lineno, indent: indent || space)
+ end
+ end
+
+ def self.comma_split(str)
+ str or return []
+ str.scan(/(([^(,)]+|\((?:,|\g<0>)*\))+)/).map(&:first)
+ end
+
+ using Module.new {
+ refine Array do
+ def to_s
+ if empty?
+ "rb_ary_new()"
+ else
+ "rb_ary_new_from_args(#{size}, #{map(&:to_s).join(', ')})"
+ end
+ end
+ end
+ }
+
+ class Var
+ class Table < Hash
+ def initialize(&block)
+ super() {|tbl, arg|
+ tbl.fetch(arg, &block)
+ }
+ end
+
+ def fetch(arg, &block)
+ super {
+ self[arg] = Var.new(self, arg, &block)
+ }
+ end
+
+ def add(&block)
+ v = new_var
+ self[v] = Var.new(self, v, &block)
+ end
+
+ def defined?(name)
+ name = name.to_s
+ any? {|_, v| v.var == name}
+ end
+
+ def new_var
+ "v#{size+1}"
+ end
+ end
+
+ attr_reader :var, :value
+
+ PRETTY_PRINT_INSTANCE_VARIABLES = instance_methods(false).freeze
+
+ def pretty_print_instance_variables
+ PRETTY_PRINT_INSTANCE_VARIABLES
+ end
+
+ alias to_s var
+
+ def initialize(table, arg, &block)
+ @var = table.new_var
+ @value = yield arg
+ @table = table
+ end
+
+ # Indexing.
+ #
+ # $:1 -> v1=get_value($:1)
+ # $:1[0] -> rb_ary_entry(v1, 0)
+ # $:1[0..1] -> [rb_ary_entry(v1, 0), rb_ary_entry(v1, 1)]
+ # *$:1[0..1] -> rb_ary_entry(v1, 0), rb_ary_entry(v1, 1)
+ #
+ # Splat needs `[range]` because `Var` does not have the length info.
+ def [](idx)
+ if ::Range === idx
+ idx.map {|i| self[i]}
+ else
+ @table.fetch("#@var[#{idx}]") {"rb_ary_entry(#{@var}, #{idx})"}
+ end
+ end
+ end
+
+ def initialize(code, options, lineno = nil, indent: "\t\t\t")
+ @lineno = lineno
+ @indent = indent
+ @events = {}
+ @error = options.include?("error")
+ if options.include?("final")
+ @final = "p->result"
+ else
+ @final = (options.grep(/\A\$#{NAME_PATTERN}\z/o)[0] || "p->s_lvalue")
+ end
+
+ bind = dsl_binding
+ @var_table = Var::Table.new {|arg| "get_value(#{arg})"}
+ code = code.gsub(%r[\G#{NOT_REF_PATTERN}\K(\$|\$:|@)#{TAG_PATTERN}?#{NAME_PATTERN}]o) {
+ if (arg = $&) == "$:$"
+ '"p->s_lvalue"'
+ elsif arg.start_with?("$:")
+ "(#{@var_table[arg]}=@var_table[#{arg.dump}])"
+ else
+ arg.dump
+ end
+ }
+ @last_value = bind.eval(code)
+ rescue SyntaxError
+ $stderr.puts "error on line #{@lineno}" if @lineno
+ raise
+ end
+
+ def dsl_binding(p = "p")
+ # struct parser_params *p
+ binding
+ end
+
+ attr_reader :events
+
+ undef lambda
+ undef hash
+ undef :class
+
+ def generate
+ s = "#@final=#@last_value;"
+ s << "ripper_error(p);" if @error
+ unless @var_table.empty?
+ vars = @var_table.map {|_, v| "#{v.var}=#{v.value}"}.join(", ")
+ s = "VALUE #{ vars }; #{ s }"
+ end
+ "#{@indent}{#{s}}"
+ end
+
+ def add_event(event, args)
+ event = event.to_s.sub(/!\z/, "")
+ @events[event] = args.size
+ vars = []
+ args.each do |arg|
+ arg = @var_table.add {arg} unless Var === arg
+ vars << arg
+ end
+ @var_table.add {"dispatch#{ args.size }(#{ [event, *vars].join(",") })"}
+ end
+
+ def method_missing(event, *args)
+ if event.to_s =~ /!\z/
+ add_event(event, args)
+ elsif args.empty? and (/\Aid[A-Z_]/ =~ event or @var_table.defined?(event))
+ event
+ else
+ "#{ event }(#{ args.map(&:to_s).join(", ") })"
+ end
+ end
+
+ def self.const_missing(name)
+ name
+ end
+end
diff --git a/ext/ripper/tools/generate-param-macros.rb b/ext/ripper/tools/generate-param-macros.rb
index f0de55a5f2..f0de55a5f2 100755..100644
--- a/ext/ripper/tools/generate-param-macros.rb
+++ b/ext/ripper/tools/generate-param-macros.rb
diff --git a/ext/ripper/tools/generate.rb b/ext/ripper/tools/generate.rb
index cf24f1398d..57ecac0b39 100755..100644
--- a/ext/ripper/tools/generate.rb
+++ b/ext/ripper/tools/generate.rb
@@ -11,7 +11,7 @@ def main
parser = @parser = OptionParser.new
parser.banner = "Usage: #{File.basename($0)} --mode=MODE [--ids1src=PATH] [--ids2src=PATH] [--output=PATH]"
- parser.on('--mode=MODE', 'check, eventids1, or eventids2table.') {|m|
+ parser.on('--mode=MODE', 'check, eventids1_h, eventids1, or eventids2table.') {|m|
mode = m
}
parser.on('--ids1src=PATH', 'A source file of event-IDs 1 (parse.y).') {|path|
@@ -45,6 +45,9 @@ def main
abort "event crash: #{common.join(' ')}"
end
exit 0
+ when 'eventids1_h'
+ usage 'no --ids1src' unless ids1src
+ result = generate_eventids1_h(read_ids1(ids1src))
when 'eventids1'
usage 'no --ids1src' unless ids1src
result = generate_eventids1(read_ids1(ids1src))
@@ -67,28 +70,44 @@ def usage(msg)
exit false
end
-def generate_eventids1(ids)
+def generate_eventids1_h(ids)
buf = "".dup
- buf << %Q[static struct {\n]
+ buf << %Q[#ifndef RIPPER_EVENTIDS1\n]
+ buf << %Q[#define RIPPER_EVENTIDS1\n]
+ buf << %Q[\n]
+ buf << %Q[#define RIPPER_ID(n) ripper_parser_ids.id_ ## n\n]
+ buf << %Q[void ripper_init_eventids1(void);\n]
+ buf << %Q[void ripper_init_eventids1_table(VALUE self);\n]
+ buf << %Q[\n]
+ buf << %Q[struct ripper_parser_ids {\n]
ids.each do |id, arity|
buf << %Q[ ID id_#{id};\n]
end
- buf << %Q[} ripper_parser_ids;\n]
+ buf << %Q[};\n]
buf << %Q[\n]
- ids.each do |id, arity|
- buf << %Q[#define ripper_id_#{id} ripper_parser_ids.id_#{id}\n]
- end
+ buf << %Q[#endif /* RIPPER_EVENTIDS1 */\n]
+ buf << %Q[\n]
+end
+
+def generate_eventids1(ids)
+ buf = "".dup
+ buf << %Q[#include "ruby/ruby.h"\n]
+ buf << %Q[#include "eventids1.h"\n]
+ buf << %Q[\n]
+ buf << %Q[struct ripper_parser_ids ripper_parser_ids;\n]
buf << %Q[\n]
- buf << %Q[static void\n]
+ buf << %Q[void\n]
buf << %Q[ripper_init_eventids1(void)\n]
buf << %Q[{\n]
- buf << %Q[#define set_id1(name) ripper_id_##name = rb_intern_const("on_"#name)\n]
+ buf << %Q[#define set_id1(name) RIPPER_ID(name) = rb_intern_const("on_"#name)\n]
ids.each do |id, arity|
buf << %Q[ set_id1(#{id});\n]
end
buf << %Q[}\n]
buf << %Q[\n]
- buf << %Q[static void\n]
+ buf << %Q[#define intern_sym(name) ID2SYM(rb_intern_const(name))\n]
+ buf << %Q[\n]
+ buf << %Q[void\n]
buf << %Q[ripper_init_eventids1_table(VALUE self)\n]
buf << %Q[{\n]
buf << %Q[ VALUE h = rb_hash_new();\n]
@@ -102,7 +121,11 @@ end
def generate_eventids2_table(ids)
buf = "".dup
- buf << %Q[static void\n]
+ buf << %Q[#include "ruby/ruby.h"\n]
+ buf << %Q[\n]
+ buf << %Q[#define intern_sym(name) ID2SYM(rb_intern_const(name))\n]
+ buf << %Q[\n]
+ buf << %Q[void\n]
buf << %Q[ripper_init_eventids2_table(VALUE self)\n]
buf << %Q[{\n]
buf << %Q[ VALUE h = rb_hash_new();\n]
@@ -111,6 +134,8 @@ def generate_eventids2_table(ids)
buf << %Q[ rb_hash_aset(h, intern_sym("#{id}"), INT2FIX(1));\n]
end
buf << %Q[}\n]
+ buf << %Q[\n]
+ buf << %Q[#define RIPPER_EVENTIDS2_TABLE_SIZE #{ids.size}\n]
buf
end
@@ -135,6 +160,8 @@ def check_arity(h)
abort if invalid
end
+require_relative "dsl"
+
def read_ids1_with_locations(path)
h = {}
File.open(path) {|f|
@@ -144,6 +171,11 @@ def read_ids1_with_locations(path)
line.scan(/\bdispatch(\d)\((\w+)/) do |arity, event|
(h[event] ||= []).push [f.lineno, arity.to_i]
end
+ if gen = DSL.line?(line, f.lineno)
+ gen.events.each do |event, arity|
+ (h[event] ||= []).push [f.lineno, arity.to_i]
+ end
+ end
end
}
h
diff --git a/ext/ripper/tools/preproc.rb b/ext/ripper/tools/preproc.rb
index 8b68579164..5e8a6e0cb5 100755..100644
--- a/ext/ripper/tools/preproc.rb
+++ b/ext/ripper/tools/preproc.rb
@@ -5,11 +5,15 @@ require 'optparse'
def main
output = nil
+ template = nil
parser = OptionParser.new
- parser.banner = "Usage: #{File.basename($0)} [--output=PATH] <parse.y>"
+ parser.banner = "Usage: #{File.basename($0)} [--output=PATH] [--template=PATH] <parse.y>"
parser.on('--output=PATH', 'An output file.') {|path|
output = path
}
+ parser.on('--template=PATH', 'An template file.') {|path|
+ template = path
+ }
parser.on('--help', 'Prints this message and quit.') {
puts parser.help
exit true
@@ -17,90 +21,104 @@ def main
begin
parser.parse!
rescue OptionParser::ParseError => err
- $stderr.puts err.message
- $stderr.puts parser.help
- exit false
- end
- unless ARGV.size == 1
- abort "wrong number of arguments (#{ARGV.size} for 1)"
+ warn err.message
+ abort parser.help
end
out = "".dup
- File.open(ARGV[0]) {|f|
- prelude f, out
- grammar f, out
- usercode f, out
- }
- if output
- File.open(output, 'w') {|f|
- f.write out
+ if ARGV[0] == "-"
+ unless ARGV.size == 2
+ abort "wrong number of arguments (#{ARGV.size} for 2)"
+ end
+ process STDIN, out, ARGV[1], template
+ else
+ unless ARGV.size == 1
+ abort "wrong number of arguments (#{ARGV.size} for 1)"
+ end
+ File.open(ARGV[0]) {|f|
+ process f, out, ARGV[0], template
}
+ end
+ if output
+ File.write(output, out)
else
print out
end
end
-def prelude(f, out)
- @exprs = {}
+def process(f, out, path, template)
+ prelude f, out
+ grammar f, out
+ usercode f, out, path, template
+end
+
+require_relative 'dsl'
+
+def generate_line(f, out)
while line = f.gets
- case line
- when %r</\*%%%\*/>
- out << '/*' << $/
- when %r</\*%>
- out << '*/' << $/
- when %r<%\*/>
- out << $/
- when /\A%%/
- out << '%%' << $/
- return
- when /\A%token/
- out << line.sub(/<\w+>/, '<val>')
- when /\A%type/
- out << line.sub(/<\w+>/, '<val>')
+ case
+ when gen = DSL.line?(line, f.lineno)
+ out << gen.generate << "\n"
+ when line.start_with?("%%")
+ out << "%%\n"
+ break
else
- if (/^enum lex_state_(?:bits|e) \{/ =~ line)..(/^\}/ =~ line)
- case line
- when /^\s*(EXPR_\w+),\s+\/\*(.+)\*\//
- @exprs[$1.chomp("_bit")] = $2.strip
- when /^\s*(EXPR_\w+)\s+=\s+(.+)$/
- name = $1
- val = $2.chomp(",")
- @exprs[name] = "equals to " + (val.start_with?("(") ? "<tt>#{val}</tt>" : "+#{val}+")
- end
+ out << yield(line)
+ end
+ end
+end
+
+def prelude(f, out)
+ @exprs = {}
+ generate_line(f, out) do |line|
+ if (/^enum lex_state_(?:bits|e) \{/ =~ line)..(/^\}/ =~ line)
+ case line
+ when /^\s*(EXPR_\w+),\s+\/\*(.+)\*\//
+ @exprs[$1.chomp("_bit")] = $2.strip
+ when /^\s*(EXPR_\w+)\s+=\s+(.+)$/
+ name = $1
+ val = $2.chomp(",")
+ @exprs[name] = "equals to " + (val.start_with?("(") ? "<tt>#{val}</tt>" : "+#{val}+")
end
- out << line
end
+ line
end
end
def grammar(f, out)
- while line = f.gets
+ generate_line(f, out) do |line|
case line
when %r</\*%%%\*/>
- out << '#if 0' << $/
- when %r</\*%c%\*/>
- out << '/*' << $/
- when %r</\*%c>
- out << '*/' << $/
+ "#if 0\n"
when %r</\*%>
- out << '#endif' << $/
+ "#endif\n"
when %r<%\*/>
- out << $/
- when /\A%%/
- out << '%%' << $/
- return
+ "\n"
else
- out << line
+ line
end
end
end
-def usercode(f, out)
+def usercode(f, out, path, template)
require 'erb'
+ lineno = nil
+ src = nil
compiler = ERB::Compiler.new('%-')
compiler.put_cmd = compiler.insert_cmd = "out.<<"
- lineno = f.lineno
- src, = compiler.compile(f.read)
- eval(src, binding, f.path, lineno)
+
+ if template
+ File.open(template) do |f|
+ out.clear
+ lineno = f.lineno
+ src, = compiler.compile(f.read)
+ path = template
+ end
+ else
+ lineno = f.lineno
+ src, = compiler.compile(f.read)
+ end
+
+ eval(src, binding, path, lineno)
end
main
diff --git a/ext/ripper/tools/strip.rb b/ext/ripper/tools/strip.rb
index 23102f797a..23102f797a 100755..100644
--- a/ext/ripper/tools/strip.rb
+++ b/ext/ripper/tools/strip.rb
generated by cgit v1.2.3 (git 2.25.1) at 2026-06-05 07:33:19 +0000