summaryrefslogtreecommitdiff
path: root/ext/ripper/tools
diff options
context:
space:
mode:
Diffstat (limited to 'ext/ripper/tools')
-rw-r--r--ext/ripper/tools/dsl.rb181
-rw-r--r--ext/ripper/tools/generate-param-macros.rb15
-rw-r--r--ext/ripper/tools/generate.rb194
-rw-r--r--ext/ripper/tools/preproc.rb124
-rw-r--r--ext/ripper/tools/strip.rb12
5 files changed, 526 insertions, 0 deletions
diff --git a/ext/ripper/tools/dsl.rb b/ext/ripper/tools/dsl.rb
new file mode 100644
index 0000000000..38f859dd97
--- /dev/null
+++ b/ext/ripper/tools/dsl.rb
@@ -0,0 +1,181 @@
+# frozen_string_literal: true
+
+# Simple DSL implementation for Ripper code generation
+#
+# input: /*% ripper: stmts_add!(stmts_new!, void_stmt!) %*/
+# output:
+# VALUE v1, v2;
+# v1 = dispatch0(stmts_new);
+# v2 = dispatch0(void_stmt);
+# $$ = dispatch2(stmts_add, v1, v2);
+#
+# - The code must be a single line.
+#
+# - The code is basically Ruby code, even if it appears like in C and
+# the result will be processed as C. e.g., comments need to be in
+# Ruby style.
+
+class DSL
+ TAG_PATTERN = /(?><[a-zA-Z0-9_]+>)/.source
+ NAME_PATTERN = /(?>\$|\d+|[a-zA-Z_][a-zA-Z0-9_]*|\[[a-zA-Z_.][-a-zA-Z0-9_.]*\])(?>(?:\.|->)[a-zA-Z_][a-zA-Z0-9_]*)*/.source
+ NOT_REF_PATTERN = /(?>\#.*|[^\"$@]*|"(?>\\.|[^\"])*")/.source
+
+ def self.line?(line, lineno = nil, indent: nil)
+ if %r<(?<space>\s*)/\*% *ripper(?:\[(?<option>.*?)\])?: *(?<code>.*?) *%\*/> =~ line
+ new(code, comma_split(option), lineno, indent: indent || space)
+ end
+ end
+
+ def self.comma_split(str)
+ str or return []
+ str.scan(/(([^(,)]+|\((?:,|\g<0>)*\))+)/).map(&:first)
+ end
+
+ using Module.new {
+ refine Array do
+ def to_s
+ if empty?
+ "rb_ary_new()"
+ else
+ "rb_ary_new_from_args(#{size}, #{map(&:to_s).join(', ')})"
+ end
+ end
+ end
+ }
+
+ class Var
+ class Table < Hash
+ def initialize(&block)
+ super() {|tbl, arg|
+ tbl.fetch(arg, &block)
+ }
+ end
+
+ def fetch(arg, &block)
+ super {
+ self[arg] = Var.new(self, arg, &block)
+ }
+ end
+
+ def add(&block)
+ v = new_var
+ self[v] = Var.new(self, v, &block)
+ end
+
+ def defined?(name)
+ name = name.to_s
+ any? {|_, v| v.var == name}
+ end
+
+ def new_var
+ "v#{size+1}"
+ end
+ end
+
+ attr_reader :var, :value
+
+ PRETTY_PRINT_INSTANCE_VARIABLES = instance_methods(false).freeze
+
+ def pretty_print_instance_variables
+ PRETTY_PRINT_INSTANCE_VARIABLES
+ end
+
+ alias to_s var
+
+ def initialize(table, arg, &block)
+ @var = table.new_var
+ @value = yield arg
+ @table = table
+ end
+
+ # Indexing.
+ #
+ # $:1 -> v1=get_value($:1)
+ # $:1[0] -> rb_ary_entry(v1, 0)
+ # $:1[0..1] -> [rb_ary_entry(v1, 0), rb_ary_entry(v1, 1)]
+ # *$:1[0..1] -> rb_ary_entry(v1, 0), rb_ary_entry(v1, 1)
+ #
+ # Splat needs `[range]` because `Var` does not have the length info.
+ def [](idx)
+ if ::Range === idx
+ idx.map {|i| self[i]}
+ else
+ @table.fetch("#@var[#{idx}]") {"rb_ary_entry(#{@var}, #{idx})"}
+ end
+ end
+ end
+
+ def initialize(code, options, lineno = nil, indent: "\t\t\t")
+ @lineno = lineno
+ @indent = indent
+ @events = {}
+ @error = options.include?("error")
+ if options.include?("final")
+ @final = "p->result"
+ else
+ @final = (options.grep(/\A\$#{NAME_PATTERN}\z/o)[0] || "p->s_lvalue")
+ end
+
+ bind = dsl_binding
+ @var_table = Var::Table.new {|arg| "get_value(#{arg})"}
+ code = code.gsub(%r[\G#{NOT_REF_PATTERN}\K(\$|\$:|@)#{TAG_PATTERN}?#{NAME_PATTERN}]o) {
+ if (arg = $&) == "$:$"
+ '"p->s_lvalue"'
+ elsif arg.start_with?("$:")
+ "(#{@var_table[arg]}=@var_table[#{arg.dump}])"
+ else
+ arg.dump
+ end
+ }
+ @last_value = bind.eval(code)
+ rescue SyntaxError
+ $stderr.puts "error on line #{@lineno}" if @lineno
+ raise
+ end
+
+ def dsl_binding(p = "p")
+ # struct parser_params *p
+ binding
+ end
+
+ attr_reader :events
+
+ undef lambda
+ undef hash
+ undef :class
+
+ def generate
+ s = "#@final=#@last_value;"
+ s << "ripper_error(p);" if @error
+ unless @var_table.empty?
+ vars = @var_table.map {|_, v| "#{v.var}=#{v.value}"}.join(", ")
+ s = "VALUE #{ vars }; #{ s }"
+ end
+ "#{@indent}{#{s}}"
+ end
+
+ def add_event(event, args)
+ event = event.to_s.sub(/!\z/, "")
+ @events[event] = args.size
+ vars = []
+ args.each do |arg|
+ arg = @var_table.add {arg} unless Var === arg
+ vars << arg
+ end
+ @var_table.add {"dispatch#{ args.size }(#{ [event, *vars].join(",") })"}
+ end
+
+ def method_missing(event, *args)
+ if event.to_s =~ /!\z/
+ add_event(event, args)
+ elsif args.empty? and (/\Aid[A-Z_]/ =~ event or @var_table.defined?(event))
+ event
+ else
+ "#{ event }(#{ args.map(&:to_s).join(", ") })"
+ end
+ end
+
+ def self.const_missing(name)
+ name
+ end
+end
diff --git a/ext/ripper/tools/generate-param-macros.rb b/ext/ripper/tools/generate-param-macros.rb
new file mode 100644
index 0000000000..f0de55a5f2
--- /dev/null
+++ b/ext/ripper/tools/generate-param-macros.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+off = true
+ARGF.each do |line|
+ case line
+ when /RIPPER_PARAMS_DECL_BEGIN/
+ off = false
+ when /RIPPER_PARAMS_DECL_END/
+ exit
+ when /ripper/
+ next if off
+ var = line.scan(/\w+/).last or next
+ base = var.sub(/ripper_/, '')
+ puts %"\#define #{base}\t\t(parser->ripper_#{base})"
+ end
+end
diff --git a/ext/ripper/tools/generate.rb b/ext/ripper/tools/generate.rb
new file mode 100644
index 0000000000..57ecac0b39
--- /dev/null
+++ b/ext/ripper/tools/generate.rb
@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+# $Id$
+
+require 'optparse'
+
+def main
+ mode = nil
+ ids1src = nil
+ ids2src = nil
+ output = nil
+
+ parser = @parser = OptionParser.new
+ parser.banner = "Usage: #{File.basename($0)} --mode=MODE [--ids1src=PATH] [--ids2src=PATH] [--output=PATH]"
+ parser.on('--mode=MODE', 'check, eventids1_h, eventids1, or eventids2table.') {|m|
+ mode = m
+ }
+ parser.on('--ids1src=PATH', 'A source file of event-IDs 1 (parse.y).') {|path|
+ ids1src = path
+ }
+ parser.on('--ids2src=PATH', 'A source file of event-IDs 2 (eventids2.c).') {|path|
+ ids2src = path
+ }
+ parser.on('--output=PATH', 'An output file.') {|path|
+ output = path
+ }
+ parser.on('--help', 'Prints this message and quit.') {
+ puts parser.help
+ exit true
+ }
+ begin
+ parser.parse!
+ rescue OptionParser::ParseError => err
+ usage err.message
+ end
+ usage 'no mode given' unless mode
+ case mode
+ when 'check'
+ usage 'no --ids1src' unless ids1src
+ usage 'no --ids2src' unless ids2src
+ h = read_ids1_with_locations(ids1src)
+ check_arity h
+ ids2 = read_ids2(ids2src)
+ common = h.keys & ids2
+ unless common.empty?
+ abort "event crash: #{common.join(' ')}"
+ end
+ exit 0
+ when 'eventids1_h'
+ usage 'no --ids1src' unless ids1src
+ result = generate_eventids1_h(read_ids1(ids1src))
+ when 'eventids1'
+ usage 'no --ids1src' unless ids1src
+ result = generate_eventids1(read_ids1(ids1src))
+ when 'eventids2table'
+ usage 'no --ids2src' unless ids2src
+ result = generate_eventids2_table(read_ids2(ids2src))
+ end
+ if output
+ File.open(output, 'w') {|f|
+ f.write result
+ }
+ else
+ puts result
+ end
+end
+
+def usage(msg)
+ $stderr.puts msg
+ $stderr.puts @parser.help
+ exit false
+end
+
+def generate_eventids1_h(ids)
+ buf = "".dup
+ buf << %Q[#ifndef RIPPER_EVENTIDS1\n]
+ buf << %Q[#define RIPPER_EVENTIDS1\n]
+ buf << %Q[\n]
+ buf << %Q[#define RIPPER_ID(n) ripper_parser_ids.id_ ## n\n]
+ buf << %Q[void ripper_init_eventids1(void);\n]
+ buf << %Q[void ripper_init_eventids1_table(VALUE self);\n]
+ buf << %Q[\n]
+ buf << %Q[struct ripper_parser_ids {\n]
+ ids.each do |id, arity|
+ buf << %Q[ ID id_#{id};\n]
+ end
+ buf << %Q[};\n]
+ buf << %Q[\n]
+ buf << %Q[#endif /* RIPPER_EVENTIDS1 */\n]
+ buf << %Q[\n]
+end
+
+def generate_eventids1(ids)
+ buf = "".dup
+ buf << %Q[#include "ruby/ruby.h"\n]
+ buf << %Q[#include "eventids1.h"\n]
+ buf << %Q[\n]
+ buf << %Q[struct ripper_parser_ids ripper_parser_ids;\n]
+ buf << %Q[\n]
+ buf << %Q[void\n]
+ buf << %Q[ripper_init_eventids1(void)\n]
+ buf << %Q[{\n]
+ buf << %Q[#define set_id1(name) RIPPER_ID(name) = rb_intern_const("on_"#name)\n]
+ ids.each do |id, arity|
+ buf << %Q[ set_id1(#{id});\n]
+ end
+ buf << %Q[}\n]
+ buf << %Q[\n]
+ buf << %Q[#define intern_sym(name) ID2SYM(rb_intern_const(name))\n]
+ buf << %Q[\n]
+ buf << %Q[void\n]
+ buf << %Q[ripper_init_eventids1_table(VALUE self)\n]
+ buf << %Q[{\n]
+ buf << %Q[ VALUE h = rb_hash_new();\n]
+ buf << %Q[ rb_define_const(self, "PARSER_EVENT_TABLE", h);\n]
+ ids.each do |id, arity|
+ buf << %Q[ rb_hash_aset(h, intern_sym("#{id}"), INT2FIX(#{arity}));\n]
+ end
+ buf << %Q[}\n]
+ buf
+end
+
+def generate_eventids2_table(ids)
+ buf = "".dup
+ buf << %Q[#include "ruby/ruby.h"\n]
+ buf << %Q[\n]
+ buf << %Q[#define intern_sym(name) ID2SYM(rb_intern_const(name))\n]
+ buf << %Q[\n]
+ buf << %Q[void\n]
+ buf << %Q[ripper_init_eventids2_table(VALUE self)\n]
+ buf << %Q[{\n]
+ buf << %Q[ VALUE h = rb_hash_new();\n]
+ buf << %Q[ rb_define_const(self, "SCANNER_EVENT_TABLE", h);\n]
+ ids.each do |id|
+ buf << %Q[ rb_hash_aset(h, intern_sym("#{id}"), INT2FIX(1));\n]
+ end
+ buf << %Q[}\n]
+ buf << %Q[\n]
+ buf << %Q[#define RIPPER_EVENTIDS2_TABLE_SIZE #{ids.size}\n]
+ buf
+end
+
+def read_ids1(path)
+ strip_locations(read_ids1_with_locations(path))
+end
+
+def strip_locations(h)
+ h.map {|event, list| [event, list.first[1]] }\
+ .sort_by {|event, arity| event.to_s }
+end
+
+def check_arity(h)
+ invalid = false
+ h.each do |event, list|
+ unless list.map {|line, arity| arity }.uniq.size == 1
+ invalid = true
+ locations = list.map {|line, a| "#{line}:#{a}" }.join(', ')
+ $stderr.puts "arity crash [event=#{event}]: #{locations}"
+ end
+ end
+ abort if invalid
+end
+
+require_relative "dsl"
+
+def read_ids1_with_locations(path)
+ h = {}
+ File.open(path) {|f|
+ f.each do |line|
+ next if /\A\#\s*define\s+dispatch/ =~ line
+ next if /ripper_dispatch/ =~ line
+ line.scan(/\bdispatch(\d)\((\w+)/) do |arity, event|
+ (h[event] ||= []).push [f.lineno, arity.to_i]
+ end
+ if gen = DSL.line?(line, f.lineno)
+ gen.events.each do |event, arity|
+ (h[event] ||= []).push [f.lineno, arity.to_i]
+ end
+ end
+ end
+ }
+ h
+end
+
+def read_ids2(path)
+ src = File.open(path) {|f| f.read}
+ ids2 = src.scan(/ID\s+ripper_id_(\w+)/).flatten.uniq.sort
+ diff = src.scan(/set_id2\((\w+)\);/).flatten - ids2
+ unless diff.empty?
+ abort "missing scanner IDs: #{diff}"
+ end
+ return ids2
+end
+
+main
diff --git a/ext/ripper/tools/preproc.rb b/ext/ripper/tools/preproc.rb
new file mode 100644
index 0000000000..5e8a6e0cb5
--- /dev/null
+++ b/ext/ripper/tools/preproc.rb
@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+# $Id$
+
+require 'optparse'
+
+def main
+ output = nil
+ template = nil
+ parser = OptionParser.new
+ parser.banner = "Usage: #{File.basename($0)} [--output=PATH] [--template=PATH] <parse.y>"
+ parser.on('--output=PATH', 'An output file.') {|path|
+ output = path
+ }
+ parser.on('--template=PATH', 'An template file.') {|path|
+ template = path
+ }
+ parser.on('--help', 'Prints this message and quit.') {
+ puts parser.help
+ exit true
+ }
+ begin
+ parser.parse!
+ rescue OptionParser::ParseError => err
+ warn err.message
+ abort parser.help
+ end
+ out = "".dup
+ if ARGV[0] == "-"
+ unless ARGV.size == 2
+ abort "wrong number of arguments (#{ARGV.size} for 2)"
+ end
+ process STDIN, out, ARGV[1], template
+ else
+ unless ARGV.size == 1
+ abort "wrong number of arguments (#{ARGV.size} for 1)"
+ end
+ File.open(ARGV[0]) {|f|
+ process f, out, ARGV[0], template
+ }
+ end
+ if output
+ File.write(output, out)
+ else
+ print out
+ end
+end
+
+def process(f, out, path, template)
+ prelude f, out
+ grammar f, out
+ usercode f, out, path, template
+end
+
+require_relative 'dsl'
+
+def generate_line(f, out)
+ while line = f.gets
+ case
+ when gen = DSL.line?(line, f.lineno)
+ out << gen.generate << "\n"
+ when line.start_with?("%%")
+ out << "%%\n"
+ break
+ else
+ out << yield(line)
+ end
+ end
+end
+
+def prelude(f, out)
+ @exprs = {}
+ generate_line(f, out) do |line|
+ if (/^enum lex_state_(?:bits|e) \{/ =~ line)..(/^\}/ =~ line)
+ case line
+ when /^\s*(EXPR_\w+),\s+\/\*(.+)\*\//
+ @exprs[$1.chomp("_bit")] = $2.strip
+ when /^\s*(EXPR_\w+)\s+=\s+(.+)$/
+ name = $1
+ val = $2.chomp(",")
+ @exprs[name] = "equals to " + (val.start_with?("(") ? "<tt>#{val}</tt>" : "+#{val}+")
+ end
+ end
+ line
+ end
+end
+
+def grammar(f, out)
+ generate_line(f, out) do |line|
+ case line
+ when %r</\*%%%\*/>
+ "#if 0\n"
+ when %r</\*%>
+ "#endif\n"
+ when %r<%\*/>
+ "\n"
+ else
+ line
+ end
+ end
+end
+
+def usercode(f, out, path, template)
+ require 'erb'
+ lineno = nil
+ src = nil
+ compiler = ERB::Compiler.new('%-')
+ compiler.put_cmd = compiler.insert_cmd = "out.<<"
+
+ if template
+ File.open(template) do |f|
+ out.clear
+ lineno = f.lineno
+ src, = compiler.compile(f.read)
+ path = template
+ end
+ else
+ lineno = f.lineno
+ src, = compiler.compile(f.read)
+ end
+
+ eval(src, binding, path, lineno)
+end
+
+main
diff --git a/ext/ripper/tools/strip.rb b/ext/ripper/tools/strip.rb
new file mode 100644
index 0000000000..23102f797a
--- /dev/null
+++ b/ext/ripper/tools/strip.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+last_is_void = false
+ARGF.each do |line|
+ case line
+ when /\A\s*\z/, /\A\#/
+ puts unless last_is_void
+ last_is_void = true
+ else
+ print line
+ last_is_void = false
+ end
+end
generated by cgit v1.2.3 (git 2.25.1) at 2026-06-04 00:25:12 +0000