summaryrefslogtreecommitdiff
path: root/ext/ripper
diff options
context:
space:
mode:
Diffstat (limited to 'ext/ripper')
-rw-r--r--ext/ripper/depend34
-rw-r--r--ext/ripper/eventids2.c3
-rw-r--r--ext/ripper/extconf.rb2
-rw-r--r--ext/ripper/lib/ripper/lexer.rb11
-rw-r--r--ext/ripper/ripper_init.c.tmpl293
-rw-r--r--ext/ripper/ripper_init.h2
-rw-r--r--ext/ripper/tools/dsl.rb173
-rw-r--r--ext/ripper/tools/generate.rb12
-rw-r--r--ext/ripper/tools/preproc.rb52
9 files changed, 365 insertions, 217 deletions
diff --git a/ext/ripper/depend b/ext/ripper/depend
index 531672d285..db83378a1d 100644
--- a/ext/ripper/depend
+++ b/ext/ripper/depend
@@ -1,7 +1,7 @@
GEN = $(srcdir)/tools/generate.rb
SRC1 = $(top_srcdir)/parse.y
SRC2 = $(srcdir)/eventids2.c
-BISON = $(BASERUBY) $(top_srcdir)/tool/lrama/exe/lrama
+LRAMA = $(BASERUBY) $(top_srcdir)/tool/lrama/exe/lrama
.SUFFIXES: .y
@@ -12,7 +12,7 @@ ripper.o: ripper.c
.y.c:
$(ECHO) compiling compiler $<
- $(Q) $(BISON) -t -v -o$@ -h$*.h - $< < $<
+ $(Q) $(LRAMA) -o$@ - $< < $<
all: check
static: check
@@ -181,6 +181,7 @@ eventids1.o: $(hdrdir)/ruby/internal/intern/re.h
eventids1.o: $(hdrdir)/ruby/internal/intern/ruby.h
eventids1.o: $(hdrdir)/ruby/internal/intern/select.h
eventids1.o: $(hdrdir)/ruby/internal/intern/select/largesize.h
+eventids1.o: $(hdrdir)/ruby/internal/intern/set.h
eventids1.o: $(hdrdir)/ruby/internal/intern/signal.h
eventids1.o: $(hdrdir)/ruby/internal/intern/sprintf.h
eventids1.o: $(hdrdir)/ruby/internal/intern/string.h
@@ -200,6 +201,7 @@ eventids1.o: $(hdrdir)/ruby/internal/special_consts.h
eventids1.o: $(hdrdir)/ruby/internal/static_assert.h
eventids1.o: $(hdrdir)/ruby/internal/stdalign.h
eventids1.o: $(hdrdir)/ruby/internal/stdbool.h
+eventids1.o: $(hdrdir)/ruby/internal/stdckdint.h
eventids1.o: $(hdrdir)/ruby/internal/symbol.h
eventids1.o: $(hdrdir)/ruby/internal/value.h
eventids1.o: $(hdrdir)/ruby/internal/value_type.h
@@ -351,6 +353,7 @@ eventids2.o: $(hdrdir)/ruby/internal/intern/re.h
eventids2.o: $(hdrdir)/ruby/internal/intern/ruby.h
eventids2.o: $(hdrdir)/ruby/internal/intern/select.h
eventids2.o: $(hdrdir)/ruby/internal/intern/select/largesize.h
+eventids2.o: $(hdrdir)/ruby/internal/intern/set.h
eventids2.o: $(hdrdir)/ruby/internal/intern/signal.h
eventids2.o: $(hdrdir)/ruby/internal/intern/sprintf.h
eventids2.o: $(hdrdir)/ruby/internal/intern/string.h
@@ -370,6 +373,7 @@ eventids2.o: $(hdrdir)/ruby/internal/special_consts.h
eventids2.o: $(hdrdir)/ruby/internal/static_assert.h
eventids2.o: $(hdrdir)/ruby/internal/stdalign.h
eventids2.o: $(hdrdir)/ruby/internal/stdbool.h
+eventids2.o: $(hdrdir)/ruby/internal/stdckdint.h
eventids2.o: $(hdrdir)/ruby/internal/symbol.h
eventids2.o: $(hdrdir)/ruby/internal/value.h
eventids2.o: $(hdrdir)/ruby/internal/value_type.h
@@ -470,6 +474,7 @@ ripper.o: $(hdrdir)/ruby/internal/core/rclass.h
ripper.o: $(hdrdir)/ruby/internal/core/rdata.h
ripper.o: $(hdrdir)/ruby/internal/core/rfile.h
ripper.o: $(hdrdir)/ruby/internal/core/rhash.h
+ripper.o: $(hdrdir)/ruby/internal/core/rmatch.h
ripper.o: $(hdrdir)/ruby/internal/core/robject.h
ripper.o: $(hdrdir)/ruby/internal/core/rregexp.h
ripper.o: $(hdrdir)/ruby/internal/core/rstring.h
@@ -530,6 +535,7 @@ ripper.o: $(hdrdir)/ruby/internal/intern/re.h
ripper.o: $(hdrdir)/ruby/internal/intern/ruby.h
ripper.o: $(hdrdir)/ruby/internal/intern/select.h
ripper.o: $(hdrdir)/ruby/internal/intern/select/largesize.h
+ripper.o: $(hdrdir)/ruby/internal/intern/set.h
ripper.o: $(hdrdir)/ruby/internal/intern/signal.h
ripper.o: $(hdrdir)/ruby/internal/intern/sprintf.h
ripper.o: $(hdrdir)/ruby/internal/intern/string.h
@@ -549,6 +555,7 @@ ripper.o: $(hdrdir)/ruby/internal/special_consts.h
ripper.o: $(hdrdir)/ruby/internal/static_assert.h
ripper.o: $(hdrdir)/ruby/internal/stdalign.h
ripper.o: $(hdrdir)/ruby/internal/stdbool.h
+ripper.o: $(hdrdir)/ruby/internal/stdckdint.h
ripper.o: $(hdrdir)/ruby/internal/symbol.h
ripper.o: $(hdrdir)/ruby/internal/value.h
ripper.o: $(hdrdir)/ruby/internal/value_type.h
@@ -560,23 +567,28 @@ ripper.o: $(hdrdir)/ruby/missing.h
ripper.o: $(hdrdir)/ruby/onigmo.h
ripper.o: $(hdrdir)/ruby/oniguruma.h
ripper.o: $(hdrdir)/ruby/ractor.h
+ripper.o: $(hdrdir)/ruby/re.h
ripper.o: $(hdrdir)/ruby/regex.h
ripper.o: $(hdrdir)/ruby/ruby.h
ripper.o: $(hdrdir)/ruby/st.h
ripper.o: $(hdrdir)/ruby/subst.h
ripper.o: $(hdrdir)/ruby/thread_native.h
ripper.o: $(hdrdir)/ruby/util.h
+ripper.o: $(hdrdir)/ruby/version.h
ripper.o: $(top_srcdir)/ccan/check_type/check_type.h
ripper.o: $(top_srcdir)/ccan/container_of/container_of.h
ripper.o: $(top_srcdir)/ccan/list/list.h
ripper.o: $(top_srcdir)/ccan/str/str.h
ripper.o: $(top_srcdir)/constant.h
+ripper.o: $(top_srcdir)/encindex.h
ripper.o: $(top_srcdir)/id_table.h
ripper.o: $(top_srcdir)/internal.h
ripper.o: $(top_srcdir)/internal/array.h
ripper.o: $(top_srcdir)/internal/basic_operators.h
ripper.o: $(top_srcdir)/internal/bignum.h
ripper.o: $(top_srcdir)/internal/bits.h
+ripper.o: $(top_srcdir)/internal/box.h
+ripper.o: $(top_srcdir)/internal/compar.h
ripper.o: $(top_srcdir)/internal/compile.h
ripper.o: $(top_srcdir)/internal/compilers.h
ripper.o: $(top_srcdir)/internal/complex.h
@@ -592,9 +604,12 @@ ripper.o: $(top_srcdir)/internal/parse.h
ripper.o: $(top_srcdir)/internal/rational.h
ripper.o: $(top_srcdir)/internal/re.h
ripper.o: $(top_srcdir)/internal/ruby_parser.h
+ripper.o: $(top_srcdir)/internal/sanitizers.h
ripper.o: $(top_srcdir)/internal/serial.h
+ripper.o: $(top_srcdir)/internal/set_table.h
ripper.o: $(top_srcdir)/internal/static_assert.h
ripper.o: $(top_srcdir)/internal/string.h
+ripper.o: $(top_srcdir)/internal/struct.h
ripper.o: $(top_srcdir)/internal/symbol.h
ripper.o: $(top_srcdir)/internal/thread.h
ripper.o: $(top_srcdir)/internal/variable.h
@@ -633,6 +648,7 @@ ripper_init.o: $(hdrdir)/ruby/backward.h
ripper_init.o: $(hdrdir)/ruby/backward/2/assume.h
ripper_init.o: $(hdrdir)/ruby/backward/2/attributes.h
ripper_init.o: $(hdrdir)/ruby/backward/2/bool.h
+ripper_init.o: $(hdrdir)/ruby/backward/2/gcc_version_since.h
ripper_init.o: $(hdrdir)/ruby/backward/2/inttypes.h
ripper_init.o: $(hdrdir)/ruby/backward/2/limits.h
ripper_init.o: $(hdrdir)/ruby/backward/2/long_long.h
@@ -764,6 +780,7 @@ ripper_init.o: $(hdrdir)/ruby/internal/intern/re.h
ripper_init.o: $(hdrdir)/ruby/internal/intern/ruby.h
ripper_init.o: $(hdrdir)/ruby/internal/intern/select.h
ripper_init.o: $(hdrdir)/ruby/internal/intern/select/largesize.h
+ripper_init.o: $(hdrdir)/ruby/internal/intern/set.h
ripper_init.o: $(hdrdir)/ruby/internal/intern/signal.h
ripper_init.o: $(hdrdir)/ruby/internal/intern/sprintf.h
ripper_init.o: $(hdrdir)/ruby/internal/intern/string.h
@@ -783,6 +800,7 @@ ripper_init.o: $(hdrdir)/ruby/internal/special_consts.h
ripper_init.o: $(hdrdir)/ruby/internal/static_assert.h
ripper_init.o: $(hdrdir)/ruby/internal/stdalign.h
ripper_init.o: $(hdrdir)/ruby/internal/stdbool.h
+ripper_init.o: $(hdrdir)/ruby/internal/stdckdint.h
ripper_init.o: $(hdrdir)/ruby/internal/symbol.h
ripper_init.o: $(hdrdir)/ruby/internal/value.h
ripper_init.o: $(hdrdir)/ruby/internal/value_type.h
@@ -796,12 +814,24 @@ ripper_init.o: $(hdrdir)/ruby/ruby.h
ripper_init.o: $(hdrdir)/ruby/st.h
ripper_init.o: $(hdrdir)/ruby/subst.h
ripper_init.o: $(top_srcdir)/internal.h
+ripper_init.o: $(top_srcdir)/internal/array.h
+ripper_init.o: $(top_srcdir)/internal/basic_operators.h
+ripper_init.o: $(top_srcdir)/internal/bignum.h
+ripper_init.o: $(top_srcdir)/internal/bits.h
+ripper_init.o: $(top_srcdir)/internal/compar.h
+ripper_init.o: $(top_srcdir)/internal/compilers.h
+ripper_init.o: $(top_srcdir)/internal/complex.h
+ripper_init.o: $(top_srcdir)/internal/fixnum.h
+ripper_init.o: $(top_srcdir)/internal/imemo.h
+ripper_init.o: $(top_srcdir)/internal/numeric.h
ripper_init.o: $(top_srcdir)/internal/parse.h
+ripper_init.o: $(top_srcdir)/internal/rational.h
ripper_init.o: $(top_srcdir)/internal/ruby_parser.h
ripper_init.o: $(top_srcdir)/internal/serial.h
ripper_init.o: $(top_srcdir)/internal/static_assert.h
ripper_init.o: $(top_srcdir)/internal/vm.h
ripper_init.o: $(top_srcdir)/node.h
+ripper_init.o: $(top_srcdir)/ruby_assert.h
ripper_init.o: $(top_srcdir)/rubyparser.h
ripper_init.o: eventids1.h
ripper_init.o: eventids2.h
diff --git a/ext/ripper/eventids2.c b/ext/ripper/eventids2.c
index ac54955857..87f2f588ec 100644
--- a/ext/ripper/eventids2.c
+++ b/ext/ripper/eventids2.c
@@ -65,6 +65,8 @@ static ripper_scanner_ids_t ripper_scanner_ids;
#include "eventids2table.c"
+STATIC_ASSERT(eventids2_table_size, RIPPER_EVENTIDS2_TABLE_SIZE == sizeof(ripper_scanner_ids)/sizeof(ID));
+
void
ripper_init_eventids2(void)
{
@@ -253,7 +255,6 @@ ripper_token2eventid(enum yytokentype tok)
[tRATIONAL] = O(rational),
[tREGEXP_BEG] = O(regexp_beg),
[tREGEXP_END] = O(regexp_end),
- [tRPAREN] = O(rparen),
[tRSHFT] = O(op),
[tSTAR] = O(op),
[tDSTAR] = O(op),
diff --git a/ext/ripper/extconf.rb b/ext/ripper/extconf.rb
index 8eb7a92148..c3c56c27c5 100644
--- a/ext/ripper/extconf.rb
+++ b/ext/ripper/extconf.rb
@@ -6,7 +6,7 @@ require 'rbconfig'
def main
$objs = %w(eventids1.o eventids2.o ripper.o ripper_init.o)
- $distcleanfiles.concat %w(ripper.y ripper.c ripper.h eventids1.c eventids1.h eventids2table.c ripper_init.c)
+ $distcleanfiles.concat %w(ripper.y ripper.c eventids1.c eventids1.h eventids2table.c ripper_init.c)
$cleanfiles.concat %w(ripper.E ripper.output y.output .eventids2-check)
$defs << '-DRIPPER'
$defs << '-DRIPPER_DEBUG' if $debug
diff --git a/ext/ripper/lib/ripper/lexer.rb b/ext/ripper/lib/ripper/lexer.rb
index 6a3c04af30..9b849dfeae 100644
--- a/ext/ripper/lib/ripper/lexer.rb
+++ b/ext/ripper/lib/ripper/lexer.rb
@@ -53,6 +53,7 @@ class Ripper
end
class Lexer < ::Ripper #:nodoc: internal use only
+ # :stopdoc:
class State
attr_reader :to_int, :to_s
@@ -67,7 +68,7 @@ class Ripper
when 0, :to_int
@to_int
when 1, :to_s
- @event
+ @to_s
else
nil
end
@@ -242,7 +243,12 @@ class Ripper
end
def on_error2(mesg, elem)
- @errors.push Elem.new(elem.pos, __callee__, elem.tok, elem.state, mesg)
+ if elem
+ elem = Elem.new(elem.pos, __callee__, elem.tok, elem.state, mesg)
+ else
+ elem = Elem.new([lineno(), column()], __callee__, token(), state(), mesg)
+ end
+ @errors.push elem
end
PARSER_EVENTS.grep(/_error\z/) do |e|
arity = PARSER_EVENT_TABLE.fetch(e)
@@ -253,6 +259,7 @@ class Ripper
(SCANNER_EVENTS.map {|event|:"on_#{event}"} - private_instance_methods(false)).each do |event|
alias_method event, :_push_token
end
+ # :startdoc:
end
# [EXPERIMENTAL]
diff --git a/ext/ripper/ripper_init.c.tmpl b/ext/ripper/ripper_init.c.tmpl
index 98f260f8b1..11e432423d 100644
--- a/ext/ripper/ripper_init.c.tmpl
+++ b/ext/ripper/ripper_init.c.tmpl
@@ -1,31 +1,56 @@
+%# -*- c -*-
#include "ruby/ruby.h"
#include "ruby/encoding.h"
#include "internal.h"
+#include "rubyparser.h"
+#define YYSTYPE_IS_DECLARED
+#include "parse.h"
#include "internal/parse.h"
#include "internal/ruby_parser.h"
#include "node.h"
-#include "rubyparser.h"
#include "eventids1.h"
-#define YYSTYPE_IS_DECLARED
-#include "parse.h"
#include "eventids2.h"
#include "ripper_init.h"
#define STR_NEW2(ptr) rb_enc_str_new((ptr),strlen(ptr),rb_ruby_parser_enc(p))
-#define NODE_RIPPER NODE_CDECL
#define RIPPER_VERSION "0.1.0"
ID id_warn, id_warning, id_gets, id_assoc;
+enum lex_type {
+ lex_type_str,
+ lex_type_io,
+ lex_type_generic,
+};
+
struct ripper {
rb_parser_t *p;
+ enum lex_type type;
+ union {
+ struct lex_pointer_string ptr_str;
+ VALUE val;
+ } data;
};
static void
ripper_parser_mark2(void *ptr)
{
struct ripper *r = (struct ripper*)ptr;
- if (r->p) ripper_parser_mark(r->p);
+ if (r->p) {
+ ripper_parser_mark(r->p);
+
+ switch (r->type) {
+ case lex_type_str:
+ rb_gc_mark(r->data.ptr_str.str);
+ break;
+ case lex_type_io:
+ rb_gc_mark(r->data.val);
+ break;
+ case lex_type_generic:
+ rb_gc_mark(r->data.val);
+ break;
+ }
+ }
}
static void
@@ -53,37 +78,18 @@ static const rb_data_type_t parser_data_type = {
0, 0, RUBY_TYPED_FREE_IMMEDIATELY
};
-ID
-ripper_get_id(VALUE v)
-{
- NODE *nd;
- if (!RB_TYPE_P(v, T_NODE)) return 0;
- nd = (NODE *)v;
- if (!nd_type_p(nd, NODE_RIPPER)) return 0;
- return nd->nd_vid;
-}
-
-VALUE
-ripper_get_value(VALUE v)
-{
- NODE *nd;
- if (UNDEF_P(v)) return Qnil;
- if (!RB_TYPE_P(v, T_NODE)) return v;
- nd = (NODE *)v;
- if (!nd_type_p(nd, NODE_RIPPER)) return Qnil;
- return nd->nd_rval;
-}
-
-static VALUE
-ripper_lex_get_generic(struct parser_params *p, VALUE src)
+static rb_parser_string_t *
+ripper_lex_get_generic(struct parser_params *p, rb_parser_input_data input, int line_count)
{
+ VALUE src = (VALUE)input;
VALUE line = rb_funcallv_public(src, id_gets, 0, 0);
- if (!NIL_P(line) && !RB_TYPE_P(line, T_STRING)) {
+ if (NIL_P(line)) return 0;
+ if (!RB_TYPE_P(line, T_STRING)) {
rb_raise(rb_eTypeError,
"gets returned %"PRIsVALUE" (expected String or nil)",
rb_obj_class(line));
}
- return line;
+ return rb_str_to_parser_string(p, line);
}
void
@@ -99,10 +105,19 @@ ripper_compile_error(struct parser_params *p, const char *fmt, ...)
ripper_error(p);
}
-static VALUE
-ripper_lex_io_get(struct parser_params *p, VALUE src)
+static rb_parser_string_t *
+ripper_lex_io_get(struct parser_params *p, rb_parser_input_data input, int line_count)
+{
+ VALUE src = (VALUE)input;
+ VALUE line = rb_io_gets(src);
+ if (NIL_P(line)) return 0;
+ return rb_str_to_parser_string(p, line);
+}
+
+static rb_parser_string_t *
+ripper_lex_get_str(struct parser_params *p, rb_parser_input_data input, int line_count)
{
- return rb_io_gets(src);
+ return rb_parser_lex_get_str(p, (struct lex_pointer_string *)input);
}
static VALUE
@@ -114,10 +129,8 @@ ripper_s_allocate(VALUE klass)
&parser_data_type, r);
#ifdef UNIVERSAL_PARSER
- rb_parser_config_t *config;
- config = rb_ruby_parser_config_new(ruby_xmalloc);
- rb_parser_config_initialize(config);
- r->p = rb_ruby_parser_allocate(config);
+ const rb_parser_config_t *config = rb_ruby_parser_config();
+ r->p = rb_ripper_parser_params_allocate(config);
#else
r->p = rb_ruby_ripper_parser_allocate();
#endif
@@ -125,6 +138,20 @@ ripper_s_allocate(VALUE klass)
return self;
}
+static struct parser_params *
+ripper_parser_params(VALUE self, bool initialized)
+{
+ struct ripper *r;
+ struct parser_params *p;
+
+ TypedData_Get_Struct(self, struct ripper, &parser_data_type, r);
+ p = r->p;
+ if (initialized && !rb_ruby_ripper_initialized_p(p)) {
+ rb_raise(rb_eArgError, "method called for uninitialized object");
+ }
+ return p;
+}
+
/*
* call-seq:
* ripper.error? -> Boolean
@@ -134,10 +161,9 @@ ripper_s_allocate(VALUE klass)
static VALUE
ripper_error_p(VALUE vparser)
{
- struct ripper *r;
+ struct parser_params *p = ripper_parser_params(vparser, false);
- TypedData_Get_Struct(vparser, struct ripper, &parser_data_type, r);
- return RBOOL(rb_ruby_parser_error_p(r->p));
+ return RBOOL(rb_ruby_parser_error_p(p));
}
/*
@@ -149,10 +175,9 @@ ripper_error_p(VALUE vparser)
static VALUE
ripper_parser_end_seen_p(VALUE vparser)
{
- struct ripper *r;
+ struct parser_params *p = ripper_parser_params(vparser, false);
- TypedData_Get_Struct(vparser, struct ripper, &parser_data_type, r);
- return RBOOL(rb_ruby_parser_end_seen_p(r->p));
+ return RBOOL(rb_ruby_parser_end_seen_p(p));
}
/*
@@ -164,10 +189,9 @@ ripper_parser_end_seen_p(VALUE vparser)
static VALUE
ripper_parser_encoding(VALUE vparser)
{
- struct ripper *r;
+ struct parser_params *p = ripper_parser_params(vparser, false);
- TypedData_Get_Struct(vparser, struct ripper, &parser_data_type, r);
- return rb_ruby_parser_encoding(r->p);
+ return rb_enc_from_encoding(rb_ruby_parser_encoding(p));
}
/*
@@ -179,10 +203,9 @@ ripper_parser_encoding(VALUE vparser)
static VALUE
ripper_parser_get_yydebug(VALUE self)
{
- struct ripper *r;
+ struct parser_params *p = ripper_parser_params(self, false);
- TypedData_Get_Struct(self, struct ripper, &parser_data_type, r);
- return RBOOL(rb_ruby_parser_get_yydebug(r->p));
+ return RBOOL(rb_ruby_parser_get_yydebug(p));
}
/*
@@ -194,10 +217,9 @@ ripper_parser_get_yydebug(VALUE self)
static VALUE
ripper_parser_set_yydebug(VALUE self, VALUE flag)
{
- struct ripper *r;
+ struct parser_params *p = ripper_parser_params(self, false);
- TypedData_Get_Struct(self, struct ripper, &parser_data_type, r);
- rb_ruby_parser_set_yydebug(r->p, RTEST(flag));
+ rb_ruby_parser_set_yydebug(p, RTEST(flag));
return flag;
}
@@ -210,10 +232,9 @@ ripper_parser_set_yydebug(VALUE self, VALUE flag)
static VALUE
ripper_parser_get_debug_output(VALUE self)
{
- struct ripper *r;
+ struct parser_params *p = ripper_parser_params(self, false);
- TypedData_Get_Struct(self, struct ripper, &parser_data_type, r);
- return rb_ruby_parser_debug_output(r->p);
+ return rb_ruby_parser_debug_output(p);
}
/*
@@ -225,13 +246,24 @@ ripper_parser_get_debug_output(VALUE self)
static VALUE
ripper_parser_set_debug_output(VALUE self, VALUE output)
{
- struct ripper *r;
+ struct parser_params *p = ripper_parser_params(self, false);
- TypedData_Get_Struct(self, struct ripper, &parser_data_type, r);
- rb_ruby_parser_set_debug_output(r->p, output);
+ rb_ruby_parser_set_debug_output(p, output);
return output;
}
+static int
+ripper_parser_dedent_string(struct parser_params *p, VALUE string, int width)
+{
+ int col;
+ rb_parser_string_t *str;
+ str = rb_str_to_parser_string(p, string);
+ col = rb_ruby_ripper_dedent_string(p, str, width);
+ rb_str_replace(string, rb_str_new_parser_string(str));
+ rb_parser_string_free(p, str);
+ return col;
+}
+
#ifdef UNIVERSAL_PARSER
struct dedent_string_arg {
struct parser_params *p;
@@ -247,16 +279,16 @@ parser_dedent_string0(VALUE a)
StringValue(arg->input);
wid = NUM2UINT(arg->width);
- col = rb_ruby_ripper_dedent_string(arg->p, arg->input, wid);
+ col = ripper_parser_dedent_string(arg->p, arg->input, wid);
return INT2NUM(col);
}
static VALUE
-parser_config_free(VALUE a)
+parser_free(VALUE a)
{
- rb_parser_config_t *config = (void *)a;
+ struct parser_params *p = (void *)a;
- rb_ruby_parser_config_free(config);
+ rb_ruby_parser_free(p);
return Qnil;
}
#endif
@@ -275,17 +307,14 @@ static VALUE
parser_dedent_string(VALUE self, VALUE input, VALUE width)
{
struct parser_params *p;
- rb_parser_config_t *config;
struct dedent_string_arg args;
- config = rb_ruby_parser_config_new(ruby_xmalloc);
- rb_parser_config_initialize(config);
- p = rb_ruby_parser_new(config);
+ p = rb_parser_params_new();
args.p = p;
args.input = input;
args.width = width;
- return rb_ensure(parser_dedent_string0, (VALUE)&args, parser_config_free, (VALUE)config);
+ return rb_ensure(parser_dedent_string0, (VALUE)&args, parser_free, (VALUE)p);
}
#else
static VALUE
@@ -295,7 +324,7 @@ parser_dedent_string(VALUE self, VALUE input, VALUE width)
StringValue(input);
wid = NUM2UINT(width);
- col = rb_ruby_ripper_dedent_string(0, input, wid);
+ col = ripper_parser_dedent_string(0, input, wid);
return INT2NUM(col);
}
#endif
@@ -316,25 +345,35 @@ ripper_initialize(int argc, VALUE *argv, VALUE self)
struct ripper *r;
struct parser_params *p;
VALUE src, fname, lineno;
- VALUE (*gets)(struct parser_params*,VALUE);
- VALUE input, sourcefile_string;
+ rb_parser_lex_gets_func *gets;
+ VALUE sourcefile_string;
const char *sourcefile;
int sourceline;
+ rb_parser_input_data input;
+ p = ripper_parser_params(self, false);
TypedData_Get_Struct(self, struct ripper, &parser_data_type, r);
- p = r->p;
rb_scan_args(argc, argv, "12", &src, &fname, &lineno);
if (RB_TYPE_P(src, T_FILE)) {
gets = ripper_lex_io_get;
+ r->type = lex_type_io;
+ r->data.val = src;
+ input = (rb_parser_input_data)src;
}
else if (rb_respond_to(src, id_gets)) {
gets = ripper_lex_get_generic;
+ r->type = lex_type_generic;
+ r->data.val = src;
+ input = (rb_parser_input_data)src;
}
else {
StringValue(src);
- gets = rb_ruby_ripper_lex_get_str;
+ gets = ripper_lex_get_str;
+ r->type = lex_type_str;
+ r->data.ptr_str.str = src;
+ r->data.ptr_str.ptr = 0;
+ input = (rb_parser_input_data)&r->data.ptr_str;
}
- input = src;
if (NIL_P(fname)) {
fname = STR_NEW2("(ripper)");
OBJ_FREEZE(fname);
@@ -357,12 +396,8 @@ ripper_initialize(int argc, VALUE *argv, VALUE self)
static VALUE
ripper_parse0(VALUE vparser)
{
- struct ripper *r;
- struct parser_params *p;
+ struct parser_params *p = ripper_parser_params(vparser, false);
- TypedData_Get_Struct(vparser, struct ripper, &parser_data_type, r);
- p = r->p;
- // RB_GC_GUARD(vparser);
rb_ruby_ripper_parse0(p);
return rb_ruby_parser_result(p);
}
@@ -370,10 +405,9 @@ ripper_parse0(VALUE vparser)
static VALUE
ripper_ensure(VALUE vparser)
{
- struct ripper *r;
+ struct parser_params *p = ripper_parser_params(vparser, false);
- TypedData_Get_Struct(vparser, struct ripper, &parser_data_type, r);
- rb_ruby_parser_set_parsing_thread(r->p, Qnil);
+ rb_ruby_parser_set_parsing_thread(p, Qnil);
return Qnil;
}
@@ -386,14 +420,9 @@ ripper_ensure(VALUE vparser)
static VALUE
ripper_parse(VALUE self)
{
- struct ripper *r;
- struct parser_params *p;
+ struct parser_params *p = ripper_parser_params(self, true);
+ VALUE result;
- TypedData_Get_Struct(self, struct ripper, &parser_data_type, r);
- p = r->p;
- if (!rb_ruby_ripper_initialized_p(p)) {
- rb_raise(rb_eArgError, "method called for uninitialized object");
- }
if (!NIL_P(rb_ruby_parser_parsing_thread(p))) {
if (rb_ruby_parser_parsing_thread(p) == rb_thread_current())
rb_raise(rb_eArgError, "Ripper#parse is not reentrant");
@@ -401,9 +430,10 @@ ripper_parse(VALUE self)
rb_raise(rb_eArgError, "Ripper#parse is not multithread-safe");
}
rb_ruby_parser_set_parsing_thread(p, rb_thread_current());
- rb_ensure(ripper_parse0, self, ripper_ensure, self);
+ result = rb_ensure(ripper_parse0, self, ripper_ensure, self);
+ RB_GC_GUARD(self);
- return rb_ruby_parser_result(p);
+ return result;
}
/*
@@ -416,15 +446,9 @@ ripper_parse(VALUE self)
static VALUE
ripper_column(VALUE self)
{
- struct ripper *r;
- struct parser_params *p;
+ struct parser_params *p = ripper_parser_params(self, true);
long col;
- TypedData_Get_Struct(self, struct ripper, &parser_data_type, r);
- p = r->p;
- if (!rb_ruby_ripper_initialized_p(p)) {
- rb_raise(rb_eArgError, "method called for uninitialized object");
- }
if (NIL_P(rb_ruby_parser_parsing_thread(p))) return Qnil;
col = rb_ruby_ripper_column(p);
return LONG2NUM(col);
@@ -439,14 +463,8 @@ ripper_column(VALUE self)
static VALUE
ripper_filename(VALUE self)
{
- struct ripper *r;
- struct parser_params *p;
+ struct parser_params *p = ripper_parser_params(self, true);
- TypedData_Get_Struct(self, struct ripper, &parser_data_type, r);
- p = r->p;
- if (!rb_ruby_ripper_initialized_p(p)) {
- rb_raise(rb_eArgError, "method called for uninitialized object");
- }
return rb_ruby_parser_ruby_sourcefile_string(p);
}
@@ -460,14 +478,8 @@ ripper_filename(VALUE self)
static VALUE
ripper_lineno(VALUE self)
{
- struct ripper *r;
- struct parser_params *p;
+ struct parser_params *p = ripper_parser_params(self, true);
- TypedData_Get_Struct(self, struct ripper, &parser_data_type, r);
- p = r->p;
- if (!rb_ruby_ripper_initialized_p(p)) {
- rb_raise(rb_eArgError, "method called for uninitialized object");
- }
if (NIL_P(rb_ruby_parser_parsing_thread(p))) return Qnil;
return INT2NUM(rb_ruby_parser_ruby_sourceline(p));
}
@@ -481,14 +493,8 @@ ripper_lineno(VALUE self)
static VALUE
ripper_state(VALUE self)
{
- struct ripper *r;
- struct parser_params *p;
+ struct parser_params *p = ripper_parser_params(self, true);
- TypedData_Get_Struct(self, struct ripper, &parser_data_type, r);
- p = r->p;
- if (!rb_ruby_ripper_initialized_p(p)) {
- rb_raise(rb_eArgError, "method called for uninitialized object");
- }
if (NIL_P(rb_ruby_parser_parsing_thread(p))) return Qnil;
return INT2NUM(rb_ruby_parser_lex_state(p));
}
@@ -502,19 +508,15 @@ ripper_state(VALUE self)
static VALUE
ripper_token(VALUE self)
{
- struct ripper *r;
- struct parser_params *p;
+ struct parser_params *p = ripper_parser_params(self, true);
long pos, len;
+ VALUE str;
- TypedData_Get_Struct(self, struct ripper, &parser_data_type, r);
- p = r->p;
- if (!rb_ruby_ripper_initialized_p(p)) {
- rb_raise(rb_eArgError, "method called for uninitialized object");
- }
if (NIL_P(rb_ruby_parser_parsing_thread(p))) return Qnil;
pos = rb_ruby_ripper_column(p);
len = rb_ruby_ripper_token_len(p);
- return rb_str_subseq(rb_ruby_ripper_lex_lastline(p), pos, len);
+ str = rb_str_new_parser_string(rb_ruby_ripper_lex_lastline(p));
+ return rb_str_subseq(str, pos, len);
}
#ifdef RIPPER_DEBUG
@@ -535,6 +537,37 @@ ripper_raw_value(VALUE self, VALUE obj)
{
return ULONG2NUM(obj);
}
+
+/* :nodoc: */
+static VALUE
+ripper_validate_object(VALUE self, VALUE x)
+{
+ if (x == Qfalse) return x;
+ if (x == Qtrue) return x;
+ if (NIL_P(x)) return x;
+ if (UNDEF_P(x))
+ rb_raise(rb_eArgError, "Qundef given");
+ if (FIXNUM_P(x)) return x;
+ if (SYMBOL_P(x)) return x;
+ switch (BUILTIN_TYPE(x)) {
+ case T_STRING:
+ case T_OBJECT:
+ case T_ARRAY:
+ case T_BIGNUM:
+ case T_FLOAT:
+ case T_COMPLEX:
+ case T_RATIONAL:
+ break;
+ default:
+ rb_raise(rb_eArgError, "wrong type of ruby object: %p (%s)",
+ (void *)x, rb_obj_classname(x));
+ }
+ if (!RBASIC_CLASS(x)) {
+ rb_raise(rb_eArgError, "hidden ruby object: %p (%s)",
+ (void *)x, rb_builtin_type_name(TYPE(x)));
+ }
+ return x;
+}
#endif
#ifdef UNIVERSAL_PARSER
@@ -563,17 +596,14 @@ static VALUE
ripper_lex_state_name(VALUE self, VALUE state)
{
struct parser_params *p;
- rb_parser_config_t *config;
struct lex_state_name_arg args;
- config = rb_ruby_parser_config_new(ruby_xmalloc);
- rb_parser_config_initialize(config);
- p = rb_ruby_parser_new(config);
+ p = rb_parser_params_new();
args.p = p;
args.state = state;
- return rb_ensure(lex_state_name0, (VALUE)&args, parser_config_free, (VALUE)config);
+ return rb_ensure(lex_state_name0, (VALUE)&args, parser_free, (VALUE)p);
}
#else
static VALUE
@@ -647,5 +677,4 @@ InitVM_ripper(void)
*/
rb_define_global_const("SCRIPT_LINES__", Qnil);
#endif
-
}
diff --git a/ext/ripper/ripper_init.h b/ext/ripper/ripper_init.h
index 82ff13b95f..9d228107d1 100644
--- a/ext/ripper/ripper_init.h
+++ b/ext/ripper/ripper_init.h
@@ -1,8 +1,6 @@
#ifndef RIPPER_INIT_H
#define RIPPER_INIT_H
-VALUE ripper_get_value(VALUE v);
-ID ripper_get_id(VALUE v);
PRINTF_ARGS(void ripper_compile_error(struct parser_params*, const char *fmt, ...), 2, 3);
#endif /* RIPPER_INIT_H */
diff --git a/ext/ripper/tools/dsl.rb b/ext/ripper/tools/dsl.rb
index 49ff51711f..38f859dd97 100644
--- a/ext/ripper/tools/dsl.rb
+++ b/ext/ripper/tools/dsl.rb
@@ -1,83 +1,177 @@
+# frozen_string_literal: true
+
# Simple DSL implementation for Ripper code generation
#
-# input: /*% ripper: stmts_add(stmts_new, void_stmt) %*/
+# input: /*% ripper: stmts_add!(stmts_new!, void_stmt!) %*/
# output:
# VALUE v1, v2;
# v1 = dispatch0(stmts_new);
# v2 = dispatch0(void_stmt);
# $$ = dispatch2(stmts_add, v1, v2);
-
-$dollar = "$$"
-alias $$ $dollar
+#
+# - The code must be a single line.
+#
+# - The code is basically Ruby code, even if it appears like in C and
+# the result will be processed as C. e.g., comments need to be in
+# Ruby style.
class DSL
- def initialize(code, options)
+ TAG_PATTERN = /(?><[a-zA-Z0-9_]+>)/.source
+ NAME_PATTERN = /(?>\$|\d+|[a-zA-Z_][a-zA-Z0-9_]*|\[[a-zA-Z_.][-a-zA-Z0-9_.]*\])(?>(?:\.|->)[a-zA-Z_][a-zA-Z0-9_]*)*/.source
+ NOT_REF_PATTERN = /(?>\#.*|[^\"$@]*|"(?>\\.|[^\"])*")/.source
+
+ def self.line?(line, lineno = nil, indent: nil)
+ if %r<(?<space>\s*)/\*% *ripper(?:\[(?<option>.*?)\])?: *(?<code>.*?) *%\*/> =~ line
+ new(code, comma_split(option), lineno, indent: indent || space)
+ end
+ end
+
+ def self.comma_split(str)
+ str or return []
+ str.scan(/(([^(,)]+|\((?:,|\g<0>)*\))+)/).map(&:first)
+ end
+
+ using Module.new {
+ refine Array do
+ def to_s
+ if empty?
+ "rb_ary_new()"
+ else
+ "rb_ary_new_from_args(#{size}, #{map(&:to_s).join(', ')})"
+ end
+ end
+ end
+ }
+
+ class Var
+ class Table < Hash
+ def initialize(&block)
+ super() {|tbl, arg|
+ tbl.fetch(arg, &block)
+ }
+ end
+
+ def fetch(arg, &block)
+ super {
+ self[arg] = Var.new(self, arg, &block)
+ }
+ end
+
+ def add(&block)
+ v = new_var
+ self[v] = Var.new(self, v, &block)
+ end
+
+ def defined?(name)
+ name = name.to_s
+ any? {|_, v| v.var == name}
+ end
+
+ def new_var
+ "v#{size+1}"
+ end
+ end
+
+ attr_reader :var, :value
+
+ PRETTY_PRINT_INSTANCE_VARIABLES = instance_methods(false).freeze
+
+ def pretty_print_instance_variables
+ PRETTY_PRINT_INSTANCE_VARIABLES
+ end
+
+ alias to_s var
+
+ def initialize(table, arg, &block)
+ @var = table.new_var
+ @value = yield arg
+ @table = table
+ end
+
+ # Indexing.
+ #
+ # $:1 -> v1=get_value($:1)
+ # $:1[0] -> rb_ary_entry(v1, 0)
+ # $:1[0..1] -> [rb_ary_entry(v1, 0), rb_ary_entry(v1, 1)]
+ # *$:1[0..1] -> rb_ary_entry(v1, 0), rb_ary_entry(v1, 1)
+ #
+ # Splat needs `[range]` because `Var` does not have the length info.
+ def [](idx)
+ if ::Range === idx
+ idx.map {|i| self[i]}
+ else
+ @table.fetch("#@var[#{idx}]") {"rb_ary_entry(#{@var}, #{idx})"}
+ end
+ end
+ end
+
+ def initialize(code, options, lineno = nil, indent: "\t\t\t")
+ @lineno = lineno
+ @indent = indent
@events = {}
@error = options.include?("error")
- @brace = options.include?("brace")
if options.include?("final")
@final = "p->result"
else
- @final = (options.grep(/\A\$(?:\$|\d+)\z/)[0] || "$$")
+ @final = (options.grep(/\A\$#{NAME_PATTERN}\z/o)[0] || "p->s_lvalue")
end
- @vars = 0
- # create $1 == "$1", $2 == "$2", ...
- s = (1..20).map {|n| "$#{n}"}
- re = Array.new(s.size, "([^\0]+)")
- /#{re.join("\0")}/ =~ s.join("\0")
+ bind = dsl_binding
+ @var_table = Var::Table.new {|arg| "get_value(#{arg})"}
+ code = code.gsub(%r[\G#{NOT_REF_PATTERN}\K(\$|\$:|@)#{TAG_PATTERN}?#{NAME_PATTERN}]o) {
+ if (arg = $&) == "$:$"
+ '"p->s_lvalue"'
+ elsif arg.start_with?("$:")
+ "(#{@var_table[arg]}=@var_table[#{arg.dump}])"
+ else
+ arg.dump
+ end
+ }
+ @last_value = bind.eval(code)
+ rescue SyntaxError
+ $stderr.puts "error on line #{@lineno}" if @lineno
+ raise
+ end
+ def dsl_binding(p = "p")
# struct parser_params *p
- p = p = "p"
-
- @code = ""
- @last_value = eval(code)
+ binding
end
attr_reader :events
undef lambda
undef hash
- undef class
+ undef :class
def generate
- s = "#@code#@final=#@last_value;"
- s = "{VALUE #{ (1..@vars).map {|v| "v#{ v }" }.join(",") };#{ s }}" if @vars > 0
+ s = "#@final=#@last_value;"
s << "ripper_error(p);" if @error
- s = "{#{ s }}" if @brace
- "\t\t\t#{s}"
- end
-
- def new_var
- "v#{ @vars += 1 }"
- end
-
- def opt_event(event, default, addend)
- add_event(event, [default, addend], true)
+ unless @var_table.empty?
+ vars = @var_table.map {|_, v| "#{v.var}=#{v.value}"}.join(", ")
+ s = "VALUE #{ vars }; #{ s }"
+ end
+ "#{@indent}{#{s}}"
end
- def add_event(event, args, qundef_check = false)
+ def add_event(event, args)
event = event.to_s.sub(/!\z/, "")
@events[event] = args.size
vars = []
args.each do |arg|
- vars << v = new_var
- @code << "#{ v }=#{ arg };"
+ arg = @var_table.add {arg} unless Var === arg
+ vars << arg
end
- v = new_var
- d = "dispatch#{ args.size }(#{ [event, *vars].join(",") })"
- d = "#{ vars.last }==Qundef ? #{ vars.first } : #{ d }" if qundef_check
- @code << "#{ v }=#{ d };"
- v
+ @var_table.add {"dispatch#{ args.size }(#{ [event, *vars].join(",") })"}
end
def method_missing(event, *args)
if event.to_s =~ /!\z/
add_event(event, args)
- elsif args.empty? and /\Aid[A-Z_]/ =~ event.to_s
+ elsif args.empty? and (/\Aid[A-Z_]/ =~ event or @var_table.defined?(event))
event
else
- "#{ event }(#{ args.join(", ") })"
+ "#{ event }(#{ args.map(&:to_s).join(", ") })"
end
end
@@ -85,4 +179,3 @@ class DSL
name
end
end
-
diff --git a/ext/ripper/tools/generate.rb b/ext/ripper/tools/generate.rb
index bb64d2fe8b..57ecac0b39 100644
--- a/ext/ripper/tools/generate.rb
+++ b/ext/ripper/tools/generate.rb
@@ -75,6 +75,7 @@ def generate_eventids1_h(ids)
buf << %Q[#ifndef RIPPER_EVENTIDS1\n]
buf << %Q[#define RIPPER_EVENTIDS1\n]
buf << %Q[\n]
+ buf << %Q[#define RIPPER_ID(n) ripper_parser_ids.id_ ## n\n]
buf << %Q[void ripper_init_eventids1(void);\n]
buf << %Q[void ripper_init_eventids1_table(VALUE self);\n]
buf << %Q[\n]
@@ -84,9 +85,6 @@ def generate_eventids1_h(ids)
end
buf << %Q[};\n]
buf << %Q[\n]
- ids.each do |id, arity|
- buf << %Q[#define ripper_id_#{id} ripper_parser_ids.id_#{id}\n]
- end
buf << %Q[#endif /* RIPPER_EVENTIDS1 */\n]
buf << %Q[\n]
end
@@ -101,7 +99,7 @@ def generate_eventids1(ids)
buf << %Q[void\n]
buf << %Q[ripper_init_eventids1(void)\n]
buf << %Q[{\n]
- buf << %Q[#define set_id1(name) ripper_id_##name = rb_intern_const("on_"#name)\n]
+ buf << %Q[#define set_id1(name) RIPPER_ID(name) = rb_intern_const("on_"#name)\n]
ids.each do |id, arity|
buf << %Q[ set_id1(#{id});\n]
end
@@ -136,6 +134,8 @@ def generate_eventids2_table(ids)
buf << %Q[ rb_hash_aset(h, intern_sym("#{id}"), INT2FIX(1));\n]
end
buf << %Q[}\n]
+ buf << %Q[\n]
+ buf << %Q[#define RIPPER_EVENTIDS2_TABLE_SIZE #{ids.size}\n]
buf
end
@@ -171,9 +171,7 @@ def read_ids1_with_locations(path)
line.scan(/\bdispatch(\d)\((\w+)/) do |arity, event|
(h[event] ||= []).push [f.lineno, arity.to_i]
end
- if line =~ %r</\*% *ripper(?:\[(.*?)\])?: *(.*?) *%\*/>
- gen = DSL.new($2, ($1 || "").split(","))
- gen.generate
+ if gen = DSL.line?(line, f.lineno)
gen.events.each do |event, arity|
(h[event] ||= []).push [f.lineno, arity.to_i]
end
diff --git a/ext/ripper/tools/preproc.rb b/ext/ripper/tools/preproc.rb
index 07ef627cf3..5e8a6e0cb5 100644
--- a/ext/ripper/tools/preproc.rb
+++ b/ext/ripper/tools/preproc.rb
@@ -51,28 +51,26 @@ def process(f, out, path, template)
usercode f, out, path, template
end
-def prelude(f, out)
- @exprs = {}
- lex_state_def = false
+require_relative 'dsl'
+
+def generate_line(f, out)
while line = f.gets
- case line
- when /\A%%/
+ case
+ when gen = DSL.line?(line, f.lineno)
+ out << gen.generate << "\n"
+ when line.start_with?("%%")
out << "%%\n"
- return
- when /\A%token/, /\A} <node>/
- out << line.sub(/<\w+>/, '<val>')
- when /\A%type/
- out << line.sub(/<\w+>/, '<val>')
- when /^enum lex_state_(?:bits|e) \{/
- lex_state_def = true
- out << line
- when /^\}/
- lex_state_def = false
- out << line
+ break
else
- out << line
+ out << yield(line)
end
- if lex_state_def
+ end
+end
+
+def prelude(f, out)
+ @exprs = {}
+ generate_line(f, out) do |line|
+ if (/^enum lex_state_(?:bits|e) \{/ =~ line)..(/^\}/ =~ line)
case line
when /^\s*(EXPR_\w+),\s+\/\*(.+)\*\//
@exprs[$1.chomp("_bit")] = $2.strip
@@ -82,27 +80,21 @@ def prelude(f, out)
@exprs[name] = "equals to " + (val.start_with?("(") ? "<tt>#{val}</tt>" : "+#{val}+")
end
end
+ line
end
end
-require_relative "dsl"
-
def grammar(f, out)
- while line = f.gets
+ generate_line(f, out) do |line|
case line
- when %r</\*% *ripper(?:\[(.*?)\])?: *(.*?) *%\*/>
- out << DSL.new($2, ($1 || "").split(",")).generate << "\n"
when %r</\*%%%\*/>
- out << "#if 0\n"
+ "#if 0\n"
when %r</\*%>
- out << "#endif\n"
+ "#endif\n"
when %r<%\*/>
- out << "\n"
- when /\A%%/
- out << "%%\n"
- return
+ "\n"
else
- out << line
+ line
end
end
end
generated by cgit v1.2.3 (git 2.25.1) at 2026-06-04 18:59:05 +0000