summaryrefslogtreecommitdiff
path: root/ext/ripper
diff options
context:
space:
mode:
Diffstat (limited to 'ext/ripper')
-rw-r--r--ext/ripper/depend31
-rw-r--r--ext/ripper/eventids2.c3
-rw-r--r--ext/ripper/lib/ripper/lexer.rb11
-rw-r--r--ext/ripper/ripper_init.c.tmpl180
-rw-r--r--ext/ripper/ripper_init.h2
-rw-r--r--ext/ripper/tools/dsl.rb158
-rw-r--r--ext/ripper/tools/generate.rb12
-rw-r--r--ext/ripper/tools/preproc.rb51
8 files changed, 312 insertions, 136 deletions
diff --git a/ext/ripper/depend b/ext/ripper/depend
index a18b67abe3..db83378a1d 100644
--- a/ext/ripper/depend
+++ b/ext/ripper/depend
@@ -1,7 +1,7 @@
GEN = $(srcdir)/tools/generate.rb
SRC1 = $(top_srcdir)/parse.y
SRC2 = $(srcdir)/eventids2.c
-BISON = $(BASERUBY) $(top_srcdir)/tool/lrama/exe/lrama
+LRAMA = $(BASERUBY) $(top_srcdir)/tool/lrama/exe/lrama
.SUFFIXES: .y
@@ -12,7 +12,7 @@ ripper.o: ripper.c
.y.c:
$(ECHO) compiling compiler $<
- $(Q) $(BISON) -t -v -o$@ - $< < $<
+ $(Q) $(LRAMA) -o$@ - $< < $<
all: check
static: check
@@ -181,6 +181,7 @@ eventids1.o: $(hdrdir)/ruby/internal/intern/re.h
eventids1.o: $(hdrdir)/ruby/internal/intern/ruby.h
eventids1.o: $(hdrdir)/ruby/internal/intern/select.h
eventids1.o: $(hdrdir)/ruby/internal/intern/select/largesize.h
+eventids1.o: $(hdrdir)/ruby/internal/intern/set.h
eventids1.o: $(hdrdir)/ruby/internal/intern/signal.h
eventids1.o: $(hdrdir)/ruby/internal/intern/sprintf.h
eventids1.o: $(hdrdir)/ruby/internal/intern/string.h
@@ -200,6 +201,7 @@ eventids1.o: $(hdrdir)/ruby/internal/special_consts.h
eventids1.o: $(hdrdir)/ruby/internal/static_assert.h
eventids1.o: $(hdrdir)/ruby/internal/stdalign.h
eventids1.o: $(hdrdir)/ruby/internal/stdbool.h
+eventids1.o: $(hdrdir)/ruby/internal/stdckdint.h
eventids1.o: $(hdrdir)/ruby/internal/symbol.h
eventids1.o: $(hdrdir)/ruby/internal/value.h
eventids1.o: $(hdrdir)/ruby/internal/value_type.h
@@ -351,6 +353,7 @@ eventids2.o: $(hdrdir)/ruby/internal/intern/re.h
eventids2.o: $(hdrdir)/ruby/internal/intern/ruby.h
eventids2.o: $(hdrdir)/ruby/internal/intern/select.h
eventids2.o: $(hdrdir)/ruby/internal/intern/select/largesize.h
+eventids2.o: $(hdrdir)/ruby/internal/intern/set.h
eventids2.o: $(hdrdir)/ruby/internal/intern/signal.h
eventids2.o: $(hdrdir)/ruby/internal/intern/sprintf.h
eventids2.o: $(hdrdir)/ruby/internal/intern/string.h
@@ -370,6 +373,7 @@ eventids2.o: $(hdrdir)/ruby/internal/special_consts.h
eventids2.o: $(hdrdir)/ruby/internal/static_assert.h
eventids2.o: $(hdrdir)/ruby/internal/stdalign.h
eventids2.o: $(hdrdir)/ruby/internal/stdbool.h
+eventids2.o: $(hdrdir)/ruby/internal/stdckdint.h
eventids2.o: $(hdrdir)/ruby/internal/symbol.h
eventids2.o: $(hdrdir)/ruby/internal/value.h
eventids2.o: $(hdrdir)/ruby/internal/value_type.h
@@ -470,6 +474,7 @@ ripper.o: $(hdrdir)/ruby/internal/core/rclass.h
ripper.o: $(hdrdir)/ruby/internal/core/rdata.h
ripper.o: $(hdrdir)/ruby/internal/core/rfile.h
ripper.o: $(hdrdir)/ruby/internal/core/rhash.h
+ripper.o: $(hdrdir)/ruby/internal/core/rmatch.h
ripper.o: $(hdrdir)/ruby/internal/core/robject.h
ripper.o: $(hdrdir)/ruby/internal/core/rregexp.h
ripper.o: $(hdrdir)/ruby/internal/core/rstring.h
@@ -530,6 +535,7 @@ ripper.o: $(hdrdir)/ruby/internal/intern/re.h
ripper.o: $(hdrdir)/ruby/internal/intern/ruby.h
ripper.o: $(hdrdir)/ruby/internal/intern/select.h
ripper.o: $(hdrdir)/ruby/internal/intern/select/largesize.h
+ripper.o: $(hdrdir)/ruby/internal/intern/set.h
ripper.o: $(hdrdir)/ruby/internal/intern/signal.h
ripper.o: $(hdrdir)/ruby/internal/intern/sprintf.h
ripper.o: $(hdrdir)/ruby/internal/intern/string.h
@@ -549,6 +555,7 @@ ripper.o: $(hdrdir)/ruby/internal/special_consts.h
ripper.o: $(hdrdir)/ruby/internal/static_assert.h
ripper.o: $(hdrdir)/ruby/internal/stdalign.h
ripper.o: $(hdrdir)/ruby/internal/stdbool.h
+ripper.o: $(hdrdir)/ruby/internal/stdckdint.h
ripper.o: $(hdrdir)/ruby/internal/symbol.h
ripper.o: $(hdrdir)/ruby/internal/value.h
ripper.o: $(hdrdir)/ruby/internal/value_type.h
@@ -560,6 +567,7 @@ ripper.o: $(hdrdir)/ruby/missing.h
ripper.o: $(hdrdir)/ruby/onigmo.h
ripper.o: $(hdrdir)/ruby/oniguruma.h
ripper.o: $(hdrdir)/ruby/ractor.h
+ripper.o: $(hdrdir)/ruby/re.h
ripper.o: $(hdrdir)/ruby/regex.h
ripper.o: $(hdrdir)/ruby/ruby.h
ripper.o: $(hdrdir)/ruby/st.h
@@ -572,12 +580,15 @@ ripper.o: $(top_srcdir)/ccan/container_of/container_of.h
ripper.o: $(top_srcdir)/ccan/list/list.h
ripper.o: $(top_srcdir)/ccan/str/str.h
ripper.o: $(top_srcdir)/constant.h
+ripper.o: $(top_srcdir)/encindex.h
ripper.o: $(top_srcdir)/id_table.h
ripper.o: $(top_srcdir)/internal.h
ripper.o: $(top_srcdir)/internal/array.h
ripper.o: $(top_srcdir)/internal/basic_operators.h
ripper.o: $(top_srcdir)/internal/bignum.h
ripper.o: $(top_srcdir)/internal/bits.h
+ripper.o: $(top_srcdir)/internal/box.h
+ripper.o: $(top_srcdir)/internal/compar.h
ripper.o: $(top_srcdir)/internal/compile.h
ripper.o: $(top_srcdir)/internal/compilers.h
ripper.o: $(top_srcdir)/internal/complex.h
@@ -593,9 +604,12 @@ ripper.o: $(top_srcdir)/internal/parse.h
ripper.o: $(top_srcdir)/internal/rational.h
ripper.o: $(top_srcdir)/internal/re.h
ripper.o: $(top_srcdir)/internal/ruby_parser.h
+ripper.o: $(top_srcdir)/internal/sanitizers.h
ripper.o: $(top_srcdir)/internal/serial.h
+ripper.o: $(top_srcdir)/internal/set_table.h
ripper.o: $(top_srcdir)/internal/static_assert.h
ripper.o: $(top_srcdir)/internal/string.h
+ripper.o: $(top_srcdir)/internal/struct.h
ripper.o: $(top_srcdir)/internal/symbol.h
ripper.o: $(top_srcdir)/internal/thread.h
ripper.o: $(top_srcdir)/internal/variable.h
@@ -634,6 +648,7 @@ ripper_init.o: $(hdrdir)/ruby/backward.h
ripper_init.o: $(hdrdir)/ruby/backward/2/assume.h
ripper_init.o: $(hdrdir)/ruby/backward/2/attributes.h
ripper_init.o: $(hdrdir)/ruby/backward/2/bool.h
+ripper_init.o: $(hdrdir)/ruby/backward/2/gcc_version_since.h
ripper_init.o: $(hdrdir)/ruby/backward/2/inttypes.h
ripper_init.o: $(hdrdir)/ruby/backward/2/limits.h
ripper_init.o: $(hdrdir)/ruby/backward/2/long_long.h
@@ -765,6 +780,7 @@ ripper_init.o: $(hdrdir)/ruby/internal/intern/re.h
ripper_init.o: $(hdrdir)/ruby/internal/intern/ruby.h
ripper_init.o: $(hdrdir)/ruby/internal/intern/select.h
ripper_init.o: $(hdrdir)/ruby/internal/intern/select/largesize.h
+ripper_init.o: $(hdrdir)/ruby/internal/intern/set.h
ripper_init.o: $(hdrdir)/ruby/internal/intern/signal.h
ripper_init.o: $(hdrdir)/ruby/internal/intern/sprintf.h
ripper_init.o: $(hdrdir)/ruby/internal/intern/string.h
@@ -784,6 +800,7 @@ ripper_init.o: $(hdrdir)/ruby/internal/special_consts.h
ripper_init.o: $(hdrdir)/ruby/internal/static_assert.h
ripper_init.o: $(hdrdir)/ruby/internal/stdalign.h
ripper_init.o: $(hdrdir)/ruby/internal/stdbool.h
+ripper_init.o: $(hdrdir)/ruby/internal/stdckdint.h
ripper_init.o: $(hdrdir)/ruby/internal/symbol.h
ripper_init.o: $(hdrdir)/ruby/internal/value.h
ripper_init.o: $(hdrdir)/ruby/internal/value_type.h
@@ -798,13 +815,23 @@ ripper_init.o: $(hdrdir)/ruby/st.h
ripper_init.o: $(hdrdir)/ruby/subst.h
ripper_init.o: $(top_srcdir)/internal.h
ripper_init.o: $(top_srcdir)/internal/array.h
+ripper_init.o: $(top_srcdir)/internal/basic_operators.h
+ripper_init.o: $(top_srcdir)/internal/bignum.h
+ripper_init.o: $(top_srcdir)/internal/bits.h
+ripper_init.o: $(top_srcdir)/internal/compar.h
+ripper_init.o: $(top_srcdir)/internal/compilers.h
+ripper_init.o: $(top_srcdir)/internal/complex.h
+ripper_init.o: $(top_srcdir)/internal/fixnum.h
ripper_init.o: $(top_srcdir)/internal/imemo.h
+ripper_init.o: $(top_srcdir)/internal/numeric.h
ripper_init.o: $(top_srcdir)/internal/parse.h
+ripper_init.o: $(top_srcdir)/internal/rational.h
ripper_init.o: $(top_srcdir)/internal/ruby_parser.h
ripper_init.o: $(top_srcdir)/internal/serial.h
ripper_init.o: $(top_srcdir)/internal/static_assert.h
ripper_init.o: $(top_srcdir)/internal/vm.h
ripper_init.o: $(top_srcdir)/node.h
+ripper_init.o: $(top_srcdir)/ruby_assert.h
ripper_init.o: $(top_srcdir)/rubyparser.h
ripper_init.o: eventids1.h
ripper_init.o: eventids2.h
diff --git a/ext/ripper/eventids2.c b/ext/ripper/eventids2.c
index ac54955857..87f2f588ec 100644
--- a/ext/ripper/eventids2.c
+++ b/ext/ripper/eventids2.c
@@ -65,6 +65,8 @@ static ripper_scanner_ids_t ripper_scanner_ids;
#include "eventids2table.c"
+STATIC_ASSERT(eventids2_table_size, RIPPER_EVENTIDS2_TABLE_SIZE == sizeof(ripper_scanner_ids)/sizeof(ID));
+
void
ripper_init_eventids2(void)
{
@@ -253,7 +255,6 @@ ripper_token2eventid(enum yytokentype tok)
[tRATIONAL] = O(rational),
[tREGEXP_BEG] = O(regexp_beg),
[tREGEXP_END] = O(regexp_end),
- [tRPAREN] = O(rparen),
[tRSHFT] = O(op),
[tSTAR] = O(op),
[tDSTAR] = O(op),
diff --git a/ext/ripper/lib/ripper/lexer.rb b/ext/ripper/lib/ripper/lexer.rb
index 6a3c04af30..9b849dfeae 100644
--- a/ext/ripper/lib/ripper/lexer.rb
+++ b/ext/ripper/lib/ripper/lexer.rb
@@ -53,6 +53,7 @@ class Ripper
end
class Lexer < ::Ripper #:nodoc: internal use only
+ # :stopdoc:
class State
attr_reader :to_int, :to_s
@@ -67,7 +68,7 @@ class Ripper
when 0, :to_int
@to_int
when 1, :to_s
- @event
+ @to_s
else
nil
end
@@ -242,7 +243,12 @@ class Ripper
end
def on_error2(mesg, elem)
- @errors.push Elem.new(elem.pos, __callee__, elem.tok, elem.state, mesg)
+ if elem
+ elem = Elem.new(elem.pos, __callee__, elem.tok, elem.state, mesg)
+ else
+ elem = Elem.new([lineno(), column()], __callee__, token(), state(), mesg)
+ end
+ @errors.push elem
end
PARSER_EVENTS.grep(/_error\z/) do |e|
arity = PARSER_EVENT_TABLE.fetch(e)
@@ -253,6 +259,7 @@ class Ripper
(SCANNER_EVENTS.map {|event|:"on_#{event}"} - private_instance_methods(false)).each do |event|
alias_method event, :_push_token
end
+ # :startdoc:
end
# [EXPERIMENTAL]
diff --git a/ext/ripper/ripper_init.c.tmpl b/ext/ripper/ripper_init.c.tmpl
index 5c1a4e5bb5..11e432423d 100644
--- a/ext/ripper/ripper_init.c.tmpl
+++ b/ext/ripper/ripper_init.c.tmpl
@@ -2,14 +2,13 @@
#include "ruby/ruby.h"
#include "ruby/encoding.h"
#include "internal.h"
-#include "internal/imemo.h" /* needed by ruby_parser.h */
+#include "rubyparser.h"
+#define YYSTYPE_IS_DECLARED
+#include "parse.h"
#include "internal/parse.h"
#include "internal/ruby_parser.h"
#include "node.h"
-#include "rubyparser.h"
#include "eventids1.h"
-#define YYSTYPE_IS_DECLARED
-#include "parse.h"
#include "eventids2.h"
#include "ripper_init.h"
@@ -18,15 +17,40 @@
ID id_warn, id_warning, id_gets, id_assoc;
+enum lex_type {
+ lex_type_str,
+ lex_type_io,
+ lex_type_generic,
+};
+
struct ripper {
rb_parser_t *p;
+ enum lex_type type;
+ union {
+ struct lex_pointer_string ptr_str;
+ VALUE val;
+ } data;
};
static void
ripper_parser_mark2(void *ptr)
{
struct ripper *r = (struct ripper*)ptr;
- if (r->p) ripper_parser_mark(r->p);
+ if (r->p) {
+ ripper_parser_mark(r->p);
+
+ switch (r->type) {
+ case lex_type_str:
+ rb_gc_mark(r->data.ptr_str.str);
+ break;
+ case lex_type_io:
+ rb_gc_mark(r->data.val);
+ break;
+ case lex_type_generic:
+ rb_gc_mark(r->data.val);
+ break;
+ }
+ }
}
static void
@@ -54,37 +78,18 @@ static const rb_data_type_t parser_data_type = {
0, 0, RUBY_TYPED_FREE_IMMEDIATELY
};
-ID
-ripper_get_id(VALUE v)
-{
- NODE *nd;
- if (!RB_TYPE_P(v, T_NODE)) return 0;
- nd = (NODE *)v;
- if (!nd_type_p(nd, NODE_RIPPER)) return 0;
- return RNODE_RIPPER(nd)->nd_vid;
-}
-
-VALUE
-ripper_get_value(VALUE v)
-{
- NODE *nd;
- if (UNDEF_P(v)) return Qnil;
- if (!RB_TYPE_P(v, T_NODE)) return v;
- nd = (NODE *)v;
- if (!nd_type_p(nd, NODE_RIPPER)) return Qnil;
- return RNODE_RIPPER(nd)->nd_rval;
-}
-
-static VALUE
-ripper_lex_get_generic(struct parser_params *p, VALUE src)
+static rb_parser_string_t *
+ripper_lex_get_generic(struct parser_params *p, rb_parser_input_data input, int line_count)
{
+ VALUE src = (VALUE)input;
VALUE line = rb_funcallv_public(src, id_gets, 0, 0);
- if (!NIL_P(line) && !RB_TYPE_P(line, T_STRING)) {
+ if (NIL_P(line)) return 0;
+ if (!RB_TYPE_P(line, T_STRING)) {
rb_raise(rb_eTypeError,
"gets returned %"PRIsVALUE" (expected String or nil)",
rb_obj_class(line));
}
- return line;
+ return rb_str_to_parser_string(p, line);
}
void
@@ -100,10 +105,19 @@ ripper_compile_error(struct parser_params *p, const char *fmt, ...)
ripper_error(p);
}
-static VALUE
-ripper_lex_io_get(struct parser_params *p, VALUE src)
+static rb_parser_string_t *
+ripper_lex_io_get(struct parser_params *p, rb_parser_input_data input, int line_count)
{
- return rb_io_gets(src);
+ VALUE src = (VALUE)input;
+ VALUE line = rb_io_gets(src);
+ if (NIL_P(line)) return 0;
+ return rb_str_to_parser_string(p, line);
+}
+
+static rb_parser_string_t *
+ripper_lex_get_str(struct parser_params *p, rb_parser_input_data input, int line_count)
+{
+ return rb_parser_lex_get_str(p, (struct lex_pointer_string *)input);
}
static VALUE
@@ -115,10 +129,8 @@ ripper_s_allocate(VALUE klass)
&parser_data_type, r);
#ifdef UNIVERSAL_PARSER
- rb_parser_config_t *config;
- config = rb_ruby_parser_config_new(ruby_xmalloc);
- rb_parser_config_initialize(config);
- r->p = rb_ruby_parser_allocate(config);
+ const rb_parser_config_t *config = rb_ruby_parser_config();
+ r->p = rb_ripper_parser_params_allocate(config);
#else
r->p = rb_ruby_ripper_parser_allocate();
#endif
@@ -179,7 +191,7 @@ ripper_parser_encoding(VALUE vparser)
{
struct parser_params *p = ripper_parser_params(vparser, false);
- return rb_ruby_parser_encoding(p);
+ return rb_enc_from_encoding(rb_ruby_parser_encoding(p));
}
/*
@@ -240,6 +252,18 @@ ripper_parser_set_debug_output(VALUE self, VALUE output)
return output;
}
+static int
+ripper_parser_dedent_string(struct parser_params *p, VALUE string, int width)
+{
+ int col;
+ rb_parser_string_t *str;
+ str = rb_str_to_parser_string(p, string);
+ col = rb_ruby_ripper_dedent_string(p, str, width);
+ rb_str_replace(string, rb_str_new_parser_string(str));
+ rb_parser_string_free(p, str);
+ return col;
+}
+
#ifdef UNIVERSAL_PARSER
struct dedent_string_arg {
struct parser_params *p;
@@ -255,16 +279,16 @@ parser_dedent_string0(VALUE a)
StringValue(arg->input);
wid = NUM2UINT(arg->width);
- col = rb_ruby_ripper_dedent_string(arg->p, arg->input, wid);
+ col = ripper_parser_dedent_string(arg->p, arg->input, wid);
return INT2NUM(col);
}
static VALUE
-parser_config_free(VALUE a)
+parser_free(VALUE a)
{
- rb_parser_config_t *config = (void *)a;
+ struct parser_params *p = (void *)a;
- rb_ruby_parser_config_free(config);
+ rb_ruby_parser_free(p);
return Qnil;
}
#endif
@@ -283,17 +307,14 @@ static VALUE
parser_dedent_string(VALUE self, VALUE input, VALUE width)
{
struct parser_params *p;
- rb_parser_config_t *config;
struct dedent_string_arg args;
- config = rb_ruby_parser_config_new(ruby_xmalloc);
- rb_parser_config_initialize(config);
- p = rb_ruby_parser_new(config);
+ p = rb_parser_params_new();
args.p = p;
args.input = input;
args.width = width;
- return rb_ensure(parser_dedent_string0, (VALUE)&args, parser_config_free, (VALUE)config);
+ return rb_ensure(parser_dedent_string0, (VALUE)&args, parser_free, (VALUE)p);
}
#else
static VALUE
@@ -303,7 +324,7 @@ parser_dedent_string(VALUE self, VALUE input, VALUE width)
StringValue(input);
wid = NUM2UINT(width);
- col = rb_ruby_ripper_dedent_string(0, input, wid);
+ col = ripper_parser_dedent_string(0, input, wid);
return INT2NUM(col);
}
#endif
@@ -321,26 +342,38 @@ parser_dedent_string(VALUE self, VALUE input, VALUE width)
static VALUE
ripper_initialize(int argc, VALUE *argv, VALUE self)
{
+ struct ripper *r;
struct parser_params *p;
VALUE src, fname, lineno;
- VALUE (*gets)(struct parser_params*,VALUE);
- VALUE input, sourcefile_string;
+ rb_parser_lex_gets_func *gets;
+ VALUE sourcefile_string;
const char *sourcefile;
int sourceline;
+ rb_parser_input_data input;
p = ripper_parser_params(self, false);
+ TypedData_Get_Struct(self, struct ripper, &parser_data_type, r);
rb_scan_args(argc, argv, "12", &src, &fname, &lineno);
if (RB_TYPE_P(src, T_FILE)) {
gets = ripper_lex_io_get;
+ r->type = lex_type_io;
+ r->data.val = src;
+ input = (rb_parser_input_data)src;
}
else if (rb_respond_to(src, id_gets)) {
gets = ripper_lex_get_generic;
+ r->type = lex_type_generic;
+ r->data.val = src;
+ input = (rb_parser_input_data)src;
}
else {
StringValue(src);
- gets = rb_ruby_ripper_lex_get_str;
+ gets = ripper_lex_get_str;
+ r->type = lex_type_str;
+ r->data.ptr_str.str = src;
+ r->data.ptr_str.ptr = 0;
+ input = (rb_parser_input_data)&r->data.ptr_str;
}
- input = src;
if (NIL_P(fname)) {
fname = STR_NEW2("(ripper)");
OBJ_FREEZE(fname);
@@ -477,11 +510,13 @@ ripper_token(VALUE self)
{
struct parser_params *p = ripper_parser_params(self, true);
long pos, len;
+ VALUE str;
if (NIL_P(rb_ruby_parser_parsing_thread(p))) return Qnil;
pos = rb_ruby_ripper_column(p);
len = rb_ruby_ripper_token_len(p);
- return rb_str_subseq(rb_ruby_ripper_lex_lastline(p), pos, len);
+ str = rb_str_new_parser_string(rb_ruby_ripper_lex_lastline(p));
+ return rb_str_subseq(str, pos, len);
}
#ifdef RIPPER_DEBUG
@@ -502,6 +537,37 @@ ripper_raw_value(VALUE self, VALUE obj)
{
return ULONG2NUM(obj);
}
+
+/* :nodoc: */
+static VALUE
+ripper_validate_object(VALUE self, VALUE x)
+{
+ if (x == Qfalse) return x;
+ if (x == Qtrue) return x;
+ if (NIL_P(x)) return x;
+ if (UNDEF_P(x))
+ rb_raise(rb_eArgError, "Qundef given");
+ if (FIXNUM_P(x)) return x;
+ if (SYMBOL_P(x)) return x;
+ switch (BUILTIN_TYPE(x)) {
+ case T_STRING:
+ case T_OBJECT:
+ case T_ARRAY:
+ case T_BIGNUM:
+ case T_FLOAT:
+ case T_COMPLEX:
+ case T_RATIONAL:
+ break;
+ default:
+ rb_raise(rb_eArgError, "wrong type of ruby object: %p (%s)",
+ (void *)x, rb_obj_classname(x));
+ }
+ if (!RBASIC_CLASS(x)) {
+ rb_raise(rb_eArgError, "hidden ruby object: %p (%s)",
+ (void *)x, rb_builtin_type_name(TYPE(x)));
+ }
+ return x;
+}
#endif
#ifdef UNIVERSAL_PARSER
@@ -530,17 +596,14 @@ static VALUE
ripper_lex_state_name(VALUE self, VALUE state)
{
struct parser_params *p;
- rb_parser_config_t *config;
struct lex_state_name_arg args;
- config = rb_ruby_parser_config_new(ruby_xmalloc);
- rb_parser_config_initialize(config);
- p = rb_ruby_parser_new(config);
+ p = rb_parser_params_new();
args.p = p;
args.state = state;
- return rb_ensure(lex_state_name0, (VALUE)&args, parser_config_free, (VALUE)config);
+ return rb_ensure(lex_state_name0, (VALUE)&args, parser_free, (VALUE)p);
}
#else
static VALUE
@@ -614,5 +677,4 @@ InitVM_ripper(void)
*/
rb_define_global_const("SCRIPT_LINES__", Qnil);
#endif
-
}
diff --git a/ext/ripper/ripper_init.h b/ext/ripper/ripper_init.h
index 82ff13b95f..9d228107d1 100644
--- a/ext/ripper/ripper_init.h
+++ b/ext/ripper/ripper_init.h
@@ -1,8 +1,6 @@
#ifndef RIPPER_INIT_H
#define RIPPER_INIT_H
-VALUE ripper_get_value(VALUE v);
-ID ripper_get_id(VALUE v);
PRINTF_ARGS(void ripper_compile_error(struct parser_params*, const char *fmt, ...), 2, 3);
#endif /* RIPPER_INIT_H */
diff --git a/ext/ripper/tools/dsl.rb b/ext/ripper/tools/dsl.rb
index 6d662b3fb8..38f859dd97 100644
--- a/ext/ripper/tools/dsl.rb
+++ b/ext/ripper/tools/dsl.rb
@@ -1,6 +1,8 @@
+# frozen_string_literal: true
+
# Simple DSL implementation for Ripper code generation
#
-# input: /*% ripper: stmts_add(stmts_new, void_stmt) %*/
+# input: /*% ripper: stmts_add!(stmts_new!, void_stmt!) %*/
# output:
# VALUE v1, v2;
# v1 = dispatch0(stmts_new);
@@ -18,69 +20,158 @@ class DSL
NAME_PATTERN = /(?>\$|\d+|[a-zA-Z_][a-zA-Z0-9_]*|\[[a-zA-Z_.][-a-zA-Z0-9_.]*\])(?>(?:\.|->)[a-zA-Z_][a-zA-Z0-9_]*)*/.source
NOT_REF_PATTERN = /(?>\#.*|[^\"$@]*|"(?>\\.|[^\"])*")/.source
- def initialize(code, options)
+ def self.line?(line, lineno = nil, indent: nil)
+ if %r<(?<space>\s*)/\*% *ripper(?:\[(?<option>.*?)\])?: *(?<code>.*?) *%\*/> =~ line
+ new(code, comma_split(option), lineno, indent: indent || space)
+ end
+ end
+
+ def self.comma_split(str)
+ str or return []
+ str.scan(/(([^(,)]+|\((?:,|\g<0>)*\))+)/).map(&:first)
+ end
+
+ using Module.new {
+ refine Array do
+ def to_s
+ if empty?
+ "rb_ary_new()"
+ else
+ "rb_ary_new_from_args(#{size}, #{map(&:to_s).join(', ')})"
+ end
+ end
+ end
+ }
+
+ class Var
+ class Table < Hash
+ def initialize(&block)
+ super() {|tbl, arg|
+ tbl.fetch(arg, &block)
+ }
+ end
+
+ def fetch(arg, &block)
+ super {
+ self[arg] = Var.new(self, arg, &block)
+ }
+ end
+
+ def add(&block)
+ v = new_var
+ self[v] = Var.new(self, v, &block)
+ end
+
+ def defined?(name)
+ name = name.to_s
+ any? {|_, v| v.var == name}
+ end
+
+ def new_var
+ "v#{size+1}"
+ end
+ end
+
+ attr_reader :var, :value
+
+ PRETTY_PRINT_INSTANCE_VARIABLES = instance_methods(false).freeze
+
+ def pretty_print_instance_variables
+ PRETTY_PRINT_INSTANCE_VARIABLES
+ end
+
+ alias to_s var
+
+ def initialize(table, arg, &block)
+ @var = table.new_var
+ @value = yield arg
+ @table = table
+ end
+
+ # Indexing.
+ #
+ # $:1 -> v1=get_value($:1)
+ # $:1[0] -> rb_ary_entry(v1, 0)
+ # $:1[0..1] -> [rb_ary_entry(v1, 0), rb_ary_entry(v1, 1)]
+ # *$:1[0..1] -> rb_ary_entry(v1, 0), rb_ary_entry(v1, 1)
+ #
+ # Splat needs `[range]` because `Var` does not have the length info.
+ def [](idx)
+ if ::Range === idx
+ idx.map {|i| self[i]}
+ else
+ @table.fetch("#@var[#{idx}]") {"rb_ary_entry(#{@var}, #{idx})"}
+ end
+ end
+ end
+
+ def initialize(code, options, lineno = nil, indent: "\t\t\t")
+ @lineno = lineno
+ @indent = indent
@events = {}
@error = options.include?("error")
- @brace = options.include?("brace")
if options.include?("final")
@final = "p->result"
else
- @final = (options.grep(/\A\$#{NAME_PATTERN}\z/o)[0] || "$$")
+ @final = (options.grep(/\A\$#{NAME_PATTERN}\z/o)[0] || "p->s_lvalue")
end
- @vars = 0
- # struct parser_params *p
- p = p = "p"
+ bind = dsl_binding
+ @var_table = Var::Table.new {|arg| "get_value(#{arg})"}
+ code = code.gsub(%r[\G#{NOT_REF_PATTERN}\K(\$|\$:|@)#{TAG_PATTERN}?#{NAME_PATTERN}]o) {
+ if (arg = $&) == "$:$"
+ '"p->s_lvalue"'
+ elsif arg.start_with?("$:")
+ "(#{@var_table[arg]}=@var_table[#{arg.dump}])"
+ else
+ arg.dump
+ end
+ }
+ @last_value = bind.eval(code)
+ rescue SyntaxError
+ $stderr.puts "error on line #{@lineno}" if @lineno
+ raise
+ end
- @code = ""
- code = code.gsub(%r[\G#{NOT_REF_PATTERN}\K[$@]#{TAG_PATTERN}?#{NAME_PATTERN}]o, '"\&"')
- @last_value = eval(code)
+ def dsl_binding(p = "p")
+ # struct parser_params *p
+ binding
end
attr_reader :events
undef lambda
undef hash
- undef class
+ undef :class
def generate
- s = "#@code#@final=#@last_value;"
- s = "{VALUE #{ (1..@vars).map {|v| "v#{ v }" }.join(",") };#{ s }}" if @vars > 0
+ s = "#@final=#@last_value;"
s << "ripper_error(p);" if @error
- s = "{#{ s }}" if @brace
- "\t\t\t#{s}"
- end
-
- def new_var
- "v#{ @vars += 1 }"
- end
-
- def opt_event(event, default, addend)
- add_event(event, [default, addend], true)
+ unless @var_table.empty?
+ vars = @var_table.map {|_, v| "#{v.var}=#{v.value}"}.join(", ")
+ s = "VALUE #{ vars }; #{ s }"
+ end
+ "#{@indent}{#{s}}"
end
- def add_event(event, args, qundef_check = false)
+ def add_event(event, args)
event = event.to_s.sub(/!\z/, "")
@events[event] = args.size
vars = []
args.each do |arg|
- vars << v = new_var
- @code << "#{ v }=#{ arg };"
+ arg = @var_table.add {arg} unless Var === arg
+ vars << arg
end
- v = new_var
- d = "dispatch#{ args.size }(#{ [event, *vars].join(",") })"
- d = "#{ vars.last }==Qundef ? #{ vars.first } : #{ d }" if qundef_check
- @code << "#{ v }=#{ d };"
- v
+ @var_table.add {"dispatch#{ args.size }(#{ [event, *vars].join(",") })"}
end
def method_missing(event, *args)
if event.to_s =~ /!\z/
add_event(event, args)
- elsif args.empty? and /\Aid[A-Z_]/ =~ event.to_s
+ elsif args.empty? and (/\Aid[A-Z_]/ =~ event or @var_table.defined?(event))
event
else
- "#{ event }(#{ args.join(", ") })"
+ "#{ event }(#{ args.map(&:to_s).join(", ") })"
end
end
@@ -88,4 +179,3 @@ class DSL
name
end
end
-
diff --git a/ext/ripper/tools/generate.rb b/ext/ripper/tools/generate.rb
index bb64d2fe8b..57ecac0b39 100644
--- a/ext/ripper/tools/generate.rb
+++ b/ext/ripper/tools/generate.rb
@@ -75,6 +75,7 @@ def generate_eventids1_h(ids)
buf << %Q[#ifndef RIPPER_EVENTIDS1\n]
buf << %Q[#define RIPPER_EVENTIDS1\n]
buf << %Q[\n]
+ buf << %Q[#define RIPPER_ID(n) ripper_parser_ids.id_ ## n\n]
buf << %Q[void ripper_init_eventids1(void);\n]
buf << %Q[void ripper_init_eventids1_table(VALUE self);\n]
buf << %Q[\n]
@@ -84,9 +85,6 @@ def generate_eventids1_h(ids)
end
buf << %Q[};\n]
buf << %Q[\n]
- ids.each do |id, arity|
- buf << %Q[#define ripper_id_#{id} ripper_parser_ids.id_#{id}\n]
- end
buf << %Q[#endif /* RIPPER_EVENTIDS1 */\n]
buf << %Q[\n]
end
@@ -101,7 +99,7 @@ def generate_eventids1(ids)
buf << %Q[void\n]
buf << %Q[ripper_init_eventids1(void)\n]
buf << %Q[{\n]
- buf << %Q[#define set_id1(name) ripper_id_##name = rb_intern_const("on_"#name)\n]
+ buf << %Q[#define set_id1(name) RIPPER_ID(name) = rb_intern_const("on_"#name)\n]
ids.each do |id, arity|
buf << %Q[ set_id1(#{id});\n]
end
@@ -136,6 +134,8 @@ def generate_eventids2_table(ids)
buf << %Q[ rb_hash_aset(h, intern_sym("#{id}"), INT2FIX(1));\n]
end
buf << %Q[}\n]
+ buf << %Q[\n]
+ buf << %Q[#define RIPPER_EVENTIDS2_TABLE_SIZE #{ids.size}\n]
buf
end
@@ -171,9 +171,7 @@ def read_ids1_with_locations(path)
line.scan(/\bdispatch(\d)\((\w+)/) do |arity, event|
(h[event] ||= []).push [f.lineno, arity.to_i]
end
- if line =~ %r</\*% *ripper(?:\[(.*?)\])?: *(.*?) *%\*/>
- gen = DSL.new($2, ($1 || "").split(","))
- gen.generate
+ if gen = DSL.line?(line, f.lineno)
gen.events.each do |event, arity|
(h[event] ||= []).push [f.lineno, arity.to_i]
end
diff --git a/ext/ripper/tools/preproc.rb b/ext/ripper/tools/preproc.rb
index a0d5e79d7d..5e8a6e0cb5 100644
--- a/ext/ripper/tools/preproc.rb
+++ b/ext/ripper/tools/preproc.rb
@@ -51,27 +51,26 @@ def process(f, out, path, template)
usercode f, out, path, template
end
-def prelude(f, out)
- @exprs = {}
- lex_state_def = false
+require_relative 'dsl'
+
+def generate_line(f, out)
while line = f.gets
- case line
- when /\A%%/
+ case
+ when gen = DSL.line?(line, f.lineno)
+ out << gen.generate << "\n"
+ when line.start_with?("%%")
out << "%%\n"
- return
- when /\A%token/, /\A%type/, /\A} <node(?>_\w+)?>/
- # types in %union which have corresponding set_yylval_* macro.
- out << line.sub(/<(?:node(?>_\w+)?|num|id)>/, '<val>')
- when /^enum lex_state_(?:bits|e) \{/
- lex_state_def = true
- out << line
- when /^\}/
- lex_state_def = false
- out << line
+ break
else
- out << line
+ out << yield(line)
end
- if lex_state_def
+ end
+end
+
+def prelude(f, out)
+ @exprs = {}
+ generate_line(f, out) do |line|
+ if (/^enum lex_state_(?:bits|e) \{/ =~ line)..(/^\}/ =~ line)
case line
when /^\s*(EXPR_\w+),\s+\/\*(.+)\*\//
@exprs[$1.chomp("_bit")] = $2.strip
@@ -81,27 +80,21 @@ def prelude(f, out)
@exprs[name] = "equals to " + (val.start_with?("(") ? "<tt>#{val}</tt>" : "+#{val}+")
end
end
+ line
end
end
-require_relative "dsl"
-
def grammar(f, out)
- while line = f.gets
+ generate_line(f, out) do |line|
case line
- when %r</\*% *ripper(?:\[(.*?)\])?: *(.*?) *%\*/>
- out << DSL.new($2, ($1 || "").split(",")).generate << "\n"
when %r</\*%%%\*/>
- out << "#if 0\n"
+ "#if 0\n"
when %r</\*%>
- out << "#endif\n"
+ "#endif\n"
when %r<%\*/>
- out << "\n"
- when /\A%%/
- out << "%%\n"
- return
+ "\n"
else
- out << line
+ line
end
end
end
generated by cgit v1.2.3 (git 2.25.1) at 2026-06-06 18:10:01 +0000