diff options
Diffstat (limited to 'ruby_2_2/ext/ripper')
| -rw-r--r-- | ruby_2_2/ext/ripper/README | 30 | ||||
| -rw-r--r-- | ruby_2_2/ext/ripper/depend | 75 | ||||
| -rw-r--r-- | ruby_2_2/ext/ripper/eventids2.c | 305 | ||||
| -rw-r--r-- | ruby_2_2/ext/ripper/extconf.rb | 21 | ||||
| -rw-r--r-- | ruby_2_2/ext/ripper/lib/ripper.rb | 73 | ||||
| -rw-r--r-- | ruby_2_2/ext/ripper/lib/ripper/core.rb | 70 | ||||
| -rw-r--r-- | ruby_2_2/ext/ripper/lib/ripper/filter.rb | 77 | ||||
| -rw-r--r-- | ruby_2_2/ext/ripper/lib/ripper/lexer.rb | 186 | ||||
| -rw-r--r-- | ruby_2_2/ext/ripper/lib/ripper/sexp.rb | 118 | ||||
| -rwxr-xr-x | ruby_2_2/ext/ripper/tools/generate-param-macros.rb | 14 | ||||
| -rwxr-xr-x | ruby_2_2/ext/ripper/tools/generate.rb | 161 | ||||
| -rwxr-xr-x | ruby_2_2/ext/ripper/tools/preproc.rb | 91 | ||||
| -rwxr-xr-x | ruby_2_2/ext/ripper/tools/strip.rb | 12 |
13 files changed, 1233 insertions, 0 deletions
diff --git a/ruby_2_2/ext/ripper/README b/ruby_2_2/ext/ripper/README new file mode 100644 index 0000000000..2ae2470e13 --- /dev/null +++ b/ruby_2_2/ext/ripper/README @@ -0,0 +1,30 @@ +Ripper README +============= + + Ripper is a Ruby script parser. You can get information + by event-based style from the parser. + + !! WARNING !! + + Ripper is still early-alpha version. + I never assure any kind of backward compatibility. + +Requirements +------------ + + * ruby 1.9 (support CVS HEAD only) + * bison 1.28 or later (Other yaccs do not work) + +Usage +----- + + See test/ripper/* and sample/ripper/*. + +License +------- + + Ruby License. + + Minero Aoki + aamine@loveruby.net + http://i.loveruby.net diff --git a/ruby_2_2/ext/ripper/depend b/ruby_2_2/ext/ripper/depend new file mode 100644 index 0000000000..64236cd592 --- /dev/null +++ b/ruby_2_2/ext/ripper/depend @@ -0,0 +1,75 @@ +GEN = $(srcdir)/tools/generate.rb +SRC1 = $(top_srcdir)/parse.y +SRC2 = $(srcdir)/eventids2.c +BISON = bison + +.SUFFIXES: .y + +src: ripper.c eventids1.c eventids2table.c + +ripper.o: ripper.c + +.y.c: + $(ECHO) compiling compiler $< + $(Q) $(BISON) -t -v -oy.tab.c $< + $(Q) sed -f $(top_srcdir)/tool/ytab.sed -e "/^#/s!y\.tab\.c!$@!" y.tab.c > $@ + @$(RM) y.tab.c + +all: check +static: check + +ripper.y: $(srcdir)/tools/preproc.rb $(top_srcdir)/parse.y + $(ECHO) extracting $@ from $(top_srcdir)/parse.y + $(Q) $(RUBY) $(top_srcdir)/tool/id2token.rb --path-separator=.$(PATH_SEPARATOR)./ --vpath=$(VPATH) id.h $(top_srcdir)/parse.y > ripper.tmp.y + $(Q) $(RUBY) $(srcdir)/tools/preproc.rb ripper.tmp.y --output=$@ + $(Q) $(RM) ripper.tmp.y + +check: $(GEN) $(SRC1) $(SRC2) + $(ECHO) checking $(SRC1) and $(SRC2) + $(Q) $(RUBY) $(GEN) --mode=check --ids1src=$(SRC1) --ids2src=$(SRC2) + +eventids1.c: $(srcdir)/tools/generate.rb $(SRC1) + $(ECHO) generating $@ from $(SRC1) + $(Q) $(RUBY) $(GEN) --mode=eventids1 --ids1src=$(SRC1) --output=$@ + +eventids2table.c: $(srcdir)/tools/generate.rb $(SRC2) + $(ECHO) generating $@ from $(SRC2) + $(Q) $(RUBY) $(GEN) --mode=eventids2table --ids2src=$(SRC2) --output=$@ + +# Entries for Ripper maintainer + +preproc: ripper.E +ripper.E: ripper.c + $(ECHO) preprocessing ripper.c + $(Q) $(CC) -E $(CPPFLAGS) ripper.c | $(RUBY) $(srcdir)/tools/strip.rb > $@ + +# AUTOGENERATED DEPENDENCIES START +ripper.o: $(RUBY_EXTCONF_H) +ripper.o: $(arch_hdrdir)/ruby/config.h +ripper.o: $(hdrdir)/ruby/defines.h +ripper.o: $(hdrdir)/ruby/encoding.h +ripper.o: $(hdrdir)/ruby/intern.h +ripper.o: $(hdrdir)/ruby/io.h +ripper.o: $(hdrdir)/ruby/missing.h +ripper.o: $(hdrdir)/ruby/oniguruma.h +ripper.o: $(hdrdir)/ruby/regex.h +ripper.o: $(hdrdir)/ruby/ruby.h +ripper.o: $(hdrdir)/ruby/st.h +ripper.o: $(hdrdir)/ruby/subst.h +ripper.o: $(hdrdir)/ruby/util.h +ripper.o: $(top_srcdir)/include/ruby.h +ripper.o: $(top_srcdir)/internal.h +ripper.o: $(top_srcdir)/node.h +ripper.o: $(top_srcdir)/regenc.h +ripper.o: $(top_srcdir)/symbol.h +ripper.o: $(top_srcdir)/vm_opts.h +ripper.o: ../../probes.h +ripper.o: eventids2.c +ripper.o: ripper.y +ripper.o: {$(VPATH)}eventids1.c +ripper.o: {$(VPATH)}eventids2table.c +ripper.o: {$(VPATH)}id.h +ripper.o: {$(VPATH)}lex.c +ripper.o: {$(VPATH)}parse.h +ripper.o: {$(VPATH)}ripper.c +# AUTOGENERATED DEPENDENCIES END diff --git a/ruby_2_2/ext/ripper/eventids2.c b/ruby_2_2/ext/ripper/eventids2.c new file mode 100644 index 0000000000..abbe584454 --- /dev/null +++ b/ruby_2_2/ext/ripper/eventids2.c @@ -0,0 +1,305 @@ +#define tIGNORED_NL (tLAST_TOKEN + 1) +#define tCOMMENT (tLAST_TOKEN + 2) +#define tEMBDOC_BEG (tLAST_TOKEN + 3) +#define tEMBDOC (tLAST_TOKEN + 4) +#define tEMBDOC_END (tLAST_TOKEN + 5) +#define tSP (tLAST_TOKEN + 6) +#define tHEREDOC_BEG (tLAST_TOKEN + 7) +#define tHEREDOC_END (tLAST_TOKEN + 8) +#define k__END__ (tLAST_TOKEN + 9) + +typedef struct { + ID ripper_id_backref; + ID ripper_id_backtick; + ID ripper_id_comma; + ID ripper_id_const; + ID ripper_id_cvar; + ID ripper_id_embexpr_beg; + ID ripper_id_embexpr_end; + ID ripper_id_embvar; + ID ripper_id_float; + ID ripper_id_gvar; + ID ripper_id_ident; + ID ripper_id_imaginary; + ID ripper_id_int; + ID ripper_id_ivar; + ID ripper_id_kw; + ID ripper_id_lbrace; + ID ripper_id_lbracket; + ID ripper_id_lparen; + ID ripper_id_nl; + ID ripper_id_op; + ID ripper_id_period; + ID ripper_id_rbrace; + ID ripper_id_rbracket; + ID ripper_id_rparen; + ID ripper_id_semicolon; + ID ripper_id_symbeg; + ID ripper_id_tstring_beg; + ID ripper_id_tstring_content; + ID ripper_id_tstring_end; + ID ripper_id_words_beg; + ID ripper_id_qwords_beg; + ID ripper_id_qsymbols_beg; + ID ripper_id_symbols_beg; + ID ripper_id_words_sep; + ID ripper_id_rational; + ID ripper_id_regexp_beg; + ID ripper_id_regexp_end; + ID ripper_id_label; + ID ripper_id_label_end; + ID ripper_id_tlambda; + ID ripper_id_tlambeg; + + ID ripper_id_ignored_nl; + ID ripper_id_comment; + ID ripper_id_embdoc_beg; + ID ripper_id_embdoc; + ID ripper_id_embdoc_end; + ID ripper_id_sp; + ID ripper_id_heredoc_beg; + ID ripper_id_heredoc_end; + ID ripper_id___end__; + ID ripper_id_CHAR; +} ripper_scanner_ids_t; + +static ripper_scanner_ids_t ripper_scanner_ids; + +#include "eventids2table.c" + +static void +ripper_init_eventids2(void) +{ +#define set_id2(name) ripper_scanner_ids.ripper_id_##name = rb_intern_const("on_"#name) + set_id2(backref); + set_id2(backtick); + set_id2(comma); + set_id2(const); + set_id2(cvar); + set_id2(embexpr_beg); + set_id2(embexpr_end); + set_id2(embvar); + set_id2(float); + set_id2(gvar); + set_id2(ident); + set_id2(imaginary); + set_id2(int); + set_id2(ivar); + set_id2(kw); + set_id2(lbrace); + set_id2(lbracket); + set_id2(lparen); + set_id2(nl); + set_id2(op); + set_id2(period); + set_id2(rbrace); + set_id2(rbracket); + set_id2(rparen); + set_id2(semicolon); + set_id2(symbeg); + set_id2(tstring_beg); + set_id2(tstring_content); + set_id2(tstring_end); + set_id2(words_beg); + set_id2(qwords_beg); + set_id2(qsymbols_beg); + set_id2(symbols_beg); + set_id2(words_sep); + set_id2(rational); + set_id2(regexp_beg); + set_id2(regexp_end); + set_id2(label); + set_id2(label_end); + set_id2(tlambda); + set_id2(tlambeg); + + set_id2(ignored_nl); + set_id2(comment); + set_id2(embdoc_beg); + set_id2(embdoc); + set_id2(embdoc_end); + set_id2(sp); + set_id2(heredoc_beg); + set_id2(heredoc_end); + set_id2(__end__); + set_id2(CHAR); +} + +STATIC_ASSERT(k__END___range, k__END__ < SHRT_MAX); +STATIC_ASSERT(ripper_scanner_ids_size, sizeof(ripper_scanner_ids) < SHRT_MAX); +#define O(member) (int)offsetof(ripper_scanner_ids_t, ripper_id_##member) + +static const struct token_assoc { + unsigned short token; + unsigned short id_offset; +} token_to_eventid[] = { + {' ', O(words_sep)}, + {'!', O(op)}, + {'%', O(op)}, + {'&', O(op)}, + {'*', O(op)}, + {'+', O(op)}, + {'-', O(op)}, + {'/', O(op)}, + {'<', O(op)}, + {'=', O(op)}, + {'>', O(op)}, + {'?', O(op)}, + {'^', O(op)}, + {'|', O(op)}, + {'~', O(op)}, + {':', O(op)}, + {',', O(comma)}, + {'.', O(period)}, + {';', O(semicolon)}, + {'`', O(backtick)}, + {'\n', O(nl)}, + {keyword_alias, O(kw)}, + {keyword_and, O(kw)}, + {keyword_begin, O(kw)}, + {keyword_break, O(kw)}, + {keyword_case, O(kw)}, + {keyword_class, O(kw)}, + {keyword_def, O(kw)}, + {keyword_defined, O(kw)}, + {keyword_do, O(kw)}, + {keyword_do_block, O(kw)}, + {keyword_do_cond, O(kw)}, + {keyword_else, O(kw)}, + {keyword_elsif, O(kw)}, + {keyword_end, O(kw)}, + {keyword_ensure, O(kw)}, + {keyword_false, O(kw)}, + {keyword_for, O(kw)}, + {keyword_if, O(kw)}, + {modifier_if, O(kw)}, + {keyword_in, O(kw)}, + {keyword_module, O(kw)}, + {keyword_next, O(kw)}, + {keyword_nil, O(kw)}, + {keyword_not, O(kw)}, + {keyword_or, O(kw)}, + {keyword_redo, O(kw)}, + {keyword_rescue, O(kw)}, + {modifier_rescue, O(kw)}, + {keyword_retry, O(kw)}, + {keyword_return, O(kw)}, + {keyword_self, O(kw)}, + {keyword_super, O(kw)}, + {keyword_then, O(kw)}, + {keyword_true, O(kw)}, + {keyword_undef, O(kw)}, + {keyword_unless, O(kw)}, + {modifier_unless, O(kw)}, + {keyword_until, O(kw)}, + {modifier_until, O(kw)}, + {keyword_when, O(kw)}, + {keyword_while, O(kw)}, + {modifier_while, O(kw)}, + {keyword_yield, O(kw)}, + {keyword__FILE__, O(kw)}, + {keyword__LINE__, O(kw)}, + {keyword__ENCODING__, O(kw)}, + {keyword_BEGIN, O(kw)}, + {keyword_END, O(kw)}, + {keyword_do_LAMBDA, O(kw)}, + {tAMPER, O(op)}, + {tANDOP, O(op)}, + {tAREF, O(op)}, + {tASET, O(op)}, + {tASSOC, O(op)}, + {tBACK_REF, O(backref)}, + {tCHAR, O(CHAR)}, + {tCMP, O(op)}, + {tCOLON2, O(op)}, + {tCOLON3, O(op)}, + {tCONSTANT, O(const)}, + {tCVAR, O(cvar)}, + {tDOT2, O(op)}, + {tDOT3, O(op)}, + {tEQ, O(op)}, + {tEQQ, O(op)}, + {tFID, O(ident)}, + {tFLOAT, O(float)}, + {tGEQ, O(op)}, + {tGVAR, O(gvar)}, + {tIDENTIFIER, O(ident)}, + {tIMAGINARY, O(imaginary)}, + {tINTEGER, O(int)}, + {tIVAR, O(ivar)}, + {tLBRACE, O(lbrace)}, + {tLBRACE_ARG, O(lbrace)}, + {'{', O(lbrace)}, + {'}', O(rbrace)}, + {tLBRACK, O(lbracket)}, + {'[', O(lbracket)}, + {']', O(rbracket)}, + {tLEQ, O(op)}, + {tLPAREN, O(lparen)}, + {tLPAREN_ARG, O(lparen)}, + {'(', O(lparen)}, + {')', O(rparen)}, + {tLSHFT, O(op)}, + {tMATCH, O(op)}, + {tNEQ, O(op)}, + {tNMATCH, O(op)}, + {tNTH_REF, O(backref)}, + {tOP_ASGN, O(op)}, + {tOROP, O(op)}, + {tPOW, O(op)}, + {tQWORDS_BEG, O(qwords_beg)}, + {tQSYMBOLS_BEG, O(qsymbols_beg)}, + {tSYMBOLS_BEG, O(symbols_beg)}, + {tRATIONAL, O(rational)}, + {tREGEXP_BEG, O(regexp_beg)}, + {tREGEXP_END, O(regexp_end)}, + {tRPAREN, O(rparen)}, + {tRSHFT, O(op)}, + {tSTAR, O(op)}, + {tDSTAR, O(op)}, + {tSTRING_BEG, O(tstring_beg)}, + {tSTRING_CONTENT, O(tstring_content)}, + {tSTRING_DBEG, O(embexpr_beg)}, + {tSTRING_DEND, O(embexpr_end)}, + {tSTRING_DVAR, O(embvar)}, + {tSTRING_END, O(tstring_end)}, + {tSYMBEG, O(symbeg)}, + {tUMINUS, O(op)}, + {tUMINUS_NUM, O(op)}, + {tUPLUS, O(op)}, + {tWORDS_BEG, O(words_beg)}, + {tXSTRING_BEG, O(backtick)}, + {tLABEL, O(label)}, + {tLABEL_END, O(label_end)}, + {tLAMBDA, O(tlambda)}, + {tLAMBEG, O(tlambeg)}, + + /* ripper specific tokens */ + {tIGNORED_NL, O(ignored_nl)}, + {tCOMMENT, O(comment)}, + {tEMBDOC_BEG, O(embdoc_beg)}, + {tEMBDOC, O(embdoc)}, + {tEMBDOC_END, O(embdoc_end)}, + {tSP, O(sp)}, + {tHEREDOC_BEG, O(heredoc_beg)}, + {tHEREDOC_END, O(heredoc_end)}, + {k__END__, O(__end__)}, +}; + +static ID +ripper_token2eventid(int tok) +{ + int i; + + for (i = 0; i < numberof(token_to_eventid); i++) { + const struct token_assoc *const a = &token_to_eventid[i]; + if (a->token == tok) + return *(const ID *)((const char *)&ripper_scanner_ids + a->id_offset); + } + if (tok < 256) { + return ripper_scanner_ids.ripper_id_CHAR; + } + rb_raise(rb_eRuntimeError, "[Ripper FATAL] unknown token %d", tok); + + UNREACHABLE; +} diff --git a/ruby_2_2/ext/ripper/extconf.rb b/ruby_2_2/ext/ripper/extconf.rb new file mode 100644 index 0000000000..db54e5ca2a --- /dev/null +++ b/ruby_2_2/ext/ripper/extconf.rb @@ -0,0 +1,21 @@ +#!ruby -s + +require 'mkmf' +require 'rbconfig' + +def main + unless find_executable('bison') + unless File.exist?('ripper.c') or File.exist?("#{$srcdir}/ripper.c") + raise 'missing bison; abort' + end + end + $objs = %w(ripper.o) + $cleanfiles.concat %w(ripper.y ripper.c ripper.E ripper.output y.output eventids1.c eventids2table.c) + $defs << '-DRIPPER' + $defs << '-DRIPPER_DEBUG' if $debug + $VPATH << '$(topdir)' << '$(top_srcdir)' + $INCFLAGS << ' -I$(topdir) -I$(top_srcdir)' + create_makefile 'ripper' +end + +main diff --git a/ruby_2_2/ext/ripper/lib/ripper.rb b/ruby_2_2/ext/ripper/lib/ripper.rb new file mode 100644 index 0000000000..542bd405d2 --- /dev/null +++ b/ruby_2_2/ext/ripper/lib/ripper.rb @@ -0,0 +1,73 @@ +require 'ripper/core' +require 'ripper/lexer' +require 'ripper/filter' +require 'ripper/sexp' + +# Ripper is a Ruby script parser. +# +# You can get information from the parser with event-based style. +# Information such as abstract syntax trees or simple lexical analysis of the +# Ruby program. +# +# == Usage +# +# Ripper provides an easy interface for parsing your program into a symbolic +# expression tree (or S-expression). +# +# Understanding the output of the parser may come as a challenge, it's +# recommended you use PP to format the output for legibility. +# +# require 'ripper' +# require 'pp' +# +# pp Ripper.sexp('def hello(world) "Hello, #{world}!"; end') +# #=> [:program, +# [[:def, +# [:@ident, "hello", [1, 4]], +# [:paren, +# [:params, [[:@ident, "world", [1, 10]]], nil, nil, nil, nil, nil, nil]], +# [:bodystmt, +# [[:string_literal, +# [:string_content, +# [:@tstring_content, "Hello, ", [1, 18]], +# [:string_embexpr, [[:var_ref, [:@ident, "world", [1, 27]]]]], +# [:@tstring_content, "!", [1, 33]]]]], +# nil, +# nil, +# nil]]]] +# +# You can see in the example above, the expression starts with +:program+. +# +# From here, a method definition at +:def+, followed by the method's identifier +# <code>:@ident</code>. After the method's identifier comes the parentheses +# +:paren+ and the method parameters under +:params+. +# +# Next is the method body, starting at +:bodystmt+ (+stmt+ meaning statement), +# which contains the full definition of the method. +# +# In our case, we're simply returning a String, so next we have the +# +:string_literal+ expression. +# +# Within our +:string_literal+ you'll notice two <code>@tstring_content</code>, +# this is the literal part for <code>Hello, </code> and <code>!</code>. Between +# the two <code>@tstring_content</code> statements is a +:string_embexpr+, +# where _embexpr_ is an embedded expression. Our expression consists of a local +# variable, or +var_ref+, with the identifier (<code>@ident</code>) of +world+. +# +# == Resources +# +# * {Ruby Inside}[http://www.rubyinside.com/using-ripper-to-see-how-ruby-is-parsing-your-code-5270.html] +# +# == Requirements +# +# * ruby 1.9 (support CVS HEAD only) +# * bison 1.28 or later (Other yaccs do not work) +# +# == License +# +# Ruby License. +# +# Minero Aoki +# aamine@loveruby.net +# http://i.loveruby.net +class Ripper; end diff --git a/ruby_2_2/ext/ripper/lib/ripper/core.rb b/ruby_2_2/ext/ripper/lib/ripper/core.rb new file mode 100644 index 0000000000..637a72f4ad --- /dev/null +++ b/ruby_2_2/ext/ripper/lib/ripper/core.rb @@ -0,0 +1,70 @@ +# +# $Id$ +# +# Copyright (c) 2003-2005 Minero Aoki +# +# This program is free software. +# You can distribute and/or modify this program under the Ruby License. +# For details of Ruby License, see ruby/COPYING. +# + +require 'ripper.so' + +class Ripper + + # Parses the given Ruby program read from +src+. + # +src+ must be a String or an IO or a object with a #gets method. + def Ripper.parse(src, filename = '(ripper)', lineno = 1) + new(src, filename, lineno).parse + end + + # This array contains name of parser events. + PARSER_EVENTS = PARSER_EVENT_TABLE.keys + + # This array contains name of scanner events. + SCANNER_EVENTS = SCANNER_EVENT_TABLE.keys + + # This array contains name of all ripper events. + EVENTS = PARSER_EVENTS + SCANNER_EVENTS + + private + + # + # Parser Events + # + + PARSER_EVENT_TABLE.each do |id, arity| + module_eval(<<-End, __FILE__, __LINE__ + 1) + def on_#{id}(#{ ('a'..'z').to_a[0, arity].join(', ') }) + #{arity == 0 ? 'nil' : 'a'} + end + End + end + + # This method is called when weak warning is produced by the parser. + # +fmt+ and +args+ is printf style. + def warn(fmt, *args) + end + + # This method is called when strong warning is produced by the parser. + # +fmt+ and +args+ is printf style. + def warning(fmt, *args) + end + + # This method is called when the parser found syntax error. + def compile_error(msg) + end + + # + # Scanner Events + # + + SCANNER_EVENTS.each do |id| + module_eval(<<-End, __FILE__, __LINE__ + 1) + def on_#{id}(token) + token + end + End + end + +end diff --git a/ruby_2_2/ext/ripper/lib/ripper/filter.rb b/ruby_2_2/ext/ripper/lib/ripper/filter.rb new file mode 100644 index 0000000000..239f9f00e1 --- /dev/null +++ b/ruby_2_2/ext/ripper/lib/ripper/filter.rb @@ -0,0 +1,77 @@ +# +# $Id$ +# +# Copyright (c) 2004,2005 Minero Aoki +# +# This program is free software. +# You can distribute and/or modify this program under the Ruby License. +# For details of Ruby License, see ruby/COPYING. +# + +require 'ripper/lexer' + +class Ripper + + # This class handles only scanner events, + # which are dispatched in the 'right' order (same with input). + class Filter + + # Creates a new Ripper::Filter instance, passes parameters +src+, + # +filename+, and +lineno+ to Ripper::Lexer.new + # + # The lexer is for internal use only. + def initialize(src, filename = '-', lineno = 1) + @__lexer = Lexer.new(src, filename, lineno) + @__line = nil + @__col = nil + end + + # The file name of the input. + def filename + @__lexer.filename + end + + # The line number of the current token. + # This value starts from 1. + # This method is valid only in event handlers. + def lineno + @__line + end + + # The column number of the current token. + # This value starts from 0. + # This method is valid only in event handlers. + def column + @__col + end + + # Starts the parser. + # +init+ is a data accumulator and is passed to the next event handler (as + # of Enumerable#inject). + def parse(init = nil) + data = init + @__lexer.lex.each do |pos, event, tok| + @__line, @__col = *pos + data = if respond_to?(event, true) + then __send__(event, tok, data) + else on_default(event, tok, data) + end + end + data + end + + private + + # This method is called when some event handler is undefined. + # +event+ is :on_XXX, +token+ is the scanned token, and +data+ is a data + # accumulator. + # + # The return value of this method is passed to the next event handler (as + # of Enumerable#inject). + def on_default(event, token, data) + data + end + + end + +end diff --git a/ruby_2_2/ext/ripper/lib/ripper/lexer.rb b/ruby_2_2/ext/ripper/lib/ripper/lexer.rb new file mode 100644 index 0000000000..c0a64d1ee5 --- /dev/null +++ b/ruby_2_2/ext/ripper/lib/ripper/lexer.rb @@ -0,0 +1,186 @@ +# +# $Id$ +# +# Copyright (c) 2004,2005 Minero Aoki +# +# This program is free software. +# You can distribute and/or modify this program under the Ruby License. +# For details of Ruby License, see ruby/COPYING. +# + +require 'ripper/core' + +class Ripper + + # Tokenizes the Ruby program and returns an array of strings. + # + # p Ripper.tokenize("def m(a) nil end") + # # => ["def", " ", "m", "(", "a", ")", " ", "nil", " ", "end"] + # + def Ripper.tokenize(src, filename = '-', lineno = 1) + Lexer.new(src, filename, lineno).tokenize + end + + # Tokenizes the Ruby program and returns an array of an array, + # which is formatted like <code>[[lineno, column], type, token]</code>. + # + # require 'ripper' + # require 'pp' + # + # pp Ripper.lex("def m(a) nil end") + # #=> [[[1, 0], :on_kw, "def"], + # [[1, 3], :on_sp, " " ], + # [[1, 4], :on_ident, "m" ], + # [[1, 5], :on_lparen, "(" ], + # [[1, 6], :on_ident, "a" ], + # [[1, 7], :on_rparen, ")" ], + # [[1, 8], :on_sp, " " ], + # [[1, 9], :on_kw, "nil"], + # [[1, 12], :on_sp, " " ], + # [[1, 13], :on_kw, "end"]] + # + def Ripper.lex(src, filename = '-', lineno = 1) + Lexer.new(src, filename, lineno).lex + end + + class Lexer < ::Ripper #:nodoc: internal use only + def tokenize + lex().map {|pos, event, tok| tok } + end + + def lex + parse().sort_by {|pos, event, tok| pos } + end + + def parse + @buf = [] + super + @buf + end + + private + + SCANNER_EVENTS.each do |event| + module_eval(<<-End, __FILE__+'/module_eval', __LINE__ + 1) + def on_#{event}(tok) + @buf.push [[lineno(), column()], :on_#{event}, tok] + end + End + end + end + + # [EXPERIMENTAL] + # Parses +src+ and return a string which was matched to +pattern+. + # +pattern+ should be described as Regexp. + # + # require 'ripper' + # + # p Ripper.slice('def m(a) nil end', 'ident') #=> "m" + # p Ripper.slice('def m(a) nil end', '[ident lparen rparen]+') #=> "m(a)" + # p Ripper.slice("<<EOS\nstring\nEOS", + # 'heredoc_beg nl $(tstring_content*) heredoc_end', 1) + # #=> "string\n" + # + def Ripper.slice(src, pattern, n = 0) + if m = token_match(src, pattern) + then m.string(n) + else nil + end + end + + def Ripper.token_match(src, pattern) #:nodoc: + TokenPattern.compile(pattern).match(src) + end + + class TokenPattern #:nodoc: + + class Error < ::StandardError # :nodoc: + end + class CompileError < Error # :nodoc: + end + class MatchError < Error # :nodoc: + end + + class << self + alias compile new + end + + def initialize(pattern) + @source = pattern + @re = compile(pattern) + end + + def match(str) + match_list(::Ripper.lex(str)) + end + + def match_list(tokens) + if m = @re.match(map_tokens(tokens)) + then MatchData.new(tokens, m) + else nil + end + end + + private + + def compile(pattern) + if m = /[^\w\s$()\[\]{}?*+\.]/.match(pattern) + raise CompileError, "invalid char in pattern: #{m[0].inspect}" + end + buf = '' + pattern.scan(/(?:\w+|\$\(|[()\[\]\{\}?*+\.]+)/) do |tok| + case tok + when /\w/ + buf.concat map_token(tok) + when '$(' + buf.concat '(' + when '(' + buf.concat '(?:' + when /[?*\[\])\.]/ + buf.concat tok + else + raise 'must not happen' + end + end + Regexp.compile(buf) + rescue RegexpError => err + raise CompileError, err.message + end + + def map_tokens(tokens) + tokens.map {|pos,type,str| map_token(type.to_s.sub(/\Aon_/,'')) }.join + end + + MAP = {} + seed = ('a'..'z').to_a + ('A'..'Z').to_a + ('0'..'9').to_a + SCANNER_EVENT_TABLE.each do |ev, | + raise CompileError, "[RIPPER FATAL] too many system token" if seed.empty? + MAP[ev.to_s.sub(/\Aon_/,'')] = seed.shift + end + + def map_token(tok) + MAP[tok] or raise CompileError, "unknown token: #{tok}" + end + + class MatchData # :nodoc: + def initialize(tokens, match) + @tokens = tokens + @match = match + end + + def string(n = 0) + return nil unless @match + match(n).join + end + + private + + def match(n = 0) + return [] unless @match + @tokens[@match.begin(n)...@match.end(n)].map {|pos,type,str| str } + end + end + + end + +end diff --git a/ruby_2_2/ext/ripper/lib/ripper/sexp.rb b/ruby_2_2/ext/ripper/lib/ripper/sexp.rb new file mode 100644 index 0000000000..150c6f0cf8 --- /dev/null +++ b/ruby_2_2/ext/ripper/lib/ripper/sexp.rb @@ -0,0 +1,118 @@ +# +# $Id$ +# +# Copyright (c) 2004,2005 Minero Aoki +# +# This program is free software. +# You can distribute and/or modify this program under the Ruby License. +# For details of Ruby License, see ruby/COPYING. +# + +require 'ripper/core' + +class Ripper + + # [EXPERIMENTAL] + # Parses +src+ and create S-exp tree. + # Returns more readable tree rather than Ripper.sexp_raw. + # This method is mainly for developer use. + # + # require 'ripper' + # require 'pp' + # + # pp Ripper.sexp("def m(a) nil end") + # #=> [:program, + # [[:def, + # [:@ident, "m", [1, 4]], + # [:paren, [:params, [[:@ident, "a", [1, 6]]], nil, nil, nil, nil]], + # [:bodystmt, [[:var_ref, [:@kw, "nil", [1, 9]]]], nil, nil, nil]]]] + # + def Ripper.sexp(src, filename = '-', lineno = 1) + builder = SexpBuilderPP.new(src, filename, lineno) + sexp = builder.parse + sexp unless builder.error? + end + + # [EXPERIMENTAL] + # Parses +src+ and create S-exp tree. + # This method is mainly for developer use. + # + # require 'ripper' + # require 'pp' + # + # pp Ripper.sexp_raw("def m(a) nil end") + # #=> [:program, + # [:stmts_add, + # [:stmts_new], + # [:def, + # [:@ident, "m", [1, 4]], + # [:paren, [:params, [[:@ident, "a", [1, 6]]], nil, nil, nil]], + # [:bodystmt, + # [:stmts_add, [:stmts_new], [:var_ref, [:@kw, "nil", [1, 9]]]], + # nil, + # nil, + # nil]]]] + # + def Ripper.sexp_raw(src, filename = '-', lineno = 1) + builder = SexpBuilder.new(src, filename, lineno) + sexp = builder.parse + sexp unless builder.error? + end + + class SexpBuilderPP < ::Ripper #:nodoc: + private + + PARSER_EVENT_TABLE.each do |event, arity| + if /_new\z/ =~ event.to_s and arity == 0 + module_eval(<<-End, __FILE__, __LINE__ + 1) + def on_#{event} + [] + end + End + elsif /_add\z/ =~ event.to_s + module_eval(<<-End, __FILE__, __LINE__ + 1) + def on_#{event}(list, item) + list.push item + list + end + End + else + module_eval(<<-End, __FILE__, __LINE__ + 1) + def on_#{event}(*args) + [:#{event}, *args] + end + End + end + end + + SCANNER_EVENTS.each do |event| + module_eval(<<-End, __FILE__, __LINE__ + 1) + def on_#{event}(tok) + [:@#{event}, tok, [lineno(), column()]] + end + End + end + end + + class SexpBuilder < ::Ripper #:nodoc: + private + + PARSER_EVENTS.each do |event| + module_eval(<<-End, __FILE__, __LINE__ + 1) + def on_#{event}(*args) + args.unshift :#{event} + args + end + End + end + + SCANNER_EVENTS.each do |event| + module_eval(<<-End, __FILE__, __LINE__ + 1) + def on_#{event}(tok) + [:@#{event}, tok, [lineno(), column()]] + end + End + end + end + +end diff --git a/ruby_2_2/ext/ripper/tools/generate-param-macros.rb b/ruby_2_2/ext/ripper/tools/generate-param-macros.rb new file mode 100755 index 0000000000..b19f6e8d5c --- /dev/null +++ b/ruby_2_2/ext/ripper/tools/generate-param-macros.rb @@ -0,0 +1,14 @@ +off = true +ARGF.each do |line| + case line + when /RIPPER_PARAMS_DECL_BEGIN/ + off = false + when /RIPPER_PARAMS_DECL_END/ + exit + when /ripper/ + next if off + var = line.scan(/\w+/).last or next + base = var.sub(/ripper_/, '') + puts %"\#define #{base}\t\t(parser->ripper_#{base})" + end +end diff --git a/ruby_2_2/ext/ripper/tools/generate.rb b/ruby_2_2/ext/ripper/tools/generate.rb new file mode 100755 index 0000000000..06b57101d1 --- /dev/null +++ b/ruby_2_2/ext/ripper/tools/generate.rb @@ -0,0 +1,161 @@ +# $Id$ + +require 'optparse' + +def main + mode = nil + ids1src = nil + ids2src = nil + output = nil + + parser = @parser = OptionParser.new + parser.banner = "Usage: #{File.basename($0)} --mode=MODE [--ids1src=PATH] [--ids2src=PATH] [--output=PATH]" + parser.on('--mode=MODE', 'check, eventids1, or eventids2table.') {|m| + mode = m + } + parser.on('--ids1src=PATH', 'A source file of event-IDs 1 (parse.y).') {|path| + ids1src = path + } + parser.on('--ids2src=PATH', 'A source file of event-IDs 2 (eventids2.c).') {|path| + ids2src = path + } + parser.on('--output=PATH', 'An output file.') {|path| + output = path + } + parser.on('--help', 'Prints this message and quit.') { + puts parser.help + exit true + } + begin + parser.parse! + rescue OptionParser::ParseError => err + usage err.message + end + usage 'no mode given' unless mode + case mode + when 'check' + usage 'no --ids1src' unless ids1src + usage 'no --ids2src' unless ids2src + h = read_ids1_with_locations(ids1src) + check_arity h + ids2 = read_ids2(ids2src) + common = h.keys & ids2 + unless common.empty? + abort "event crash: #{common.join(' ')}" + end + exit 0 + when 'eventids1' + usage 'no --ids1src' unless ids1src + result = generate_eventids1(read_ids1(ids1src)) + when 'eventids2table' + usage 'no --ids2src' unless ids2src + result = generate_eventids2_table(read_ids2(ids2src)) + end + if output + File.open(output, 'w') {|f| + f.write result + } + else + puts result + end +end + +def usage(msg) + $stderr.puts msg + $stderr.puts @parser.help + exit false +end + +def generate_eventids1(ids) + buf = "" + buf << %Q[static struct {\n] + ids.each do |id, arity| + buf << %Q[ ID id_#{id};\n] + end + buf << %Q[} ripper_parser_ids;\n] + buf << %Q[\n] + ids.each do |id, arity| + buf << %Q[#define ripper_id_#{id} ripper_parser_ids.id_#{id}\n] + end + buf << %Q[\n] + buf << %Q[static void\n] + buf << %Q[ripper_init_eventids1(void)\n] + buf << %Q[{\n] + buf << %Q[#define set_id1(name) ripper_id_##name = rb_intern_const("on_"#name)\n] + ids.each do |id, arity| + buf << %Q[ set_id1(#{id});\n] + end + buf << %Q[}\n] + buf << %Q[\n] + buf << %Q[static void\n] + buf << %Q[ripper_init_eventids1_table(VALUE self)\n] + buf << %Q[{\n] + buf << %Q[ VALUE h = rb_hash_new();\n] + buf << %Q[ rb_define_const(self, "PARSER_EVENT_TABLE", h);\n] + ids.each do |id, arity| + buf << %Q[ rb_hash_aset(h, intern_sym("#{id}"), INT2FIX(#{arity}));\n] + end + buf << %Q[}\n] + buf +end + +def generate_eventids2_table(ids) + buf = "" + buf << %Q[static void\n] + buf << %Q[ripper_init_eventids2_table(VALUE self)\n] + buf << %Q[{\n] + buf << %Q[ VALUE h = rb_hash_new();\n] + buf << %Q[ rb_define_const(self, "SCANNER_EVENT_TABLE", h);\n] + ids.each do |id| + buf << %Q[ rb_hash_aset(h, intern_sym("#{id}"), INT2FIX(1));\n] + end + buf << %Q[}\n] + buf +end + +def read_ids1(path) + strip_locations(read_ids1_with_locations(path)) +end + +def strip_locations(h) + h.map {|event, list| [event, list.first[1]] }\ + .sort_by {|event, arity| event.to_s } +end + +def check_arity(h) + invalid = false + h.each do |event, list| + unless list.map {|line, arity| arity }.uniq.size == 1 + invalid = true + locations = list.map {|line, a| "#{line}:#{a}" }.join(', ') + $stderr.puts "arity crash [event=#{event}]: #{locations}" + end + end + abort if invalid +end + +def read_ids1_with_locations(path) + h = {} + File.open(path) {|f| + f.each do |line| + next if /\A\#\s*define\s+dispatch/ =~ line + next if /ripper_dispatch/ =~ line + line.scan(/\bdispatch(\d)\((\w+)/) do |arity, event| + (h[event] ||= []).push [f.lineno, arity.to_i] + end + end + } + h +end + +def read_ids2(path) + src = File.open(path) {|f| f.read} + ids2 = src.scan(/ID\s+ripper_id_(\w+)/).flatten.uniq.sort + diff = src.scan(/set_id2\((\w+)\);/).flatten - ids2 + unless diff.empty? + abort "missing scanner IDs: #{diff}" + end + return ids2 +end + +main diff --git a/ruby_2_2/ext/ripper/tools/preproc.rb b/ruby_2_2/ext/ripper/tools/preproc.rb new file mode 100755 index 0000000000..06397cea05 --- /dev/null +++ b/ruby_2_2/ext/ripper/tools/preproc.rb @@ -0,0 +1,91 @@ +# $Id$ + +require 'optparse' + +def main + output = nil + parser = OptionParser.new + parser.banner = "Usage: #{File.basename($0)} [--output=PATH] <parse.y>" + parser.on('--output=PATH', 'An output file.') {|path| + output = path + } + parser.on('--help', 'Prints this message and quit.') { + puts parser.help + exit true + } + begin + parser.parse! + rescue OptionParser::ParseError => err + $stderr.puts err.message + $stderr.puts parser.help + exit false + end + unless ARGV.size == 1 + abort "wrong number of arguments (#{ARGV.size} for 1)" + end + out = "" + File.open(ARGV[0]) {|f| + prelude f, out + grammar f, out + usercode f, out + } + if output + File.open(output, 'w') {|f| + f.write out + } + else + print out + end +end + +def prelude(f, out) + while line = f.gets + case line + when %r</\*%%%\*/> + out << '/*' << $/ + when %r</\*%> + out << '*/' << $/ + when %r<%\*/> + out << $/ + when /\A%%/ + out << '%%' << $/ + return + when /\A%token/ + out << line.sub(/<\w+>/, '<val>') + when /\A%type/ + out << line.sub(/<\w+>/, '<val>') + else + out << line + end + end +end + +def grammar(f, out) + while line = f.gets + case line + when %r</\*%%%\*/> + out << '#if 0' << $/ + when %r</\*%c%\*/> + out << '/*' << $/ + when %r</\*%c> + out << '*/' << $/ + when %r</\*%> + out << '#endif' << $/ + when %r<%\*/> + out << $/ + when /\A%%/ + out << '%%' << $/ + return + else + out << line + end + end +end + +def usercode(f, out) + while line = f.gets + out << line + end +end + +main diff --git a/ruby_2_2/ext/ripper/tools/strip.rb b/ruby_2_2/ext/ripper/tools/strip.rb new file mode 100755 index 0000000000..99413c361d --- /dev/null +++ b/ruby_2_2/ext/ripper/tools/strip.rb @@ -0,0 +1,12 @@ +last_is_void = false +ARGF.each do |line| + if line.strip.empty? + #puts() unless last_is_void + last_is_void = true + elsif /\A\#/ === line + ; + else + print line + last_is_void = false + end +end |