From 144339d4bce9a8e91d44ee9942dc31d7fd27c239 Mon Sep 17 00:00:00 2001 From: aamine Date: Mon, 20 Sep 2004 05:40:23 +0000 Subject: * parse.y [ripper]: adjust lineno and columns for multi-line strings. * parse.y [ripper]: delay heredocument events until seeing end-of-line. * parse.y [ripper]: event on__heredoc_contentn -> on__tstring_content. * ext/ripper/eventids2.c: ditto. * ext/ripper/lib/ripper.rb: sync with eventids2.c. * test/ripper/test_scanner_events.rb: test it. * ext/ripper/tools/generate-ripper_rb.rb: show basename of input. * ext/ripper/Makefile.dev: support objdir build. git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@6931 b2dd03c8-39d4-4d8f-98ff-823fe69b080e --- ChangeLog | 21 +++++++ ext/ripper/Makefile.dev | 2 +- ext/ripper/eventids2.c | 10 +-- ext/ripper/lib/ripper.rb | 7 +-- ext/ripper/tools/generate-ripper_rb.rb | 2 +- parse.y | 108 ++++++++++++++++++++++----------- test/ripper/test_scanner_events.rb | 14 +++-- 7 files changed, 107 insertions(+), 57 deletions(-) diff --git a/ChangeLog b/ChangeLog index 413d81c42f..cd9efc32e4 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,24 @@ +Mon Sep 20 14:39:42 2004 Minero Aoki + + * parse.y [ripper]: adjust lineno and columns for multi-line + strings. + + * parse.y [ripper]: delay heredocument events until seeing + end-of-line. + + * parse.y [ripper]: event on__heredoc_contentn -> + on__tstring_content. + + * ext/ripper/eventids2.c: ditto. + + * ext/ripper/lib/ripper.rb: sync with eventids2.c. + + * test/ripper/test_scanner_events.rb: test it. + + * ext/ripper/tools/generate-ripper_rb.rb: show basename of input. + + * ext/ripper/Makefile.dev: support objdir build. + Mon Sep 20 13:22:55 2004 Minero Aoki * parse.y [ripper]: remove Ripper#pos. diff --git a/ext/ripper/Makefile.dev b/ext/ripper/Makefile.dev index 59615e56d6..ab16a7a74b 100644 --- a/ext/ripper/Makefile.dev +++ b/ext/ripper/Makefile.dev @@ -15,4 +15,4 @@ test: preproc: ripper.E ripper.E: ripper.c - $(CC) -E $(CPPFLAGS) ripper.c | ruby tools/strip.rb > $@ + $(CC) -E $(CPPFLAGS) ripper.c | ruby $(srcdir)/tools/strip.rb > $@ diff --git a/ext/ripper/eventids2.c b/ext/ripper/eventids2.c index cab4b45657..f26b5f8433 100644 --- a/ext/ripper/eventids2.c +++ b/ext/ripper/eventids2.c @@ -5,10 +5,9 @@ #define tEMBDOC_END (tLAST_TOKEN + 5) #define tSP (tLAST_TOKEN + 6) #define tHEREDOC_BEG (tLAST_TOKEN + 7) -#define tHEREDOC_CONTENT (tLAST_TOKEN + 8) -#define tHEREDOC_END (tLAST_TOKEN + 9) -#define k__END__ (tLAST_TOKEN + 10) -#define tCHAR (tLAST_TOKEN + 11) +#define tHEREDOC_END (tLAST_TOKEN + 8) +#define k__END__ (tLAST_TOKEN + 9) +#define tCHAR (tLAST_TOKEN + 10) static ID ripper_id_scan; @@ -53,7 +52,6 @@ static ID ripper_id_embdoc; static ID ripper_id_embdoc_end; static ID ripper_id_sp; static ID ripper_id_heredoc_beg; -static ID ripper_id_heredoc_content; static ID ripper_id_heredoc_end; static ID ripper_id___end__; static ID ripper_id_CHAR; @@ -104,7 +102,6 @@ ripper_init_eventids2() ripper_id_embdoc_end = rb_intern("on__embdoc_end"); ripper_id_sp = rb_intern("on__sp"); ripper_id_heredoc_beg = rb_intern("on__heredoc_beg"); - ripper_id_heredoc_content = rb_intern("on__heredoc_content"); ripper_id_heredoc_end = rb_intern("on__heredoc_end"); ripper_id___end__ = rb_intern("on____end__"); ripper_id_CHAR = rb_intern("on__CHAR"); @@ -250,7 +247,6 @@ static struct token_assoc { {tEMBDOC_END, &ripper_id_embdoc_end}, {tSP, &ripper_id_sp}, {tHEREDOC_BEG, &ripper_id_heredoc_beg}, - {tHEREDOC_CONTENT, &ripper_id_heredoc_content}, {tHEREDOC_END, &ripper_id_heredoc_end}, {k__END__, &ripper_id___end__}, {0, NULL} diff --git a/ext/ripper/lib/ripper.rb b/ext/ripper/lib/ripper.rb index 16db062ad4..c0715de988 100644 --- a/ext/ripper/lib/ripper.rb +++ b/ext/ripper/lib/ripper.rb @@ -1,5 +1,5 @@ # -# This file is automatically generated from /home/aamine/c/ruby/ext/ripper/ripper.rb.in and parse.y. +# This file is automatically generated from ripper.rb.in and parse.y. # DO NOT MODIFY!!!!!! # # @@ -165,7 +165,6 @@ class Ripper :float => 1, :gvar => 1, :heredoc_beg => 1, - :heredoc_content => 1, :heredoc_end => 1, :ident => 1, :ignored_nl => 1, @@ -774,10 +773,6 @@ class Ripper token end - def on__heredoc_content(token) - token - end - def on__heredoc_end(token) token end diff --git a/ext/ripper/tools/generate-ripper_rb.rb b/ext/ripper/tools/generate-ripper_rb.rb index b85d2745a0..b6fb166568 100755 --- a/ext/ripper/tools/generate-ripper_rb.rb +++ b/ext/ripper/tools/generate-ripper_rb.rb @@ -4,7 +4,7 @@ def main template, ids1, ids2 = *ARGV print <
parser_yylval))) +#ifndef RIPPER static struct parser_params* parser_new _((void)); +#endif static int parser_regx_options _((struct parser_params*)); static int parser_tokadd_string _((struct parser_params*,int,int,int,long*)); static int parser_parse_string _((struct parser_params*,NODE*)); @@ -4188,46 +4192,62 @@ static int parser_here_document _((struct parser_params*,NODE*)); #endif #ifdef RIPPER -static void -ripper_save_token(parser) - struct parser_params *parser; -{ - rb_ary_push(parser->delayed, - rb_str_new(parser->tokp, lex_pend - parser->tokp)); -} +#define ripper_flush(p) (p->tokp = p->parser_lex_p) static void -ripper_dispatch_delayed(parser, t) +ripper_dispatch_scan_event(parser, t) struct parser_params *parser; int t; { - long i; - - for (i = 0; i < RARRAY(parser->delayed)->len; i++) { - ID event = ripper_token2eventid(t); - - ripper_dispatch2(parser, ripper_id_scan, - ID2SYM(event), RARRAY(parser->delayed)->ptr[i]); - ripper_dispatch1(parser, event, RARRAY(parser->delayed)->ptr[i]); - } - rb_ary_clear(parser->delayed); + VALUE str; + ID event; + + if (lex_p < parser->tokp) rb_raise(rb_eRuntimeError, "lex_p < tokp"); + if (lex_p == parser->tokp) return; + str = rb_str_new(parser->tokp, lex_p - parser->tokp); + event = ripper_token2eventid(t); + if (parser->in_heredoc) { + int col = parser->tokp - lex_pbeg; + rb_ary_push(parser->delayed, ID2SYM(event)); + rb_ary_push(parser->delayed, str); + rb_ary_push(parser->delayed, INT2NUM(ruby_sourceline)); + rb_ary_push(parser->delayed, INT2NUM(col)); + ripper_flush(parser); + return; + } + ripper_dispatch2(parser, ripper_id_scan, ID2SYM(event), rb_str_dup(str)); + ripper_dispatch1(parser, event, str); + ripper_flush(parser); } -#define ripper_flush(p) (p->tokp = p->parser_lex_p) - static void -ripper_dispatch_scan_event(parser, t) +ripper_dispatch_delayed_events(parser) struct parser_params *parser; - int t; { - if (lex_p > parser->tokp) { - VALUE str = rb_str_new(parser->tokp, lex_p - parser->tokp); - ID event = ripper_token2eventid(t); - - ripper_dispatch2(parser, ripper_id_scan, ID2SYM(event), rb_str_dup(str)); - ripper_dispatch1(parser, event, str); - ripper_flush(parser); + long i; + int saved_line; + char *saved_tokp; + + if (RARRAY(parser->delayed)->len == 0) return; + if (RARRAY(parser->delayed)->len % 4 != 0) + rb_raise(rb_eRuntimeError, "[RIPPER BUG] parser->delayed % 4 != 0"); + saved_line = ruby_sourceline; + saved_tokp = parser->tokp; + i = 0; + while (i < RARRAY(parser->delayed)->len) { + VALUE event, str, vline, vcol; + event = RARRAY(parser->delayed)->ptr[i++]; + str = RARRAY(parser->delayed)->ptr[i++]; + vline = RARRAY(parser->delayed)->ptr[i++]; + vcol = RARRAY(parser->delayed)->ptr[i++]; + ruby_sourceline = NUM2INT(vline); + parser->tokp = lex_pbeg + NUM2INT(vcol); + ripper_dispatch2(parser, ripper_id_scan, event, rb_str_dup(str)); + ripper_dispatch1(parser, SYM2ID(event), str); } + rb_ary_clear(parser->delayed); + ruby_sourceline = saved_line; + parser->tokp = saved_tokp; } #endif /* RIPPER */ @@ -4465,9 +4485,11 @@ parser_nextc(parser) return -1; } #ifdef RIPPER - if (parser->tokp < lex_pend) { - ripper_save_token(parser); - } + if (parser->tokp < lex_pend) { + if (!parser->current_t) + rb_raise(rb_eRuntimeError, "[Ripper BUG] no current_t"); + ripper_dispatch_scan_event(parser, parser->current_t); + } #endif if (heredoc_end > 0) { ruby_sourceline = heredoc_end; @@ -4917,6 +4939,9 @@ parser_parse_string(parser, quote) if (func == -1) return tSTRING_END; c = nextc(); if ((func & STR_FUNC_QWORDS) && ISSPACE(c)) { +#ifdef RIPPER + parser->current_t = ' '; +#endif do {c = nextc();} while (ISSPACE(c)); space = 1; } @@ -5032,7 +5057,6 @@ parser_heredoc_restore(parser, here) VALUE line; #ifdef RIPPER - ripper_dispatch_delayed(parser, tHEREDOC_CONTENT); lex_goto_eol(parser); ripper_dispatch_scan_event(parser, tHEREDOC_END); #endif @@ -5047,6 +5071,7 @@ parser_heredoc_restore(parser, here) rb_gc_force_recycle((VALUE)here); #ifdef RIPPER ripper_flush(parser); + parser->in_heredoc = Qfalse; #endif } @@ -5197,7 +5222,13 @@ parser_yylex(parser) if (lex_strterm) { int token; +#ifdef RIPPER + parser->current_t = tSTRING_CONTENT; +#endif if (nd_type(lex_strterm) == NODE_HEREDOC) { +#ifdef RIPPER + parser->in_heredoc = Qtrue; +#endif token = here_document(lex_strterm); if (token == tSTRING_END) { lex_strterm = 0; @@ -6413,9 +6444,11 @@ yylex(p) #endif t = parser_yylex(parser); #ifdef RIPPER + parser->current_t = 0; if (t != 0) { - ripper_dispatch_delayed(parser, t); ripper_dispatch_scan_event(parser, t); + if (t == '\n' && !parser->in_heredoc) + ripper_dispatch_delayed_events(parser); } #endif @@ -8105,6 +8138,8 @@ parser_initialize(parser) parser->parser_lex_p = 0; parser->parser_lex_pend = 0; #ifdef RIPPER + parser->current_t = 0; + parser->in_heredoc = Qfalse; parser->delayed = rb_ary_new(); #endif } @@ -8139,6 +8174,7 @@ ripper_free(ptr) free(p); } +#ifndef RIPPER struct parser_params * parser_new() { @@ -8150,9 +8186,9 @@ parser_new() parser_initialize(p); return p; } +#endif #ifdef RIPPER - #ifdef RIPPER_DEBUG extern int rb_is_pointer_to_heap _((VALUE)); diff --git a/test/ripper/test_scanner_events.rb b/test/ripper/test_scanner_events.rb index a4dd37bad7..6b95c4f5dd 100644 --- a/test/ripper/test_scanner_events.rb +++ b/test/ripper/test_scanner_events.rb @@ -573,19 +573,21 @@ class TestRipper_ScannerEvents < Test::Unit::TestCase R.scan('heredoc_beg', "<<'EOS'\nheredoc\nEOS") assert_equal [%q(<<`EOS`)], R.scan('heredoc_beg', "<<`EOS`\nheredoc\nEOS") + assert_equal [%q(<<" ")], + R.scan('heredoc_beg', %Q[<<" "\nheredoc\nEOS]) end - def test_heredoc_content + def test_tstring_content_HEREDOC assert_equal [], - R.scan('heredoc_content', '') + R.scan('tstring_content', '') assert_equal ["heredoc\n"], - R.scan('heredoc_content', "<