summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authoraamine <aamine@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>2004-09-20 05:40:23 +0000
committeraamine <aamine@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>2004-09-20 05:40:23 +0000
commit144339d4bce9a8e91d44ee9942dc31d7fd27c239 (patch)
treec3ea3b92d704c3b0be7514c84fec0c032eeb8198
parent9f18c235727fc3875b4423bd1ad7eba69f9520e9 (diff)
* parse.y [ripper]: adjust lineno and columns for multi-line strings.
* parse.y [ripper]: delay heredocument events until seeing end-of-line. * parse.y [ripper]: event on__heredoc_contentn -> on__tstring_content. * ext/ripper/eventids2.c: ditto. * ext/ripper/lib/ripper.rb: sync with eventids2.c. * test/ripper/test_scanner_events.rb: test it. * ext/ripper/tools/generate-ripper_rb.rb: show basename of input. * ext/ripper/Makefile.dev: support objdir build. git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@6931 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
-rw-r--r--ChangeLog21
-rw-r--r--ext/ripper/Makefile.dev2
-rw-r--r--ext/ripper/eventids2.c10
-rw-r--r--ext/ripper/lib/ripper.rb7
-rwxr-xr-xext/ripper/tools/generate-ripper_rb.rb2
-rw-r--r--parse.y108
-rw-r--r--test/ripper/test_scanner_events.rb14
7 files changed, 107 insertions, 57 deletions
diff --git a/ChangeLog b/ChangeLog
index 413d81c42f..cd9efc32e4 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,24 @@
+Mon Sep 20 14:39:42 2004 Minero Aoki <aamine@loveruby.net>
+
+ * parse.y [ripper]: adjust lineno and columns for multi-line
+ strings.
+
+ * parse.y [ripper]: delay heredocument events until seeing
+ end-of-line.
+
+ * parse.y [ripper]: event on__heredoc_contentn ->
+ on__tstring_content.
+
+ * ext/ripper/eventids2.c: ditto.
+
+ * ext/ripper/lib/ripper.rb: sync with eventids2.c.
+
+ * test/ripper/test_scanner_events.rb: test it.
+
+ * ext/ripper/tools/generate-ripper_rb.rb: show basename of input.
+
+ * ext/ripper/Makefile.dev: support objdir build.
+
Mon Sep 20 13:22:55 2004 Minero Aoki <aamine@loveruby.net>
* parse.y [ripper]: remove Ripper#pos.
diff --git a/ext/ripper/Makefile.dev b/ext/ripper/Makefile.dev
index 59615e56d6..ab16a7a74b 100644
--- a/ext/ripper/Makefile.dev
+++ b/ext/ripper/Makefile.dev
@@ -15,4 +15,4 @@ test:
preproc: ripper.E
ripper.E: ripper.c
- $(CC) -E $(CPPFLAGS) ripper.c | ruby tools/strip.rb > $@
+ $(CC) -E $(CPPFLAGS) ripper.c | ruby $(srcdir)/tools/strip.rb > $@
diff --git a/ext/ripper/eventids2.c b/ext/ripper/eventids2.c
index cab4b45657..f26b5f8433 100644
--- a/ext/ripper/eventids2.c
+++ b/ext/ripper/eventids2.c
@@ -5,10 +5,9 @@
#define tEMBDOC_END (tLAST_TOKEN + 5)
#define tSP (tLAST_TOKEN + 6)
#define tHEREDOC_BEG (tLAST_TOKEN + 7)
-#define tHEREDOC_CONTENT (tLAST_TOKEN + 8)
-#define tHEREDOC_END (tLAST_TOKEN + 9)
-#define k__END__ (tLAST_TOKEN + 10)
-#define tCHAR (tLAST_TOKEN + 11)
+#define tHEREDOC_END (tLAST_TOKEN + 8)
+#define k__END__ (tLAST_TOKEN + 9)
+#define tCHAR (tLAST_TOKEN + 10)
static ID ripper_id_scan;
@@ -53,7 +52,6 @@ static ID ripper_id_embdoc;
static ID ripper_id_embdoc_end;
static ID ripper_id_sp;
static ID ripper_id_heredoc_beg;
-static ID ripper_id_heredoc_content;
static ID ripper_id_heredoc_end;
static ID ripper_id___end__;
static ID ripper_id_CHAR;
@@ -104,7 +102,6 @@ ripper_init_eventids2()
ripper_id_embdoc_end = rb_intern("on__embdoc_end");
ripper_id_sp = rb_intern("on__sp");
ripper_id_heredoc_beg = rb_intern("on__heredoc_beg");
- ripper_id_heredoc_content = rb_intern("on__heredoc_content");
ripper_id_heredoc_end = rb_intern("on__heredoc_end");
ripper_id___end__ = rb_intern("on____end__");
ripper_id_CHAR = rb_intern("on__CHAR");
@@ -250,7 +247,6 @@ static struct token_assoc {
{tEMBDOC_END, &ripper_id_embdoc_end},
{tSP, &ripper_id_sp},
{tHEREDOC_BEG, &ripper_id_heredoc_beg},
- {tHEREDOC_CONTENT, &ripper_id_heredoc_content},
{tHEREDOC_END, &ripper_id_heredoc_end},
{k__END__, &ripper_id___end__},
{0, NULL}
diff --git a/ext/ripper/lib/ripper.rb b/ext/ripper/lib/ripper.rb
index 16db062ad4..c0715de988 100644
--- a/ext/ripper/lib/ripper.rb
+++ b/ext/ripper/lib/ripper.rb
@@ -1,5 +1,5 @@
#
-# This file is automatically generated from /home/aamine/c/ruby/ext/ripper/ripper.rb.in and parse.y.
+# This file is automatically generated from ripper.rb.in and parse.y.
# DO NOT MODIFY!!!!!!
#
#
@@ -165,7 +165,6 @@ class Ripper
:float => 1,
:gvar => 1,
:heredoc_beg => 1,
- :heredoc_content => 1,
:heredoc_end => 1,
:ident => 1,
:ignored_nl => 1,
@@ -774,10 +773,6 @@ class Ripper
token
end
- def on__heredoc_content(token)
- token
- end
-
def on__heredoc_end(token)
token
end
diff --git a/ext/ripper/tools/generate-ripper_rb.rb b/ext/ripper/tools/generate-ripper_rb.rb
index b85d2745a0..b6fb166568 100755
--- a/ext/ripper/tools/generate-ripper_rb.rb
+++ b/ext/ripper/tools/generate-ripper_rb.rb
@@ -4,7 +4,7 @@ def main
template, ids1, ids2 = *ARGV
print <<header
#
-# This file is automatically generated from #{template} and parse.y.
+# This file is automatically generated from #{File.basename(template)} and parse.y.
# DO NOT MODIFY!!!!!!
#
header
diff --git a/parse.y b/parse.y
index 068542473d..214a0f2914 100644
--- a/parse.y
+++ b/parse.y
@@ -144,8 +144,10 @@ struct parser_params {
int parser_ruby__end__seen;
int parser_ruby_sourceline;
VALUE parser_ruby_sourcefile;
- VALUE delayed;
char *tokp;
+ int current_t;
+ int in_heredoc;
+ VALUE delayed;
#endif
};
@@ -4148,7 +4150,9 @@ none : /* none */
# undef yylval
# define yylval (*((YYSTYPE*)(parser->parser_yylval)))
+#ifndef RIPPER
static struct parser_params* parser_new _((void));
+#endif
static int parser_regx_options _((struct parser_params*));
static int parser_tokadd_string _((struct parser_params*,int,int,int,long*));
static int parser_parse_string _((struct parser_params*,NODE*));
@@ -4188,46 +4192,62 @@ static int parser_here_document _((struct parser_params*,NODE*));
#endif
#ifdef RIPPER
-static void
-ripper_save_token(parser)
- struct parser_params *parser;
-{
- rb_ary_push(parser->delayed,
- rb_str_new(parser->tokp, lex_pend - parser->tokp));
-}
+#define ripper_flush(p) (p->tokp = p->parser_lex_p)
static void
-ripper_dispatch_delayed(parser, t)
+ripper_dispatch_scan_event(parser, t)
struct parser_params *parser;
int t;
{
- long i;
-
- for (i = 0; i < RARRAY(parser->delayed)->len; i++) {
- ID event = ripper_token2eventid(t);
-
- ripper_dispatch2(parser, ripper_id_scan,
- ID2SYM(event), RARRAY(parser->delayed)->ptr[i]);
- ripper_dispatch1(parser, event, RARRAY(parser->delayed)->ptr[i]);
- }
- rb_ary_clear(parser->delayed);
+ VALUE str;
+ ID event;
+
+ if (lex_p < parser->tokp) rb_raise(rb_eRuntimeError, "lex_p < tokp");
+ if (lex_p == parser->tokp) return;
+ str = rb_str_new(parser->tokp, lex_p - parser->tokp);
+ event = ripper_token2eventid(t);
+ if (parser->in_heredoc) {
+ int col = parser->tokp - lex_pbeg;
+ rb_ary_push(parser->delayed, ID2SYM(event));
+ rb_ary_push(parser->delayed, str);
+ rb_ary_push(parser->delayed, INT2NUM(ruby_sourceline));
+ rb_ary_push(parser->delayed, INT2NUM(col));
+ ripper_flush(parser);
+ return;
+ }
+ ripper_dispatch2(parser, ripper_id_scan, ID2SYM(event), rb_str_dup(str));
+ ripper_dispatch1(parser, event, str);
+ ripper_flush(parser);
}
-#define ripper_flush(p) (p->tokp = p->parser_lex_p)
-
static void
-ripper_dispatch_scan_event(parser, t)
+ripper_dispatch_delayed_events(parser)
struct parser_params *parser;
- int t;
{
- if (lex_p > parser->tokp) {
- VALUE str = rb_str_new(parser->tokp, lex_p - parser->tokp);
- ID event = ripper_token2eventid(t);
-
- ripper_dispatch2(parser, ripper_id_scan, ID2SYM(event), rb_str_dup(str));
- ripper_dispatch1(parser, event, str);
- ripper_flush(parser);
+ long i;
+ int saved_line;
+ char *saved_tokp;
+
+ if (RARRAY(parser->delayed)->len == 0) return;
+ if (RARRAY(parser->delayed)->len % 4 != 0)
+ rb_raise(rb_eRuntimeError, "[RIPPER BUG] parser->delayed % 4 != 0");
+ saved_line = ruby_sourceline;
+ saved_tokp = parser->tokp;
+ i = 0;
+ while (i < RARRAY(parser->delayed)->len) {
+ VALUE event, str, vline, vcol;
+ event = RARRAY(parser->delayed)->ptr[i++];
+ str = RARRAY(parser->delayed)->ptr[i++];
+ vline = RARRAY(parser->delayed)->ptr[i++];
+ vcol = RARRAY(parser->delayed)->ptr[i++];
+ ruby_sourceline = NUM2INT(vline);
+ parser->tokp = lex_pbeg + NUM2INT(vcol);
+ ripper_dispatch2(parser, ripper_id_scan, event, rb_str_dup(str));
+ ripper_dispatch1(parser, SYM2ID(event), str);
}
+ rb_ary_clear(parser->delayed);
+ ruby_sourceline = saved_line;
+ parser->tokp = saved_tokp;
}
#endif /* RIPPER */
@@ -4465,9 +4485,11 @@ parser_nextc(parser)
return -1;
}
#ifdef RIPPER
- if (parser->tokp < lex_pend) {
- ripper_save_token(parser);
- }
+ if (parser->tokp < lex_pend) {
+ if (!parser->current_t)
+ rb_raise(rb_eRuntimeError, "[Ripper BUG] no current_t");
+ ripper_dispatch_scan_event(parser, parser->current_t);
+ }
#endif
if (heredoc_end > 0) {
ruby_sourceline = heredoc_end;
@@ -4917,6 +4939,9 @@ parser_parse_string(parser, quote)
if (func == -1) return tSTRING_END;
c = nextc();
if ((func & STR_FUNC_QWORDS) && ISSPACE(c)) {
+#ifdef RIPPER
+ parser->current_t = ' ';
+#endif
do {c = nextc();} while (ISSPACE(c));
space = 1;
}
@@ -5032,7 +5057,6 @@ parser_heredoc_restore(parser, here)
VALUE line;
#ifdef RIPPER
- ripper_dispatch_delayed(parser, tHEREDOC_CONTENT);
lex_goto_eol(parser);
ripper_dispatch_scan_event(parser, tHEREDOC_END);
#endif
@@ -5047,6 +5071,7 @@ parser_heredoc_restore(parser, here)
rb_gc_force_recycle((VALUE)here);
#ifdef RIPPER
ripper_flush(parser);
+ parser->in_heredoc = Qfalse;
#endif
}
@@ -5197,7 +5222,13 @@ parser_yylex(parser)
if (lex_strterm) {
int token;
+#ifdef RIPPER
+ parser->current_t = tSTRING_CONTENT;
+#endif
if (nd_type(lex_strterm) == NODE_HEREDOC) {
+#ifdef RIPPER
+ parser->in_heredoc = Qtrue;
+#endif
token = here_document(lex_strterm);
if (token == tSTRING_END) {
lex_strterm = 0;
@@ -6413,9 +6444,11 @@ yylex(p)
#endif
t = parser_yylex(parser);
#ifdef RIPPER
+ parser->current_t = 0;
if (t != 0) {
- ripper_dispatch_delayed(parser, t);
ripper_dispatch_scan_event(parser, t);
+ if (t == '\n' && !parser->in_heredoc)
+ ripper_dispatch_delayed_events(parser);
}
#endif
@@ -8105,6 +8138,8 @@ parser_initialize(parser)
parser->parser_lex_p = 0;
parser->parser_lex_pend = 0;
#ifdef RIPPER
+ parser->current_t = 0;
+ parser->in_heredoc = Qfalse;
parser->delayed = rb_ary_new();
#endif
}
@@ -8139,6 +8174,7 @@ ripper_free(ptr)
free(p);
}
+#ifndef RIPPER
struct parser_params *
parser_new()
{
@@ -8150,9 +8186,9 @@ parser_new()
parser_initialize(p);
return p;
}
+#endif
#ifdef RIPPER
-
#ifdef RIPPER_DEBUG
extern int rb_is_pointer_to_heap _((VALUE));
diff --git a/test/ripper/test_scanner_events.rb b/test/ripper/test_scanner_events.rb
index a4dd37bad7..6b95c4f5dd 100644
--- a/test/ripper/test_scanner_events.rb
+++ b/test/ripper/test_scanner_events.rb
@@ -573,19 +573,21 @@ class TestRipper_ScannerEvents < Test::Unit::TestCase
R.scan('heredoc_beg', "<<'EOS'\nheredoc\nEOS")
assert_equal [%q(<<`EOS`)],
R.scan('heredoc_beg', "<<`EOS`\nheredoc\nEOS")
+ assert_equal [%q(<<" ")],
+ R.scan('heredoc_beg', %Q[<<" "\nheredoc\nEOS])
end
- def test_heredoc_content
+ def test_tstring_content_HEREDOC
assert_equal [],
- R.scan('heredoc_content', '')
+ R.scan('tstring_content', '')
assert_equal ["heredoc\n"],
- R.scan('heredoc_content', "<<EOS\nheredoc\nEOS")
+ R.scan('tstring_content', "<<EOS\nheredoc\nEOS")
assert_equal ["heredoc\n"],
- R.scan('heredoc_content', "<<EOS\nheredoc\nEOS\n")
+ R.scan('tstring_content', "<<EOS\nheredoc\nEOS\n")
assert_equal ["heredoc \n"],
- R.scan('heredoc_content', "<<EOS\nheredoc \nEOS \n")
+ R.scan('tstring_content', "<<EOS\nheredoc \nEOS \n")
assert_equal ["heredoc\n"],
- R.scan('heredoc_content', "<<-EOS\nheredoc\n\tEOS \n")
+ R.scan('tstring_content', "<<-EOS\nheredoc\n\tEOS \n")
end
def test_heredoc_end