summaryrefslogtreecommitdiff
path: root/ext/ripper
diff options
context:
space:
mode:
authoryui-knk <spiketeika@gmail.com>2022-09-23 22:40:02 +0900
committerYuichiro Kaneko <spiketeika@gmail.com>2022-11-21 09:01:34 +0900
commitd8601621edcf29e3323b90dcf04b774edd9fb45e (patch)
treefd9b08ca7008ce5ad7f8acafd83accff2347697f /ext/ripper
parentbbc4cf5f76c0e17e645ba81ae0b3625c8cbfc4c4 (diff)
Enhance keep_tokens option for RubyVM::AbstractSyntaxTree parsing methods
Implementation for Language Server Protocol (LSP) sometimes needs token information. For example both `m(1)` and `m(1, )` has same AST structure other than node locations then it's impossible to check the existence of `,` from AST. However in later case, it might be better to suggest variables list for the second argument. Token information is important for such case. This commit adds these methods. * Add `keep_tokens` option for `RubyVM::AbstractSyntaxTree.parse`, `.parse_file` and `.of` * Add `RubyVM::AbstractSyntaxTree::Node#tokens` which returns tokens for the node including tokens for descendants nodes. * Add `RubyVM::AbstractSyntaxTree::Node#all_tokens` which returns all tokens for the input script regardless the receiver node. [Feature #19070] Impacts on memory usage and performance are below: Memory usage: ``` $ cat test.rb root = RubyVM::AbstractSyntaxTree.parse_file(File.expand_path('../test/ruby/test_keyword.rb', __FILE__), keep_tokens: true) $ /usr/bin/time -f %Mkb /usr/local/bin/ruby -v ruby 3.2.0dev (2022-11-19T09:41:54Z 19070-keep_tokens d3af1b8057) [x86_64-linux] 11408kb # keep_tokens :false $ /usr/bin/time -f %Mkb /usr/local/bin/ruby test.rb 17508kb # keep_tokens :true $ /usr/bin/time -f %Mkb /usr/local/bin/ruby test.rb 30960kb ``` Performance: ``` $ cat ../ast_keep_tokens.yml prelude: | src = <<~SRC module M class C def m1(a, b) 1 + a + b end end end SRC benchmark: without_keep_tokens: | RubyVM::AbstractSyntaxTree.parse(src, keep_tokens: false) with_keep_tokens: | RubyVM::AbstractSyntaxTree.parse(src, keep_tokens: true) $ make benchmark COMPARE_RUBY="./ruby" ARGS=../ast_keep_tokens.yml /home/kaneko.y/.rbenv/shims/ruby --disable=gems -rrubygems -I../benchmark/lib ../benchmark/benchmark-driver/exe/benchmark-driver \ --executables="compare-ruby::./ruby -I.ext/common --disable-gem" \ --executables="built-ruby::./miniruby -I../lib -I. -I.ext/common ../tool/runruby.rb --extout=.ext -- --disable-gems --disable-gem" \ --output=markdown --output-compare -v ../ast_keep_tokens.yml compare-ruby: ruby 3.2.0dev (2022-11-19T09:41:54Z 19070-keep_tokens d3af1b8057) [x86_64-linux] built-ruby: ruby 3.2.0dev (2022-11-19T09:41:54Z 19070-keep_tokens d3af1b8057) [x86_64-linux] warming up.. | |compare-ruby|built-ruby| |:--------------------|-----------:|---------:| |without_keep_tokens | 21.659k| 21.303k| | | 1.02x| -| |with_keep_tokens | 6.220k| 5.691k| | | 1.09x| -| ```
Notes
Notes: Merged: https://github.com/ruby/ruby/pull/6770
Diffstat (limited to 'ext/ripper')
-rw-r--r--ext/ripper/eventids2.c19
1 files changed, 0 insertions, 19 deletions
diff --git a/ext/ripper/eventids2.c b/ext/ripper/eventids2.c
index ac38663f2d..05687497ac 100644
--- a/ext/ripper/eventids2.c
+++ b/ext/ripper/eventids2.c
@@ -1,22 +1,3 @@
-enum {
- tIGNORED_NL = tLAST_TOKEN + 1,
-# define tIGNORED_NL ((enum yytokentype)tIGNORED_NL)
- tCOMMENT,
-# define tCOMMENT ((enum yytokentype)tCOMMENT)
- tEMBDOC_BEG,
-# define tEMBDOC_BEG ((enum yytokentype)tEMBDOC_BEG)
- tEMBDOC,
-# define tEMBDOC ((enum yytokentype)tEMBDOC)
- tEMBDOC_END,
-# define tEMBDOC_END ((enum yytokentype)tEMBDOC_END)
- tHEREDOC_BEG,
-# define tHEREDOC_BEG ((enum yytokentype)tHEREDOC_BEG)
- tHEREDOC_END,
-# define tHEREDOC_END ((enum yytokentype)tHEREDOC_END)
- k__END__,
-# define k__END__ ((enum yytokentype)k__END__)
-};
-
typedef struct {
ID ripper_id_backref;
ID ripper_id_backtick;