Add RDoc

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@5073 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

1 files changed, 25 insertions, 0 deletions

diff --git a/lib/rdoc/tokenstream.rb b/lib/rdoc/tokenstream.rb
new file mode 100644
index 0000000000..0a0720d8a9
--- /dev/null
+++ b/lib/rdoc/tokenstream.rb
@@ -0,0 +1,25 @@
+# A TokenStream is a list of tokens, gathered during the parse
+# of some entity (say a method). Entities populate these streams
+# by being registered with the lexer. Any class can collect tokens
+# by including TokenStream. From the outside, you use such an object
+# by calling the start_collecting_tokens method, followed by calls
+# to add_token and pop_token
+
+module TokenStream
+  def token_stream
+    @token_stream
+  end
+
+  def start_collecting_tokens
+    @token_stream = []
+  end
+  def add_token(tk)
+    @token_stream << tk
+  end
+  def add_tokens(tks)
+    tks.each  {|tk| add_token(tk)}
+  end
+  def pop_token
+    @token_stream.pop
+  end
+end