added tag v1_9_0_4

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/tags/v1_9_0_4@18845 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

1 files changed, 33 insertions, 0 deletions

diff --git a/trunk/lib/rdoc/tokenstream.rb b/trunk/lib/rdoc/tokenstream.rb
new file mode 100644
index 0000000000..0a1eb9130b
--- /dev/null
+++ b/trunk/lib/rdoc/tokenstream.rb
@@ -0,0 +1,33 @@
+module RDoc; end
+
+##
+# A TokenStream is a list of tokens, gathered during the parse of some entity
+# (say a method). Entities populate these streams by being registered with the
+# lexer. Any class can collect tokens by including TokenStream. From the
+# outside, you use such an object by calling the start_collecting_tokens
+# method, followed by calls to add_token and pop_token.
+
+module RDoc::TokenStream
+
+  def token_stream
+    @token_stream
+  end
+
+  def start_collecting_tokens
+    @token_stream = []
+  end
+
+  def add_token(tk)
+    @token_stream << tk
+  end
+
+  def add_tokens(tks)
+    tks.each  {|tk| add_token(tk)}
+  end
+
+  def pop_token
+    @token_stream.pop
+  end
+
+end
+