webrick/httpproxy: stream request and response bodies

Reading entire request or response bodies into memory can lead
to trivial denial-of-service attacks.  Introduce Fibers in both
cases to allow streaming.

WEBrick::HTTPRequest gains a new body_reader method to prepare
itself as a source for IO.copy_stream.  This allows the
WEBrick::HTTPRequest object to be used as the
Net::HTTPGenericRequest#body_stream= arg for Net::HTTP.

For HTTP proxy response bodies, we also use a Fiber to
to make the HTTP request and read the response body.

* lib/webrick/httprequest.rb (body_reader): new method
  (readpartial): ditto
* lib/webrick/httpproxy.rb (perform_proxy_request): use Fiber
  to stream response body
  (do_GET, do_HEAD): adjust call
  (do_POST): adjust call and supply body_reader
* test/webrick/test_httprequest.rb (test_chunked): test
  for IO.copy_stream compatibility
* test/webrick/test_httpproxy.rb (test_big_bodies): new test

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/branches/ruby_2_5@62985 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

2 files changed, 103 insertions, 1 deletions

diff --git a/test/webrick/test_httpproxy.rb b/test/webrick/test_httpproxy.rb
index 452e7b94b7..52ecc91b28 100644
--- a/test/webrick/test_httpproxy.rb
+++ b/test/webrick/test_httpproxy.rb
@@ -118,6 +118,100 @@ class TestWEBrickHTTPProxy < Test::Unit::TestCase
     }
   end
 
+  def test_big_bodies
+    require 'digest/md5'
+    rand_str = File.read(__FILE__)
+    rand_str.freeze
+    nr = 1024 ** 2 / rand_str.size # bigger works, too
+    exp = Digest::MD5.new
+    nr.times { exp.update(rand_str) }
+    exp = exp.hexdigest
+    TestWEBrick.start_httpserver do |o_server, o_addr, o_port, o_log|
+      o_server.mount_proc('/') do |req, res|
+        case req.request_method
+        when 'GET'
+          res['content-type'] = 'application/octet-stream'
+          if req.path == '/length'
+            res['content-length'] = (nr * rand_str.size).to_s
+          else
+            res.chunked = true
+          end
+          res.body = ->(socket) { nr.times { socket.write(rand_str) } }
+        when 'POST'
+          dig = Digest::MD5.new
+          req.body { |buf| dig.update(buf); buf.clear }
+          res['content-type'] = 'text/plain'
+          res['content-length'] = '32'
+          res.body = dig.hexdigest
+        end
+      end
+        http = Net::HTTP.new(o_addr, o_port)
+        IO.pipe do |rd, wr|
+          headers = {
+            'Content-Type' => 'application/octet-stream',
+            'Transfer-Encoding' => 'chunked',
+          }
+          post = Net::HTTP::Post.new('/', headers)
+          th = Thread.new { nr.times { wr.write(rand_str) }; wr.close }
+          post.body_stream = rd
+          http.request(post) do |res|
+            assert_equal 'text/plain', res['content-type']
+            assert_equal 32, res.content_length
+            assert_equal exp, res.body
+          end
+          assert_nil th.value
+        end
+
+      TestWEBrick.start_httpproxy do |p_server, p_addr, p_port, p_log|
+        http = Net::HTTP.new(o_addr, o_port, p_addr, p_port)
+        http.request_get('/length') do |res|
+          assert_equal(nr * rand_str.size, res.content_length)
+          dig = Digest::MD5.new
+          res.read_body { |buf| dig.update(buf); buf.clear }
+          assert_equal exp, dig.hexdigest
+        end
+        http.request_get('/') do |res|
+          assert_predicate res, :chunked?
+          dig = Digest::MD5.new
+          res.read_body { |buf| dig.update(buf); buf.clear }
+          assert_equal exp, dig.hexdigest
+        end
+
+        IO.pipe do |rd, wr|
+          headers = {
+            'Content-Type' => 'application/octet-stream',
+            'Content-Length' => (nr * rand_str.size).to_s,
+          }
+          post = Net::HTTP::Post.new('/', headers)
+          th = Thread.new { nr.times { wr.write(rand_str) }; wr.close }
+          post.body_stream = rd
+          http.request(post) do |res|
+            assert_equal 'text/plain', res['content-type']
+            assert_equal 32, res.content_length
+            assert_equal exp, res.body
+          end
+          assert_nil th.value
+        end
+
+        IO.pipe do |rd, wr|
+          headers = {
+            'Content-Type' => 'application/octet-stream',
+            'Transfer-Encoding' => 'chunked',
+          }
+          post = Net::HTTP::Post.new('/', headers)
+          th = Thread.new { nr.times { wr.write(rand_str) }; wr.close }
+          post.body_stream = rd
+          http.request(post) do |res|
+            assert_equal 'text/plain', res['content-type']
+            assert_equal 32, res.content_length
+            assert_equal exp, res.body
+          end
+          assert_nil th.value
+        end
+      end
+    end
+  end
+
   def make_certificate(key, cn)
     subject = OpenSSL::X509::Name.parse("/DC=org/DC=ruby-lang/CN=#{cn}")
     exts = [
diff --git a/test/webrick/test_httprequest.rb b/test/webrick/test_httprequest.rb
index 855ff9d4a7..6b99e34569 100644
--- a/test/webrick/test_httprequest.rb
+++ b/test/webrick/test_httprequest.rb
@@ -237,6 +237,7 @@ GET /
 
   def test_chunked
     crlf = "\x0d\x0a"
+    expect = File.read(__FILE__).freeze
     msg = <<-_end_of_message_
       POST /path HTTP/1.1
       Host: test.ruby-lang.org:8080
@@ -253,7 +254,14 @@ GET /
     msg << "0" << crlf
     req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP)
     req.parse(StringIO.new(msg))
-    assert_equal(File.read(__FILE__), req.body)
+    assert_equal(expect, req.body)
+
+    # chunked req.body_reader
+    req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP)
+    req.parse(StringIO.new(msg))
+    dst = StringIO.new
+    IO.copy_stream(req.body_reader, dst)
+    assert_equal(expect, dst.string)
   end
 
   def test_forwarded