From 3cf7d1b57e3622430065f6a6ce8cbd5548d3d894 Mon Sep 17 00:00:00 2001 From: drbrain Date: Tue, 10 Jul 2012 18:28:40 +0000 Subject: * ext/zlib/zlib.c: Added streaming support to inflate processing. This allows zlib streams to be processed without huge memory growth. [Feature #6612] * NEWS: ditto * ext/zlib/zlib.c (zstream_expand_buffer): Uses rb_yield when a block is given for streaming support. Refactored to use zstream_expand_buffer_into to remove duplicate code. * ext/zlib/zlib.c (zstream_expand_buffer_protect): Added wrapper function to pass jump state back through GVL-free section to allow zstream clean-up before terminating the ruby call. * ext/zlib/zlib.c (zstream_expand_buffer_without_gvl): Acquire GVL to yield processed chunk of output stream. * ext/zlib/zlib.c (zstream_detach_buffer): When a block is given, returns Qnil mid-stream and yields the output buffer at the end of the stream. * test/zlib/test_zlib.rb: Updated tests git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@36356 b2dd03c8-39d4-4d8f-98ff-823fe69b080e --- test/zlib/test_zlib.rb | 153 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) (limited to 'test/zlib') diff --git a/test/zlib/test_zlib.rb b/test/zlib/test_zlib.rb index 61e24e4d44..aa3486359c 100644 --- a/test/zlib/test_zlib.rb +++ b/test/zlib/test_zlib.rb @@ -39,6 +39,62 @@ if defined? Zlib assert_raise(Zlib::StreamError) { Zlib::Deflate.deflate("foo", 10000) } end + def test_deflate_chunked + original = '' + chunks = [] + r = Random.new 0 + + z = Zlib::Deflate.new + + 2.times do + input = r.bytes(20000) + original << input + z.deflate(input) do |chunk| + chunks << chunk + end + end + + assert_equal [16384, 16384], + chunks.map { |chunk| chunk.length } + + final = z.finish + + assert_equal 7253, final.length + + chunks << final + all = chunks.join + + inflated = Zlib.inflate all + + assert_equal original, inflated + end + + def test_deflate_chunked_break + chunks = [] + r = Random.new 0 + + z = Zlib::Deflate.new + + input = r.bytes(20000) + z.deflate(input) do |chunk| + chunks << chunk + break + end + + assert_equal [16384], chunks.map { |chunk| chunk.length } + + final = z.finish + + assert_equal 3632, final.length + + all = chunks.join + all << final + + original = Zlib.inflate all + + assert_equal input, original + end + def test_addstr z = Zlib::Deflate.new z << "foo" @@ -202,6 +258,38 @@ if defined? Zlib assert_equal "foofoofoo", out end + def test_finish_chunked + # zeros = Zlib::Deflate.deflate("0" * 100_000) + zeros = "x\234\355\3011\001\000\000\000\302\240J\353\237\316\032\036@" \ + "\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000" \ + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000" \ + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000" \ + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000" \ + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000" \ + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000" \ + "\000\000\000\000\000\000\000\257\006\351\247BH" + + chunks = [] + + z = Zlib::Inflate.new + + z.inflate(zeros) do |chunk| + chunks << chunk + break + end + + z.finish do |chunk| + chunks << chunk + end + + assert_equal [16384, 16384, 16384, 16384, 16384, 16384, 1696], + chunks.map { |chunk| chunk.size } + + assert chunks.all? { |chunk| + chunk =~ /\A0+\z/ + } + end + def test_inflate s = Zlib::Deflate.deflate("foo") z = Zlib::Inflate.new @@ -231,6 +319,58 @@ if defined? Zlib assert_equal "\0", inflated end + def test_inflate_chunked + # s = Zlib::Deflate.deflate("0" * 100_000) + zeros = "x\234\355\3011\001\000\000\000\302\240J\353\237\316\032\036@" \ + "\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000" \ + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000" \ + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000" \ + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000" \ + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000" \ + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000" \ + "\000\000\000\000\000\000\000\257\006\351\247BH" + + chunks = [] + + z = Zlib::Inflate.new + + z.inflate(zeros) do |chunk| + chunks << chunk + end + + assert_equal [16384, 16384, 16384, 16384, 16384, 16384, 1696], + chunks.map { |chunk| chunk.size } + + assert chunks.all? { |chunk| + chunk =~ /\A0+\z/ + } + end + + def test_inflate_chunked_break + # zeros = Zlib::Deflate.deflate("0" * 100_000) + zeros = "x\234\355\3011\001\000\000\000\302\240J\353\237\316\032\036@" \ + "\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000" \ + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000" \ + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000" \ + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000" \ + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000" \ + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000" \ + "\000\000\000\000\000\000\000\257\006\351\247BH" + + chunks = [] + + z = Zlib::Inflate.new + + z.inflate(zeros) do |chunk| + chunks << chunk + break + end + + out = z.inflate nil + + assert_equal 100_000 - chunks.first.length, out.length + end + def test_inflate_dictionary dictionary = "foo" @@ -896,5 +1036,18 @@ if defined? Zlib def test_deflate TestZlibDeflate.new(__name__).test_deflate end + + def test_deflate_stream + r = Random.new 0 + + deflated = '' + + Zlib.deflate(r.bytes(20000)) do |chunk| + deflated << chunk + end + + assert_equal 20016, deflated.length + end + end end -- cgit v1.2.3