summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authornormal <normal@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>2018-01-05 20:48:55 +0000
committernormal <normal@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>2018-01-05 20:48:55 +0000
commita55abcc0ca6f628fc05304f81e5a044d65ab4a68 (patch)
tree9b0f3d3bb2f81c3797f666dd4cc25535a3f8fdc2
parentfb7ed3b32f82d807c5a6232d8c90ed56b4739739 (diff)
zlib: reduce garbage on gzip writes (deflate)
Zlib::GzipWriter generated large amounts of garbage from (struct zstream).input. Reuse the .input field when it is hidden, and recycle it when its lifetime is over. This change alone reduced memory usage of the writer from 90MB to 4.5MB. For the detached buffer of compressed data used by gzfile_write_raw, we can only clear the string (not recycle it) since user code may hold references to it (but the data would be clobbered, anyways). This reduced memory usage slightly by around 0.5MB (because it's smaller compressed data). Combined, these changes reduce the anonymous RSS memory of a dedicated writer process from over 90MB to under 4MB. before: # user system total real writer 7.823332 0.053333 7.876665 ( 7.881464) writer RssAnon: 92944 kB reader 6.969999 0.076666 7.046665 ( 7.906377) reader RssAnon: 109820 kB after: writer 7.359999 0.000000 7.359999 ( 7.360639) writer RssAnon: 4040 kB reader 6.346667 0.070000 6.416667 ( 7.387654) reader RssAnon: 98272 kB Script used: ------- require 'zlib' require 'benchmark' nr = 16384 * 2 def stats(pfx, bm) str = "#{bm}#{File.readlines("/proc/#$$/status").grep(/^RssAnon:/)[0]}" puts str.gsub!(/^/m, pfx) end rd, wr = IO.pipe pid = fork do buf = ((0..255).map(&:chr).join * 128).freeze rd.close gzip = Zlib::GzipWriter.new(wr) bm = Benchmark.measure do nr.times { gzip.write(buf) } gzip.close wr.close end stats('writer ', bm) end wr.close buf = '' gunzip = Zlib::GzipReader.new(rd) n = 0 bm = Benchmark.measure do begin gunzip.readpartial(16384, buf) n += buf.size rescue EOFError break end while true end stats('reader ', bm) Process.waitall ------- * ext/zlib/zlib.c (zstream_discard_input): reuse or recycle hidden input (zstream_reset_input): clear hidden input (zstream_run): detach input and recycle after use (gzfile_write_raw): clear buffer after write [ruby-core:84638] [Feature #14315] git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@61631 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
-rw-r--r--ext/zlib/zlib.c58
1 files changed, 47 insertions, 11 deletions
diff --git a/ext/zlib/zlib.c b/ext/zlib/zlib.c
index 7906e0f..ea3531d 100644
--- a/ext/zlib/zlib.c
+++ b/ext/zlib/zlib.c
@@ -845,19 +845,50 @@ zstream_append_input(struct zstream *z, const Bytef *src, long len)
static void
zstream_discard_input(struct zstream *z, long len)
{
- if (NIL_P(z->input) || RSTRING_LEN(z->input) <= len) {
- z->input = Qnil;
+ if (NIL_P(z->input)) {
}
- else {
- z->input = rb_str_substr(z->input, len,
- RSTRING_LEN(z->input) - len);
+ else if (RBASIC_CLASS(z->input) == 0) {
+ /* hidden, we created z->input and have complete control */
+ char *ptr;
+ long oldlen, newlen;
+
+ RSTRING_GETMEM(z->input, ptr, oldlen);
+ newlen = oldlen - len;
+ if (newlen > 0) {
+ memmove(ptr, ptr + len, newlen);
+ }
+ if (newlen < 0) {
+ newlen = 0;
+ }
+ rb_str_resize(z->input, newlen);
+ if (newlen == 0) {
+ rb_gc_force_recycle(z->input);
+ z->input = Qnil;
+ }
+ else {
+ rb_str_set_len(z->input, newlen);
+ }
+ }
+ else { /* do not mangle user-provided data */
+ if (RSTRING_LEN(z->input) <= len) {
+ z->input = Qnil;
+ }
+ else {
+ z->input = rb_str_substr(z->input, len,
+ RSTRING_LEN(z->input) - len);
+ }
}
}
static void
zstream_reset_input(struct zstream *z)
{
- z->input = Qnil;
+ if (!NIL_P(z->input) && RBASIC_CLASS(z->input) == 0) {
+ rb_str_resize(z->input, 0);
+ }
+ else {
+ z->input = Qnil;
+ }
}
static void
@@ -994,7 +1025,7 @@ zstream_run(struct zstream *z, Bytef *src, long len, int flush)
{
struct zstream_run_args args;
int err;
- VALUE guard = Qnil;
+ VALUE old_input = Qnil;
args.z = z;
args.flush = flush;
@@ -1008,12 +1039,13 @@ zstream_run(struct zstream *z, Bytef *src, long len, int flush)
}
else {
zstream_append_input(z, src, len);
- z->stream.next_in = (Bytef*)RSTRING_PTR(z->input);
- z->stream.avail_in = MAX_UINT(RSTRING_LEN(z->input));
/* keep reference to `z->input' so as not to be garbage collected
after zstream_reset_input() and prevent `z->stream.next_in'
from dangling. */
- guard = z->input;
+ old_input = zstream_detach_input(z);
+ rb_obj_hide(old_input); /* for GVL release and later recycle */
+ z->stream.next_in = (Bytef*)RSTRING_PTR(old_input);
+ z->stream.avail_in = MAX_UINT(RSTRING_LEN(old_input));
}
if (z->stream.avail_out == 0) {
@@ -1051,7 +1083,10 @@ loop:
if (z->stream.avail_in > 0) {
zstream_append_input(z, z->stream.next_in, z->stream.avail_in);
- RB_GC_GUARD(guard); /* prevent tail call to make guard effective */
+ }
+ if (!NIL_P(old_input)) {
+ rb_str_resize(old_input, 0);
+ rb_gc_force_recycle(old_input);
}
if (args.jump_state)
@@ -2330,6 +2365,7 @@ gzfile_write_raw(struct gzfile *gz)
str = zstream_detach_buffer(&gz->z);
OBJ_TAINT(str); /* for safe */
rb_funcall(gz->io, id_write, 1, str);
+ rb_str_resize(str, 0);
if ((gz->z.flags & GZFILE_FLAG_SYNC)
&& rb_respond_to(gz->io, id_flush))
rb_funcall(gz->io, id_flush, 0);