[ruby/json] Fix memory leak when exception is raised during JSON generation part 2

Commit https://github.com/ruby/json/commit/44df509dc2de fixed it for StandardError, but other exceptions and
jumps are also possible. Use rb_ensure() to release FBuffer instead of
rb_rescue().

A reproducer:

    o = Object.new
    def o.to_json(a) = throw :a
    a = ["make heap allocation"*100, o]

    10.times do
      100_000.times do
        catch(:a) { JSON(a) }
      end

      puts `ps -o rss= -p #{$$}`
    end

https://github.com/ruby/json/commit/9b7b648ecd

2 files changed, 6 insertions, 17 deletions

diff --git a/ext/json/fbuffer/fbuffer.h b/ext/json/fbuffer/fbuffer.h
index d5fd8ac6d7..7d57a87b14 100644
--- a/ext/json/fbuffer/fbuffer.h
+++ b/ext/json/fbuffer/fbuffer.h
@@ -283,13 +283,10 @@ static VALUE fbuffer_finalize(FBuffer *fb)
 {
     if (fb->io) {
         fbuffer_flush(fb);
-        fbuffer_free(fb);
         rb_io_flush(fb->io);
         return fb->io;
     } else {
-        VALUE result = rb_utf8_str_new(FBUFFER_PTR(fb), FBUFFER_LEN(fb));
-        fbuffer_free(fb);
-        return result;
+        return rb_utf8_str_new(FBUFFER_PTR(fb), FBUFFER_LEN(fb));
     }
 }
 
diff --git a/ext/json/generator/generator.c b/ext/json/generator/generator.c
index d04c8a9079..72155abe52 100644
--- a/ext/json/generator/generator.c
+++ b/ext/json/generator/generator.c
@@ -1444,16 +1444,14 @@ static VALUE generate_json_try(VALUE d)
 
     data->func(data->buffer, data, data->obj);
 
-    return Qnil;
+    return fbuffer_finalize(data->buffer);
 }
 
-static VALUE generate_json_rescue(VALUE d, VALUE exc)
+static VALUE generate_json_ensure(VALUE d)
 {
     struct generate_json_data *data = (struct generate_json_data *)d;
     fbuffer_free(data->buffer);
 
-    rb_exc_raise(exc);
-
     return Qundef;
 }
 
@@ -1474,9 +1472,7 @@ static VALUE cState_partial_generate(VALUE self, VALUE obj, generator_func func,
         .obj = obj,
         .func = func
     };
-    rb_rescue(generate_json_try, (VALUE)&data, generate_json_rescue, (VALUE)&data);
-
-    return fbuffer_finalize(&buffer);
+    return rb_ensure(generate_json_try, (VALUE)&data, generate_json_ensure, (VALUE)&data);
 }
 
 /* call-seq:
@@ -1522,9 +1518,7 @@ static VALUE cState_generate_new(int argc, VALUE *argv, VALUE self)
         .obj = obj,
         .func = generate_json
     };
-    rb_rescue(generate_json_try, (VALUE)&data, generate_json_rescue, (VALUE)&data);
-
-    return fbuffer_finalize(&buffer);
+    return rb_ensure(generate_json_try, (VALUE)&data, generate_json_ensure, (VALUE)&data);
 }
 
 static VALUE cState_initialize(int argc, VALUE *argv, VALUE self)
@@ -2030,9 +2024,7 @@ static VALUE cState_m_generate(VALUE klass, VALUE obj, VALUE opts, VALUE io)
         .obj = obj,
         .func = generate_json,
     };
-    rb_rescue(generate_json_try, (VALUE)&data, generate_json_rescue, (VALUE)&data);
-
-    return fbuffer_finalize(&buffer);
+    return rb_ensure(generate_json_try, (VALUE)&data, generate_json_ensure, (VALUE)&data);
 }
 
 /*