Fix memory leak in String#encode when StringValue raises

The following script leaks memory:

    10.times do
      100_000.times do
        "\ufffd".encode(Encoding::US_ASCII, fallback: proc { Object.new })
      rescue
      end

      puts `ps -o rss= -p #{$$}`
    end

Before:

    450244
    887748
    1325124
    1762756
    2200260
    2637508
    3075012
    3512516
    3950020
    4387524

After:

    12236
    12364
    12748
    13004
    13388
    13516
    13772
    13772
    13772
    13772

2 files changed, 37 insertions, 2 deletions

diff --git a/test/ruby/test_string.rb b/test/ruby/test_string.rb
index adbfe328cc..99ab85ddbe 100644
--- a/test/ruby/test_string.rb
+++ b/test/ruby/test_string.rb
@@ -3807,6 +3807,36 @@ CODE
     end
   end
 
+  def test_encode_fallback_not_string_memory_leak
+    {
+      "hash" => <<~RUBY,
+        fallback = Hash.new { Object.new }
+      RUBY
+      "proc" => <<~RUBY,
+        fallback = proc { Object.new }
+      RUBY
+      "method" => <<~RUBY,
+        def my_method(_str) = Object.new
+        fallback = method(:my_method)
+      RUBY
+      "aref" => <<~RUBY,
+        fallback = Object.new
+        def fallback.[](_str) = Object.new
+      RUBY
+    }.each do |type, code|
+      assert_no_memory_leak([], '', <<~RUBY, "fallback type is #{type}", rss: true)
+        class MyError < StandardError; end
+
+        #{code}
+
+        100_000.times do |i|
+          "\\ufffd".encode(Encoding::US_ASCII, fallback:)
+        rescue TypeError
+        end
+      RUBY
+    end
+  end
+
   private
 
   def assert_bytesplice_result(expected, s, *args)
diff --git a/transcode.c b/transcode.c
index 20b92b66f7..86e828c479 100644
--- a/transcode.c
+++ b/transcode.c
@@ -2360,7 +2360,13 @@ transcode_loop_fallback_try(VALUE a)
 {
     struct transcode_loop_fallback_args *args = (struct transcode_loop_fallback_args *)a;
 
-    return args->fallback_func(args->fallback, args->rep);
+    VALUE ret = args->fallback_func(args->fallback, args->rep);
+
+    if (!UNDEF_P(ret) && !NIL_P(ret)) {
+        StringValue(ret);
+    }
+
+    return ret;
 }
 
 static void
@@ -2428,7 +2434,6 @@ transcode_loop(const unsigned char **in_pos, unsigned char **out_pos,
         }
 
         if (!UNDEF_P(rep) && !NIL_P(rep)) {
-            StringValue(rep);
             ret = rb_econv_insert_output(ec, (const unsigned char *)RSTRING_PTR(rep),
                     RSTRING_LEN(rep), rb_enc_name(rb_enc_get(rep)));
             if ((int)ret == -1) {