From 6ef0f272ebb2a4bd95471afcfe5224e72d2dad62 Mon Sep 17 00:00:00 2001
From: Kazuki Yamaguchi <k@rhe.jp>
Date: Sat, 20 Mar 2021 23:16:16 +0900
Subject: [ruby/openssl] pkey: use OSSL_DECODER to load encrypted PEM on
 OpenSSL 3.0

OpenSSL 3.0 has rewritten routines to load pkeys (PEM_read_bio_* and
d2i_* functions) around the newly introduced OSSL_DECODER API.

This comes with a slight behavior change. They now decrypt and parse
each encountered PEM block, then check the kind of the block. This used
to be the reverse: they checked the PEM header to see the kind, and then
decrypted the content. This means that the password callback may now be
called repeatedly.

Let's use the OSSL_DECODER API directly on OpenSSL 3.0 so that the
return value from the password callback will be reused automatically.

https://github.com/ruby/openssl/commit/a84ea531bb
---
 ext/openssl/ossl_pkey.c | 40 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)

(limited to 'ext/openssl/ossl_pkey.c')

diff --git a/ext/openssl/ossl_pkey.c b/ext/openssl/ossl_pkey.c
index f9f5162e41..b08168a581 100644
--- a/ext/openssl/ossl_pkey.c
+++ b/ext/openssl/ossl_pkey.c
@@ -79,6 +79,45 @@ ossl_pkey_new(EVP_PKEY *pkey)
     return obj;
 }
 
+#if OSSL_OPENSSL_PREREQ(3, 0, 0)
+# include <openssl/decoder.h>
+
+EVP_PKEY *
+ossl_pkey_read_generic(BIO *bio, VALUE pass)
+{
+    void *ppass = (void *)pass;
+    OSSL_DECODER_CTX *dctx;
+    EVP_PKEY *pkey = NULL;
+    int pos = 0, pos2;
+
+    dctx = OSSL_DECODER_CTX_new_for_pkey(&pkey, "DER", NULL, NULL, 0, NULL, NULL);
+    if (!dctx)
+        goto out;
+    if (OSSL_DECODER_CTX_set_pem_password_cb(dctx, ossl_pem_passwd_cb, ppass) != 1)
+        goto out;
+
+    /* First check DER */
+    if (OSSL_DECODER_from_bio(dctx, bio) == 1)
+        goto out;
+
+    /* Then check PEM; multiple OSSL_DECODER_from_bio() calls may be needed */
+    OSSL_BIO_reset(bio);
+    if (OSSL_DECODER_CTX_set_input_type(dctx, "PEM") != 1)
+        goto out;
+    while (OSSL_DECODER_from_bio(dctx, bio) != 1) {
+        if (BIO_eof(bio))
+            goto out;
+        pos2 = BIO_tell(bio);
+        if (pos2 < 0 || pos2 <= pos)
+            goto out;
+        pos = pos2;
+    }
+
+  out:
+    OSSL_DECODER_CTX_free(dctx);
+    return pkey;
+}
+#else
 EVP_PKEY *
 ossl_pkey_read_generic(BIO *bio, VALUE pass)
 {
@@ -107,6 +146,7 @@ ossl_pkey_read_generic(BIO *bio, VALUE pass)
   out:
     return pkey;
 }
+#endif
 
 /*
  *  call-seq:
-- 
cgit v1.2.3