[ruby/openssl] pkey: disallow {DH,DSA,EC,RSA}.new without arguments with OpenSSL 3.0

Raise ArgumentError if this is attempted when the extension is compiled
with OpenSSL 3.0 or later. The form will be fully removed when we drop
support for OpenSSL 1.1.1.

When OpenSSL::PKey::{DH,DSA,EC,RSA}.new is called without any arguments,
it sets up an empty corresponding low-level struct and wraps it in an
EVP_PKEY. This is useful when the user later fills the missing fields
using low-level setter methods such as OpenSSL::PKey::RSA#set_key.

Such setter methods are not compatible with OpenSSL 3.0 or later, where
EVP_PKEY is immutable once created. This means that the ability to
create an empty instance is useless.

https://github.com/ruby/openssl/commit/affd569f78

8 files changed, 71 insertions, 20 deletions

diff --git a/ext/openssl/ossl_pkey_dh.c b/ext/openssl/ossl_pkey_dh.c
index 118d29f04f..77082d5c34 100644
--- a/ext/openssl/ossl_pkey_dh.c
+++ b/ext/openssl/ossl_pkey_dh.c
@@ -43,6 +43,7 @@ static VALUE eDHError;
  * If called without arguments, an empty instance without any parameter or key
  * components is created. Use #set_pqg to manually set the parameters afterwards
  * (and optionally #set_key to set private and public key components).
+ * This form is not compatible with OpenSSL 3.0 or later.
  *
  * If a String is given, tries to parse it as a DER- or PEM- encoded parameters.
  * See also OpenSSL::PKey.read which can parse keys of any kinds.
@@ -58,14 +59,15 @@ static VALUE eDHError;
  *
  * Examples:
  *   # Creating an instance from scratch
- *   # Note that this is deprecated and will not work on OpenSSL 3.0 or later.
+ *   # Note that this is deprecated and will result in ArgumentError when
+ *   # using OpenSSL 3.0 or later.
  *   dh = OpenSSL::PKey::DH.new
  *   dh.set_pqg(bn_p, nil, bn_g)
  *
  *   # Generating a parameters and a key pair
  *   dh = OpenSSL::PKey::DH.new(2048) # An alias of OpenSSL::PKey::DH.generate(2048)
  *
- *   # Reading DH parameters
+ *   # Reading DH parameters from a PEM-encoded string
  *   dh_params = OpenSSL::PKey::DH.new(File.read('parameters.pem')) # loads parameters only
  *   dh = OpenSSL::PKey.generate_key(dh_params) # generates a key pair
  */
@@ -84,10 +86,15 @@ ossl_dh_initialize(int argc, VALUE *argv, VALUE self)
 
     /* The DH.new(size, generator) form is handled by lib/openssl/pkey.rb */
     if (rb_scan_args(argc, argv, "01", &arg) == 0) {
+#ifdef OSSL_HAVE_IMMUTABLE_PKEY
+        rb_raise(rb_eArgError, "OpenSSL::PKey::DH.new cannot be called " \
+                 "without arguments; pkeys are immutable with OpenSSL 3.0");
+#else
         dh = DH_new();
         if (!dh)
             ossl_raise(eDHError, "DH_new");
         goto legacy;
+#endif
     }
 
     arg = ossl_to_der_if_possible(arg);
diff --git a/ext/openssl/ossl_pkey_dsa.c b/ext/openssl/ossl_pkey_dsa.c
index 9f4f012cfe..bf92e1ceac 100644
--- a/ext/openssl/ossl_pkey_dsa.c
+++ b/ext/openssl/ossl_pkey_dsa.c
@@ -56,6 +56,7 @@ static VALUE eDSAError;
  *
  * If called without arguments, creates a new instance with no key components
  * set. They can be set individually by #set_pqg and #set_key.
+ * This form is not compatible with OpenSSL 3.0 or later.
  *
  * If called with a String, tries to parse as DER or PEM encoding of a \DSA key.
  * See also OpenSSL::PKey.read which can parse keys of any kinds.
@@ -96,10 +97,15 @@ ossl_dsa_initialize(int argc, VALUE *argv, VALUE self)
     /* The DSA.new(size, generator) form is handled by lib/openssl/pkey.rb */
     rb_scan_args(argc, argv, "02", &arg, &pass);
     if (argc == 0) {
+#ifdef OSSL_HAVE_IMMUTABLE_PKEY
+        rb_raise(rb_eArgError, "OpenSSL::PKey::DSA.new cannot be called " \
+                 "without arguments; pkeys are immutable with OpenSSL 3.0");
+#else
         dsa = DSA_new();
         if (!dsa)
             ossl_raise(eDSAError, "DSA_new");
         goto legacy;
+#endif
     }
 
     pass = ossl_pem_passwd_value(pass);
diff --git a/ext/openssl/ossl_pkey_ec.c b/ext/openssl/ossl_pkey_ec.c
index a2b68cc1d1..e3553c4418 100644
--- a/ext/openssl/ossl_pkey_ec.c
+++ b/ext/openssl/ossl_pkey_ec.c
@@ -147,9 +147,14 @@ static VALUE ossl_ec_key_initialize(int argc, VALUE *argv, VALUE self)
 
     rb_scan_args(argc, argv, "02", &arg, &pass);
     if (NIL_P(arg)) {
+#ifdef OSSL_HAVE_IMMUTABLE_PKEY
+        rb_raise(rb_eArgError, "OpenSSL::PKey::EC.new cannot be called " \
+                 "without arguments; pkeys are immutable with OpenSSL 3.0");
+#else
         if (!(ec = EC_KEY_new()))
             ossl_raise(eECError, "EC_KEY_new");
         goto legacy;
+#endif
     }
     else if (rb_obj_is_kind_of(arg, cEC_GROUP)) {
         ec = ec_key_new_from_group(arg);
diff --git a/ext/openssl/ossl_pkey_rsa.c b/ext/openssl/ossl_pkey_rsa.c
index 185b47cbfb..4f7862023a 100644
--- a/ext/openssl/ossl_pkey_rsa.c
+++ b/ext/openssl/ossl_pkey_rsa.c
@@ -59,6 +59,7 @@ static VALUE eRSAError;
  * If called without arguments, creates a new instance with no key components
  * set. They can be set individually by #set_key, #set_factors, and
  * #set_crt_params.
+ * This form is not compatible with OpenSSL 3.0 or later.
  *
  * If called with a String, tries to parse as DER or PEM encoding of an \RSA key.
  * Note that if _password_ is not specified, but the key is encrypted with a
@@ -89,10 +90,15 @@ ossl_rsa_initialize(int argc, VALUE *argv, VALUE self)
     /* The RSA.new(size, generator) form is handled by lib/openssl/pkey.rb */
     rb_scan_args(argc, argv, "02", &arg, &pass);
     if (argc == 0) {
+#ifdef OSSL_HAVE_IMMUTABLE_PKEY
+        rb_raise(rb_eArgError, "OpenSSL::PKey::RSA.new cannot be called " \
+                 "without arguments; pkeys are immutable with OpenSSL 3.0");
+#else
 	rsa = RSA_new();
         if (!rsa)
             ossl_raise(eRSAError, "RSA_new");
         goto legacy;
+#endif
     }
 
     pass = ossl_pem_passwd_value(pass);
diff --git a/test/openssl/test_pkey_dh.rb b/test/openssl/test_pkey_dh.rb
index f0c42866ea..ace9f8c0e0 100644
--- a/test/openssl/test_pkey_dh.rb
+++ b/test/openssl/test_pkey_dh.rb
@@ -7,9 +7,14 @@ class OpenSSL::TestPKeyDH < OpenSSL::PKeyTestCase
   NEW_KEYLEN = 2048
 
   def test_new_empty
-    dh = OpenSSL::PKey::DH.new
-    assert_equal nil, dh.p
-    assert_equal nil, dh.priv_key
+    # pkeys are immutable with OpenSSL >= 3.0
+    if openssl?(3, 0, 0)
+      assert_raise(ArgumentError) { OpenSSL::PKey::DH.new }
+    else
+      dh = OpenSSL::PKey::DH.new
+      assert_nil(dh.p)
+      assert_nil(dh.priv_key)
+    end
   end
 
   def test_new_generate
diff --git a/test/openssl/test_pkey_dsa.rb b/test/openssl/test_pkey_dsa.rb
index f3324b04af..0779483bd4 100644
--- a/test/openssl/test_pkey_dsa.rb
+++ b/test/openssl/test_pkey_dsa.rb
@@ -34,9 +34,14 @@ class OpenSSL::TestPKeyDSA < OpenSSL::PKeyTestCase
   end
 
   def test_new_empty
-    key = OpenSSL::PKey::DSA.new
-    assert_nil(key.p)
-    assert_raise(OpenSSL::PKey::PKeyError) { key.to_der }
+    # pkeys are immutable with OpenSSL >= 3.0
+    if openssl?(3, 0, 0)
+      assert_raise(ArgumentError) { OpenSSL::PKey::DSA.new }
+    else
+      key = OpenSSL::PKey::DSA.new
+      assert_nil(key.p)
+      assert_raise(OpenSSL::PKey::PKeyError) { key.to_der }
+    end
   end
 
   def test_generate
diff --git a/test/openssl/test_pkey_ec.rb b/test/openssl/test_pkey_ec.rb
index e569397c0a..58857cb038 100644
--- a/test/openssl/test_pkey_ec.rb
+++ b/test/openssl/test_pkey_ec.rb
@@ -4,19 +4,9 @@ require_relative 'utils'
 if defined?(OpenSSL)
 
 class OpenSSL::TestEC < OpenSSL::PKeyTestCase
-  def test_ec_key
+  def test_ec_key_new
     key1 = OpenSSL::PKey::EC.generate("prime256v1")
 
-    # PKey is immutable in OpenSSL >= 3.0; constructing an empty EC object is
-    # deprecated
-    if !openssl?(3, 0, 0)
-      key2 = OpenSSL::PKey::EC.new
-      key2.group = key1.group
-      key2.private_key = key1.private_key
-      key2.public_key = key1.public_key
-      assert_equal key1.to_der, key2.to_der
-    end
-
     key3 = OpenSSL::PKey::EC.new(key1)
     assert_equal key1.to_der, key3.to_der
 
@@ -35,6 +25,23 @@ class OpenSSL::TestEC < OpenSSL::PKeyTestCase
     end
   end
 
+  def test_ec_key_new_empty
+    # pkeys are immutable with OpenSSL >= 3.0; constructing an empty EC object is
+    # disallowed
+    if openssl?(3, 0, 0)
+      assert_raise(ArgumentError) { OpenSSL::PKey::EC.new }
+    else
+      key = OpenSSL::PKey::EC.new
+      assert_nil(key.group)
+
+      p256 = Fixtures.pkey("p256")
+      key.group = p256.group
+      key.private_key = p256.private_key
+      key.public_key = p256.public_key
+      assert_equal(p256.to_der, key.to_der)
+    end
+  end
+
   def test_builtin_curves
     builtin_curves = OpenSSL::PKey::EC.builtin_curves
     assert_not_empty builtin_curves
diff --git a/test/openssl/test_pkey_rsa.rb b/test/openssl/test_pkey_rsa.rb
index 850c16a029..6a8768d1ff 100644
--- a/test/openssl/test_pkey_rsa.rb
+++ b/test/openssl/test_pkey_rsa.rb
@@ -61,6 +61,16 @@ class OpenSSL::TestPKeyRSA < OpenSSL::PKeyTestCase
     assert_equal 3, key.e
   end
 
+  def test_new_empty
+    # pkeys are immutable with OpenSSL >= 3.0
+    if openssl?(3, 0, 0)
+      assert_raise(ArgumentError) { OpenSSL::PKey::RSA.new }
+    else
+      key = OpenSSL::PKey::RSA.new
+      assert_nil(key.n)
+    end
+  end
+
   def test_s_generate
     key1 = OpenSSL::PKey::RSA.generate(2048)
     assert_equal 2048, key1.n.num_bits
@@ -181,7 +191,7 @@ class OpenSSL::TestPKeyRSA < OpenSSL::PKeyTestCase
     assert_raise(OpenSSL::PKey::PKeyError, "[Bug #12783]") {
       rsa.verify("SHA1", "a", "b")
     }
-  end
+  end unless openssl?(3, 0, 0) # Empty RSA is not possible with OpenSSL >= 3.0
 
   def test_sign_verify_pss
     key = Fixtures.pkey("rsa2048")