openssl: avoid NULL dereference in {DH,DSA,RSA}_size()

* ext/openssl/ossl_pkey_dh.c (ossl_dh_compute_key): Check that the DH
  has 'p' (the prime) before calling DH_size(). We can create a DH with
  no parameter but DH_size() does not check and dereferences NULL.
  [ruby-core:75720] [Bug #12428]

* ext/openssl/ossl_pkey_dsa.c (ossl_dsa_sign): Ditto. DSA_size() does
  not check dsa->q.

* ext/openssl/ossl_pkey_rsa.c (ossl_rsa_public_encrypt,
  ossl_rsa_public_decrypt, ossl_rsa_private_encrypt,
  ossl_rsa_private_decrypt): Ditto. RSA_size() does not check rsa->n.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@55175 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

1 files changed, 12 insertions, 6 deletions

diff --git a/ext/openssl/ossl_pkey_rsa.c b/ext/openssl/ossl_pkey_rsa.c
index 9cb50893c8..ea2af251cf 100644
--- a/ext/openssl/ossl_pkey_rsa.c
+++ b/ext/openssl/ossl_pkey_rsa.c
@@ -382,6 +382,8 @@ ossl_rsa_public_encrypt(int argc, VALUE *argv, VALUE self)
     VALUE str, buffer, padding;
 
     GetPKeyRSA(self, pkey);
+    if (!pkey->pkey.rsa->n)
+	ossl_raise(eRSAError, "incomplete RSA");
     rb_scan_args(argc, argv, "11", &buffer, &padding);
     pad = (argc == 1) ? RSA_PKCS1_PADDING : NUM2INT(padding);
     StringValue(buffer);
@@ -411,6 +413,8 @@ ossl_rsa_public_decrypt(int argc, VALUE *argv, VALUE self)
     VALUE str, buffer, padding;
 
     GetPKeyRSA(self, pkey);
+    if (!pkey->pkey.rsa->n)
+	ossl_raise(eRSAError, "incomplete RSA");
     rb_scan_args(argc, argv, "11", &buffer, &padding);
     pad = (argc == 1) ? RSA_PKCS1_PADDING : NUM2INT(padding);
     StringValue(buffer);
@@ -440,9 +444,10 @@ ossl_rsa_private_encrypt(int argc, VALUE *argv, VALUE self)
     VALUE str, buffer, padding;
 
     GetPKeyRSA(self, pkey);
-    if (!RSA_PRIVATE(self, pkey->pkey.rsa)) {
-	ossl_raise(eRSAError, "private key needed.");
-    }
+    if (!pkey->pkey.rsa->n)
+	ossl_raise(eRSAError, "incomplete RSA");
+    if (!RSA_PRIVATE(self, pkey->pkey.rsa))
+	ossl_raise(eRSAError, "private key needed");
     rb_scan_args(argc, argv, "11", &buffer, &padding);
     pad = (argc == 1) ? RSA_PKCS1_PADDING : NUM2INT(padding);
     StringValue(buffer);
@@ -472,9 +477,10 @@ ossl_rsa_private_decrypt(int argc, VALUE *argv, VALUE self)
     VALUE str, buffer, padding;
 
     GetPKeyRSA(self, pkey);
-    if (!RSA_PRIVATE(self, pkey->pkey.rsa)) {
-	ossl_raise(eRSAError, "private key needed.");
-    }
+    if (!pkey->pkey.rsa->n)
+	ossl_raise(eRSAError, "incomplete RSA");
+    if (!RSA_PRIVATE(self, pkey->pkey.rsa))
+	ossl_raise(eRSAError, "private key needed");
     rb_scan_args(argc, argv, "11", &buffer, &padding);
     pad = (argc == 1) ? RSA_PKCS1_PADDING : NUM2INT(padding);
     StringValue(buffer);