openssl: avoid NULL dereference in {DH,DSA,RSA}_size()

* ext/openssl/ossl_pkey_dh.c (ossl_dh_compute_key): Check that the DH
  has 'p' (the prime) before calling DH_size(). We can create a DH with
  no parameter but DH_size() does not check and dereferences NULL.
  [ruby-core:75720] [Bug #12428]

* ext/openssl/ossl_pkey_dsa.c (ossl_dsa_sign): Ditto. DSA_size() does
  not check dsa->q.

* ext/openssl/ossl_pkey_rsa.c (ossl_rsa_public_encrypt,
  ossl_rsa_public_decrypt, ossl_rsa_private_encrypt,
  ossl_rsa_private_decrypt): Ditto. RSA_size() does not check rsa->n.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@55175 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

3 files changed, 18 insertions, 9 deletions

diff --git a/ext/openssl/ossl_pkey_dh.c b/ext/openssl/ossl_pkey_dh.c
index fd2b80497d..ac951a6e58 100644
--- a/ext/openssl/ossl_pkey_dh.c
+++ b/ext/openssl/ossl_pkey_dh.c
@@ -501,6 +501,8 @@ ossl_dh_compute_key(VALUE self, VALUE pub)
 
     GetPKeyDH(self, pkey);
     dh = pkey->pkey.dh;
+    if (!dh->p)
+	ossl_raise(eDHError, "incomplete DH");
     pub_key = GetBNPtr(pub);
     len = DH_size(dh);
     str = rb_str_new(0, len);
diff --git a/ext/openssl/ossl_pkey_dsa.c b/ext/openssl/ossl_pkey_dsa.c
index 2903380964..704fad6d4d 100644
--- a/ext/openssl/ossl_pkey_dsa.c
+++ b/ext/openssl/ossl_pkey_dsa.c
@@ -488,10 +488,11 @@ ossl_dsa_sign(VALUE self, VALUE data)
     VALUE str;
 
     GetPKeyDSA(self, pkey);
-    StringValue(data);
-    if (!DSA_PRIVATE(self, pkey->pkey.dsa)) {
+    if (!pkey->pkey.dsa->q)
+	ossl_raise(eDSAError, "incomplete DSA");
+    if (!DSA_PRIVATE(self, pkey->pkey.dsa))
 	ossl_raise(eDSAError, "Private DSA key needed!");
-    }
+    StringValue(data);
     str = rb_str_new(0, ossl_dsa_buf_size(pkey));
     if (!DSA_sign(0, (unsigned char *)RSTRING_PTR(data), RSTRING_LENINT(data),
 		  (unsigned char *)RSTRING_PTR(str),
diff --git a/ext/openssl/ossl_pkey_rsa.c b/ext/openssl/ossl_pkey_rsa.c
index 9cb50893c8..ea2af251cf 100644
--- a/ext/openssl/ossl_pkey_rsa.c
+++ b/ext/openssl/ossl_pkey_rsa.c
@@ -382,6 +382,8 @@ ossl_rsa_public_encrypt(int argc, VALUE *argv, VALUE self)
     VALUE str, buffer, padding;
 
     GetPKeyRSA(self, pkey);
+    if (!pkey->pkey.rsa->n)
+	ossl_raise(eRSAError, "incomplete RSA");
     rb_scan_args(argc, argv, "11", &buffer, &padding);
     pad = (argc == 1) ? RSA_PKCS1_PADDING : NUM2INT(padding);
     StringValue(buffer);
@@ -411,6 +413,8 @@ ossl_rsa_public_decrypt(int argc, VALUE *argv, VALUE self)
     VALUE str, buffer, padding;
 
     GetPKeyRSA(self, pkey);
+    if (!pkey->pkey.rsa->n)
+	ossl_raise(eRSAError, "incomplete RSA");
     rb_scan_args(argc, argv, "11", &buffer, &padding);
     pad = (argc == 1) ? RSA_PKCS1_PADDING : NUM2INT(padding);
     StringValue(buffer);
@@ -440,9 +444,10 @@ ossl_rsa_private_encrypt(int argc, VALUE *argv, VALUE self)
     VALUE str, buffer, padding;
 
     GetPKeyRSA(self, pkey);
-    if (!RSA_PRIVATE(self, pkey->pkey.rsa)) {
-	ossl_raise(eRSAError, "private key needed.");
-    }
+    if (!pkey->pkey.rsa->n)
+	ossl_raise(eRSAError, "incomplete RSA");
+    if (!RSA_PRIVATE(self, pkey->pkey.rsa))
+	ossl_raise(eRSAError, "private key needed");
     rb_scan_args(argc, argv, "11", &buffer, &padding);
     pad = (argc == 1) ? RSA_PKCS1_PADDING : NUM2INT(padding);
     StringValue(buffer);
@@ -472,9 +477,10 @@ ossl_rsa_private_decrypt(int argc, VALUE *argv, VALUE self)
     VALUE str, buffer, padding;
 
     GetPKeyRSA(self, pkey);
-    if (!RSA_PRIVATE(self, pkey->pkey.rsa)) {
-	ossl_raise(eRSAError, "private key needed.");
-    }
+    if (!pkey->pkey.rsa->n)
+	ossl_raise(eRSAError, "incomplete RSA");
+    if (!RSA_PRIVATE(self, pkey->pkey.rsa))
+	ossl_raise(eRSAError, "private key needed");
     rb_scan_args(argc, argv, "11", &buffer, &padding);
     pad = (argc == 1) ? RSA_PKCS1_PADDING : NUM2INT(padding);
     StringValue(buffer);