[ruby/openssl] pkey/dh, pkey/ec: use EVP_PKEY_check() family

Use EVP_PKEY_param_check() instead of DH_check() if available. Also,
use EVP_PKEY_public_check() instead of EC_KEY_check_key().

EVP_PKEY_*check() is part of the EVP API and is meant to replace those
low-level functions. They were added by OpenSSL 1.1.1. It is currently
not provided by LibreSSL.

https://github.com/ruby/openssl/commit/797e9f8e08

3 files changed, 45 insertions, 8 deletions

diff --git a/ext/openssl/extconf.rb b/ext/openssl/extconf.rb
index 5cb28f3092..8812b7de86 100644
--- a/ext/openssl/extconf.rb
+++ b/ext/openssl/extconf.rb
@@ -165,6 +165,9 @@ have_func("TS_RESP_CTX_set_time_cb")
 have_func("EVP_PBE_scrypt")
 have_func("SSL_CTX_set_post_handshake_auth")
 
+# added in 1.1.1
+have_func("EVP_PKEY_check")
+
 Logging::message "=== Checking done. ===\n"
 
 create_header
diff --git a/ext/openssl/ossl_pkey_dh.c b/ext/openssl/ossl_pkey_dh.c
index a512b209d3..ca782bbe59 100644
--- a/ext/openssl/ossl_pkey_dh.c
+++ b/ext/openssl/ossl_pkey_dh.c
@@ -273,19 +273,38 @@ ossl_dh_get_params(VALUE self)
  * Validates the Diffie-Hellman parameters associated with this instance.
  * It checks whether a safe prime and a suitable generator are used. If this
  * is not the case, +false+ is returned.
+ *
+ * See also the man page EVP_PKEY_param_check(3).
  */
 static VALUE
 ossl_dh_check_params(VALUE self)
 {
+    int ret;
+#ifdef HAVE_EVP_PKEY_CHECK
+    EVP_PKEY *pkey;
+    EVP_PKEY_CTX *pctx;
+
+    GetPKey(self, pkey);
+    pctx = EVP_PKEY_CTX_new(pkey, /* engine */NULL);
+    if (!pctx)
+        ossl_raise(eDHError, "EVP_PKEY_CTX_new");
+    ret = EVP_PKEY_param_check(pctx);
+    EVP_PKEY_CTX_free(pctx);
+#else
     DH *dh;
     int codes;
 
     GetDH(self, dh);
-    if (!DH_check(dh, &codes)) {
-	return Qfalse;
-    }
+    ret = DH_check(dh, &codes) == 1 && codes == 0;
+#endif
 
-    return codes == 0 ? Qtrue : Qfalse;
+    if (ret == 1)
+        return Qtrue;
+    else {
+        /* DH_check_ex() will put error entry on failure */
+        ossl_clear_error();
+        return Qfalse;
+    }
 }
 
 /*
diff --git a/ext/openssl/ossl_pkey_ec.c b/ext/openssl/ossl_pkey_ec.c
index ca2e8e508a..41f63289bd 100644
--- a/ext/openssl/ossl_pkey_ec.c
+++ b/ext/openssl/ossl_pkey_ec.c
@@ -438,20 +438,35 @@ static VALUE ossl_ec_key_generate_key(VALUE self)
 }
 
 /*
- *  call-seq:
- *     key.check_key   => true
+ * call-seq:
+ *    key.check_key   => true
  *
- *  Raises an exception if the key is invalid.
+ * Raises an exception if the key is invalid.
  *
- *  See the OpenSSL documentation for EC_KEY_check_key()
+ * See also the man page EVP_PKEY_public_check(3).
  */
 static VALUE ossl_ec_key_check_key(VALUE self)
 {
+#ifdef HAVE_EVP_PKEY_CHECK
+    EVP_PKEY *pkey;
+    EVP_PKEY_CTX *pctx;
+    int ret;
+
+    GetPKey(self, pkey);
+    pctx = EVP_PKEY_CTX_new(pkey, /* engine */NULL);
+    if (!pctx)
+        ossl_raise(eDHError, "EVP_PKEY_CTX_new");
+    ret = EVP_PKEY_public_check(pctx);
+    EVP_PKEY_CTX_free(pctx);
+    if (ret != 1)
+        ossl_raise(eECError, "EVP_PKEY_public_check");
+#else
     EC_KEY *ec;
 
     GetEC(self, ec);
     if (EC_KEY_check_key(ec) != 1)
 	ossl_raise(eECError, "EC_KEY_check_key");
+#endif
 
     return Qtrue;
 }