openssl: clear OpenSSL error queue before return to Ruby

* ext/openssl/ossl_x509cert.c (ossl_x509_verify): X509_verify()
  family may put errors on 0 return (0 means verification failure).
  Clear OpenSSL error queue before return to Ruby. Since the queue is
  thread global, remaining errors in the queue can cause an unexpected
  error in the next OpenSSL operation.  [ruby-core:48284] [Bug #7215]

* ext/openssl/ossl_x509crl.c (ossl_x509crl_verify): ditto.

* ext/openssl/ossl_x509req.c (ossl_x509req_verify): ditto.

* ext/openssl/ossl_x509store.c (ossl_x509stctx_verify): ditto.

* ext/openssl/ossl_pkey_dh.c (dh_generate): clear the OpenSSL error
  queue before re-raising exception.

* ext/openssl/ossl_pkey_dsa.c (dsa_generate): ditto.

* ext/openssl/ossl_pkey_rsa.c (rsa_generate): ditto.

* ext/openssl/ossl_ssl.c (ossl_start_ssl): ditto.

* test/openssl: check that OpenSSL.errors is empty every time after
  running a test case.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@55051 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

8 files changed, 56 insertions, 31 deletions

diff --git a/ext/openssl/ossl_pkey_dh.c b/ext/openssl/ossl_pkey_dh.c
index 2f79bfb2f6..19c517fd3c 100644
--- a/ext/openssl/ossl_pkey_dh.c
+++ b/ext/openssl/ossl_pkey_dh.c
@@ -129,7 +129,11 @@ dh_generate(int size, int gen)
 
     if (!gen_arg.result) {
 	DH_free(dh);
-	if (cb_arg.state) rb_jump_tag(cb_arg.state);
+	if (cb_arg.state) {
+	    /* Clear OpenSSL error queue before re-raising. */
+	    ossl_clear_error();
+	    rb_jump_tag(cb_arg.state);
+	}
 	return 0;
     }
 #else
diff --git a/ext/openssl/ossl_pkey_dsa.c b/ext/openssl/ossl_pkey_dsa.c
index 2e42a0cef5..4c0c3f1bd7 100644
--- a/ext/openssl/ossl_pkey_dsa.c
+++ b/ext/openssl/ossl_pkey_dsa.c
@@ -135,7 +135,14 @@ dsa_generate(int size)
     }
     if (!gen_arg.result) {
 	DSA_free(dsa);
-	if (cb_arg.state) rb_jump_tag(cb_arg.state);
+	if (cb_arg.state) {
+	    /* Clear OpenSSL error queue before re-raising. By the way, the
+	     * documentation of DSA_generate_parameters_ex() says the error code
+	     * can be obtained by ERR_get_error(), but the default
+	     * implementation, dsa_builtin_paramgen() doesn't put any error... */
+	    ossl_clear_error();
+	    rb_jump_tag(cb_arg.state);
+	}
 	return 0;
     }
 #else
diff --git a/ext/openssl/ossl_pkey_rsa.c b/ext/openssl/ossl_pkey_rsa.c
index 20b993abb8..6ad9f3eda5 100644
--- a/ext/openssl/ossl_pkey_rsa.c
+++ b/ext/openssl/ossl_pkey_rsa.c
@@ -139,7 +139,11 @@ rsa_generate(int size, unsigned long exp)
     if (!gen_arg.result) {
 	BN_free(e);
 	RSA_free(rsa);
-	if (cb_arg.state) rb_jump_tag(cb_arg.state);
+	if (cb_arg.state) {
+	    /* must clear OpenSSL error stack */
+	    ossl_clear_error();
+	    rb_jump_tag(cb_arg.state);
+	}
 	return 0;
     }
 
diff --git a/ext/openssl/ossl_ssl.c b/ext/openssl/ossl_ssl.c
index 10797109fd..938e36f18a 100644
--- a/ext/openssl/ossl_ssl.c
+++ b/ext/openssl/ossl_ssl.c
@@ -1288,8 +1288,11 @@ ossl_start_ssl(VALUE self, int (*func)(), const char *funcname, VALUE opts)
 	ret = func(ssl);
 
         cb_state = rb_ivar_get(self, ID_callback_state);
-        if (!NIL_P(cb_state))
-            rb_jump_tag(NUM2INT(cb_state));
+        if (!NIL_P(cb_state)) {
+	    /* must cleanup OpenSSL error stack before re-raising */
+	    ossl_clear_error();
+	    rb_jump_tag(NUM2INT(cb_state));
+	}
 
 	if (ret > 0)
 	    break;
diff --git a/ext/openssl/ossl_x509cert.c b/ext/openssl/ossl_x509cert.c
index 4dafae17b9..226704efc6 100644
--- a/ext/openssl/ossl_x509cert.c
+++ b/ext/openssl/ossl_x509cert.c
@@ -591,18 +591,19 @@ ossl_x509_verify(VALUE self, VALUE key)
 {
     X509 *x509;
     EVP_PKEY *pkey;
-    int i;
 
     pkey = GetPKeyPtr(key); /* NO NEED TO DUP */
     GetX509(self, x509);
-    if ((i = X509_verify(x509, pkey)) < 0) {
-	ossl_raise(eX509CertError, NULL);
-    }
-    if (i > 0) {
+
+    switch (X509_verify(x509, pkey)) {
+      case 1:
 	return Qtrue;
+      case 0:
+	ossl_clear_error();
+	return Qfalse;
+      default:
+	ossl_raise(eX509CertError, NULL);
     }
-
-    return Qfalse;
 }
 
 /*
diff --git a/ext/openssl/ossl_x509crl.c b/ext/openssl/ossl_x509crl.c
index f64712efcd..a660cccebc 100644
--- a/ext/openssl/ossl_x509crl.c
+++ b/ext/openssl/ossl_x509crl.c
@@ -360,17 +360,17 @@ static VALUE
 ossl_x509crl_verify(VALUE self, VALUE key)
 {
     X509_CRL *crl;
-    int ret;
 
     GetX509CRL(self, crl);
-    if ((ret = X509_CRL_verify(crl, GetPKeyPtr(key))) < 0) {
-	ossl_raise(eX509CRLError, NULL);
-    }
-    if (ret == 1) {
+    switch (X509_CRL_verify(crl, GetPKeyPtr(key))) {
+      case 1:
 	return Qtrue;
+      case 0:
+	ossl_clear_error();
+	return Qfalse;
+      default:
+	ossl_raise(eX509CRLError, NULL);
     }
-
-    return Qfalse;
 }
 
 static VALUE
diff --git a/ext/openssl/ossl_x509req.c b/ext/openssl/ossl_x509req.c
index e5ce088a15..c1cdca5fbe 100644
--- a/ext/openssl/ossl_x509req.c
+++ b/ext/openssl/ossl_x509req.c
@@ -375,18 +375,18 @@ ossl_x509req_verify(VALUE self, VALUE key)
 {
     X509_REQ *req;
     EVP_PKEY *pkey;
-    int i;
 
     GetX509Req(self, req);
     pkey = GetPKeyPtr(key); /* NO NEED TO DUP */
-    if ((i = X509_REQ_verify(req, pkey)) < 0) {
-	ossl_raise(eX509ReqError, NULL);
-    }
-    if (i > 0) {
+    switch (X509_REQ_verify(req, pkey)) {
+      case 1:
 	return Qtrue;
+      case 0:
+	ossl_clear_error();
+	return Qfalse;
+      default:
+	ossl_raise(eX509ReqError, NULL);
     }
-
-    return Qfalse;
 }
 
 static VALUE
diff --git a/ext/openssl/ossl_x509store.c b/ext/openssl/ossl_x509store.c
index bb6fe14d87..aca25b150c 100644
--- a/ext/openssl/ossl_x509store.c
+++ b/ext/openssl/ossl_x509store.c
@@ -464,14 +464,20 @@ static VALUE
 ossl_x509stctx_verify(VALUE self)
 {
     X509_STORE_CTX *ctx;
-    int result;
 
     GetX509StCtx(self, ctx);
     X509_STORE_CTX_set_ex_data(ctx, ossl_verify_cb_idx,
-                               (void*)rb_iv_get(self, "@verify_callback"));
-    result = X509_verify_cert(ctx);
-
-    return result ? Qtrue : Qfalse;
+			       (void *)rb_iv_get(self, "@verify_callback"));
+
+    switch (X509_verify_cert(ctx)) {
+      case 1:
+	return Qtrue;
+      case 0:
+	ossl_clear_error();
+	return Qfalse;
+      default:
+	ossl_raise(eX509CertError, NULL);
+    }
 }
 
 static VALUE