[ruby/openssl] x509, ssl, pkcs7: try to parse as DER-encoding first

Methods that take both PEM-encoding and DER-encoding have not been
consistent in the order in which encoding to attempt to parse.

A DER-encoding may contain a valid PEM block ("\n-----BEGIN ..-----" to
"-----END ...-----") embedded within it. Also, the PEM-encoding parser
allows arbitrary data around the PEM block and silently skips it. As a
result, attempting to parse data in DER-encoding as PEM-encoding first
can incorrectly finds the embedded PEM block instead.

This commit ensures that DER encoding will always be attempted before
PEM encoding. OpenSSL::X509::Certificate is one of the updated classes.
With this, the following will always be true:

    # obj is an OpenSSL::X509::Certificate
    obj == OpenSSL::X509::Certificate.new(obj.to_der)
    obj == OpenSSL::X509::Certificate.new(obj.to_pem)

https://github.com/ruby/openssl/commit/b280eb1fd0

1 files changed, 24 insertions, 29 deletions

diff --git a/ext/openssl/ossl_ssl_session.c b/ext/openssl/ossl_ssl_session.c
index 5514087387..92eb1365fe 100644
--- a/ext/openssl/ossl_ssl_session.c
+++ b/ext/openssl/ossl_ssl_session.c
@@ -34,43 +34,38 @@ static VALUE ossl_ssl_session_alloc(VALUE klass)
  * Creates a new Session object from an instance of SSLSocket or DER/PEM encoded
  * String.
  */
-static VALUE ossl_ssl_session_initialize(VALUE self, VALUE arg1)
+static VALUE
+ossl_ssl_session_initialize(VALUE self, VALUE arg1)
 {
-	SSL_SESSION *ctx = NULL;
-
-	if (RDATA(self)->data)
-		ossl_raise(eSSLSession, "SSL Session already initialized");
-
-	if (rb_obj_is_instance_of(arg1, cSSLSocket)) {
-		SSL *ssl;
-
-		GetSSL(arg1, ssl);
-
-		if ((ctx = SSL_get1_session(ssl)) == NULL)
-			ossl_raise(eSSLSession, "no session available");
-	} else {
-		BIO *in = ossl_obj2bio(&arg1);
+    SSL_SESSION *ctx;
 
-		ctx = PEM_read_bio_SSL_SESSION(in, NULL, NULL, NULL);
+    if (RTYPEDDATA_DATA(self))
+        ossl_raise(eSSLSession, "SSL Session already initialized");
 
-		if (!ctx) {
-		        OSSL_BIO_reset(in);
-			ctx = d2i_SSL_SESSION_bio(in, NULL);
-		}
+    if (rb_obj_is_instance_of(arg1, cSSLSocket)) {
+        SSL *ssl;
 
-		BIO_free(in);
+        GetSSL(arg1, ssl);
 
-		if (!ctx)
-			ossl_raise(rb_eArgError, "unknown type");
-	}
+        if ((ctx = SSL_get1_session(ssl)) == NULL)
+            ossl_raise(eSSLSession, "no session available");
+    }
+    else {
+        BIO *in = ossl_obj2bio(&arg1);
 
-	/* should not happen */
-	if (ctx == NULL)
-		ossl_raise(eSSLSession, "ctx not set - internal error");
+        ctx = d2i_SSL_SESSION_bio(in, NULL);
+        if (!ctx) {
+            OSSL_BIO_reset(in);
+            ctx = PEM_read_bio_SSL_SESSION(in, NULL, NULL, NULL);
+        }
+        BIO_free(in);
+        if (!ctx)
+            ossl_raise(rb_eArgError, "unknown type");
+    }
 
-	RDATA(self)->data = ctx;
+    RTYPEDDATA_DATA(self) = ctx;
 
-	return self;
+    return self;
 }
 
 static VALUE