backport some changes from openssl gem v2.0.6 and v2.0.7.

	[Backport #13935]


git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/branches/ruby_2_3@62951 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

3 files changed, 62 insertions, 36 deletions

diff --git a/ext/openssl/ossl.c b/ext/openssl/ossl.c
index fb8f2c8171..0ee380507a 100644
--- a/ext/openssl/ossl.c
+++ b/ext/openssl/ossl.c
@@ -468,32 +468,46 @@ ossl_fips_mode_set(VALUE self, VALUE enabled)
  * Stores locks needed for OpenSSL thread safety
  */
 #include "ruby/thread_native.h"
-static rb_nativethread_lock_t *ossl_locks;
+struct CRYPTO_dynlock_value {
+    rb_nativethread_lock_t lock;
+    rb_nativethread_id_t owner;
+    size_t count;
+};
 
 static void
-ossl_lock_unlock(int mode, rb_nativethread_lock_t *lock)
+ossl_lock_init(struct CRYPTO_dynlock_value *l)
 {
-    if (mode & CRYPTO_LOCK) {
-	rb_nativethread_lock_lock(lock);
-    } else {
-	rb_nativethread_lock_unlock(lock);
-    }
+    rb_nativethread_lock_initialize(&l->lock);
+    l->count = 0;
 }
 
 static void
-ossl_lock_callback(int mode, int type, const char *file, int line)
+ossl_lock_unlock(int mode, struct CRYPTO_dynlock_value *l)
 {
-    ossl_lock_unlock(mode, &ossl_locks[type]);
+    if (mode & CRYPTO_LOCK) {
+	/* TODO: rb_nativethread_id_t is not necessarily compared with ==. */
+	rb_nativethread_id_t tid = rb_nativethread_self();
+	if (l->count && l->owner == tid) {
+	    l->count++;
+	    return;
+	}
+	rb_nativethread_lock_lock(&l->lock);
+	l->owner = tid;
+	l->count = 1;
+    } else {
+	if (!--l->count)
+	    rb_nativethread_lock_unlock(&l->lock);
+    }
 }
 
-struct CRYPTO_dynlock_value {
-    rb_nativethread_lock_t lock;
-};
-
 static struct CRYPTO_dynlock_value *
 ossl_dyn_create_callback(const char *file, int line)
 {
-    struct CRYPTO_dynlock_value *dynlock = (struct CRYPTO_dynlock_value *)OPENSSL_malloc((int)sizeof(struct CRYPTO_dynlock_value));
+    /* Do not use xmalloc() here, since it may raise NoMemoryError */
+    struct CRYPTO_dynlock_value *dynlock =
+	OPENSSL_malloc(sizeof(struct CRYPTO_dynlock_value));
+    if (dynlock)
+	ossl_lock_init(dynlock);
     rb_nativethread_lock_initialize(&dynlock->lock);
     return dynlock;
 }
@@ -501,7 +515,7 @@ ossl_dyn_create_callback(const char *file, int line)
 static void
 ossl_dyn_lock_callback(int mode, struct CRYPTO_dynlock_value *l, const char *file, int line)
 {
-    ossl_lock_unlock(mode, &l->lock);
+    ossl_lock_unlock(mode, l);
 }
 
 static void
@@ -525,21 +539,22 @@ static unsigned long ossl_thread_id(void)
 }
 #endif
 
+static struct CRYPTO_dynlock_value *ossl_locks;
+
+static void
+ossl_lock_callback(int mode, int type, const char *file, int line)
+{
+    ossl_lock_unlock(mode, &ossl_locks[type]);
+}
+
 static void Init_ossl_locks(void)
 {
     int i;
     int num_locks = CRYPTO_num_locks();
 
-    if ((unsigned)num_locks >= INT_MAX / (int)sizeof(VALUE)) {
-	rb_raise(rb_eRuntimeError, "CRYPTO_num_locks() is too big: %d", num_locks);
-    }
-    ossl_locks = (rb_nativethread_lock_t *) OPENSSL_malloc(num_locks * (int)sizeof(rb_nativethread_lock_t));
-    if (!ossl_locks) {
-	rb_raise(rb_eNoMemError, "CRYPTO_num_locks() is too big: %d", num_locks);
-    }
-    for (i = 0; i < num_locks; i++) {
-	rb_nativethread_lock_initialize(&ossl_locks[i]);
-    }
+    ossl_locks = ALLOC_N(struct CRYPTO_dynlock_value, num_locks);
+    for (i = 0; i < num_locks; i++)
+	ossl_lock_init(&ossl_locks[i]);
 
 #ifdef HAVE_CRYPTO_THREADID_PTR
     CRYPTO_THREADID_set_callback(ossl_threadid_func);
diff --git a/ext/openssl/ossl_cipher.c b/ext/openssl/ossl_cipher.c
index 24caba6e37..e167995d49 100644
--- a/ext/openssl/ossl_cipher.c
+++ b/ext/openssl/ossl_cipher.c
@@ -560,6 +560,8 @@ ossl_cipher_set_auth_data(VALUE self, VALUE data)
     in_len = RSTRING_LEN(data);
 
     GetCipher(self, ctx);
+    if (!(EVP_CIPHER_flags(EVP_CIPHER_CTX_cipher(ctx)) & EVP_CIPH_FLAG_AEAD_CIPHER))
+	ossl_raise(eCipherError, "AEAD not supported by this cipher");
 
     if (!ossl_cipher_update_long(ctx, NULL, &out_len, in, in_len))
         ossl_raise(eCipherError, "couldn't set additional authenticated data");
diff --git a/ext/openssl/ossl_ssl.c b/ext/openssl/ossl_ssl.c
index 7a0eb4ec90..cd35ee3771 100644
--- a/ext/openssl/ossl_ssl.c
+++ b/ext/openssl/ossl_ssl.c
@@ -427,6 +427,13 @@ ossl_sslctx_session_remove_cb(SSL_CTX *ctx, SSL_SESSION *sess)
     void *ptr;
     int state = 0;
 
+    /*
+     * This callback is also called for all sessions in the internal store
+     * when SSL_CTX_free() is called.
+     */
+    if (rb_during_gc())
+	return;
+
     OSSL_Debug("SSL SESSION remove callback entered");
 
     if ((ptr = SSL_CTX_get_ex_data(ctx, ossl_ssl_ex_ptr_idx)) == NULL)
@@ -1427,21 +1434,25 @@ ossl_ssl_read_internal(int argc, VALUE *argv, VALUE self, int nonblock)
     }
 
     ilen = NUM2INT(len);
-    if(NIL_P(str)) str = rb_str_new(0, ilen);
-    else{
-        StringValue(str);
-        rb_str_modify(str);
-        rb_str_resize(str, ilen);
+    if (NIL_P(str))
+	str = rb_str_new(0, ilen);
+    else {
+	StringValue(str);
+	if (RSTRING_LEN(str) >= ilen)
+	    rb_str_modify(str);
+	else
+	    rb_str_modify_expand(str, ilen - RSTRING_LEN(str));
     }
-    if(ilen == 0) return str;
+    OBJ_TAINT(str);
+    rb_str_set_len(str, 0);
+    if (ilen == 0)
+	return str;
 
     GetSSL(self, ssl);
     GetOpenFile(ossl_ssl_get_io(self), fptr);
     if (ssl) {
-	if(!nonblock && SSL_pending(ssl) <= 0)
-	    rb_thread_wait_fd(FPTR_TO_FD(fptr));
 	for (;;){
-	    nread = SSL_read(ssl, RSTRING_PTR(str), RSTRING_LENINT(str));
+	    nread = SSL_read(ssl, RSTRING_PTR(str), ilen);
 	    switch(ssl_get_error(ssl, nread)){
 	    case SSL_ERROR_NONE:
 		goto end;
@@ -1481,8 +1492,6 @@ ossl_ssl_read_internal(int argc, VALUE *argv, VALUE self, int nonblock)
 
   end:
     rb_str_set_len(str, nread);
-    OBJ_TAINT(str);
-
     return str;
 }