From 1eb6d8aa63d7ada403adb0db12382d264dea5521 Mon Sep 17 00:00:00 2001
From: Kazuki Yamaguchi <k@rhe.jp>
Date: Wed, 17 Feb 2021 22:58:40 +0900
Subject: [ruby/openssl] bn: check -1 return from BIGNUM functions

Although the manpage says that BIGNUM functions return 0 on error,
OpenSSL versions before 1.0.2n and current LibreSSL versions may return
-1 instead.

Note that the implementation of OpenSSL::BN#mod_inverse is extracted
from BIGNUM_2c() macro as it didn't really share the same function
signature with others.

https://github.com/ruby/openssl/commit/9b59f34345
---
 ext/openssl/ossl_bn.c | 34 ++++++++++++++++++++++------------
 1 file changed, 22 insertions(+), 12 deletions(-)

(limited to 'ext/openssl')

diff --git a/ext/openssl/ossl_bn.c b/ext/openssl/ossl_bn.c
index d94b8e375c..bec37299f7 100644
--- a/ext/openssl/ossl_bn.c
+++ b/ext/openssl/ossl_bn.c
@@ -453,7 +453,7 @@ ossl_bn_is_negative(VALUE self)
 	if (!(result = BN_new())) {			\
 	    ossl_raise(eBNError, NULL);			\
 	}						\
-	if (!BN_##func(result, bn, ossl_bn_ctx)) {	\
+	if (BN_##func(result, bn, ossl_bn_ctx) <= 0) {	\
 	    BN_free(result);				\
 	    ossl_raise(eBNError, NULL);			\
 	}						\
@@ -479,7 +479,7 @@ BIGNUM_1c(sqr)
 	if (!(result = BN_new())) {			\
 	    ossl_raise(eBNError, NULL);			\
 	}						\
-	if (!BN_##func(result, bn1, bn2)) {		\
+	if (BN_##func(result, bn1, bn2) <= 0) {		\
 	    BN_free(result);				\
 	    ossl_raise(eBNError, NULL);			\
 	}						\
@@ -512,7 +512,7 @@ BIGNUM_2(sub)
 	if (!(result = BN_new())) {				\
 	    ossl_raise(eBNError, NULL);				\
 	}							\
-	if (!BN_##func(result, bn1, bn2, ossl_bn_ctx)) {	\
+	if (BN_##func(result, bn1, bn2, ossl_bn_ctx) <= 0) {	\
 	    BN_free(result);					\
 	    ossl_raise(eBNError, NULL);				\
 	}							\
@@ -556,11 +556,21 @@ BIGNUM_2c(gcd)
 BIGNUM_2c(mod_sqr)
 
 /*
- * Document-method: OpenSSL::BN#mod_inverse
  * call-seq:
- *   bn.mod_inverse(bn2) => aBN
+ *    bn.mod_inverse(bn2) => aBN
  */
-BIGNUM_2c(mod_inverse)
+static VALUE
+ossl_bn_mod_inverse(VALUE self, VALUE other)
+{
+    BIGNUM *bn1, *bn2 = GetBNPtr(other), *result;
+    VALUE obj;
+    GetBN(self, bn1);
+    obj = NewBN(rb_obj_class(self));
+    if (!(result = BN_mod_inverse(NULL, bn1, bn2, ossl_bn_ctx)))
+        ossl_raise(eBNError, "BN_mod_inverse");
+    SetBN(obj, result);
+    return obj;
+}
 
 /*
  * call-seq:
@@ -609,7 +619,7 @@ ossl_bn_div(VALUE self, VALUE other)
 	if (!(result = BN_new())) {				\
 	    ossl_raise(eBNError, NULL);				\
 	}							\
-	if (!BN_##func(result, bn1, bn2, bn3, ossl_bn_ctx)) {	\
+	if (BN_##func(result, bn1, bn2, bn3, ossl_bn_ctx) <= 0) { \
 	    BN_free(result);					\
 	    ossl_raise(eBNError, NULL);				\
 	}							\
@@ -651,7 +661,7 @@ BIGNUM_3c(mod_exp)
     {							\
 	BIGNUM *bn;					\
 	GetBN(self, bn);				\
-	if (!BN_##func(bn, NUM2INT(bit))) {		\
+	if (BN_##func(bn, NUM2INT(bit)) <= 0) {		\
 	    ossl_raise(eBNError, NULL);			\
 	}						\
 	return self;					\
@@ -711,7 +721,7 @@ ossl_bn_is_bit_set(VALUE self, VALUE bit)
 	if (!(result = BN_new())) {			\
 		ossl_raise(eBNError, NULL);		\
 	}						\
-	if (!BN_##func(result, bn, b)) {		\
+	if (BN_##func(result, bn, b) <= 0) {		\
 		BN_free(result);			\
 		ossl_raise(eBNError, NULL);		\
 	}						\
@@ -741,7 +751,7 @@ BIGNUM_SHIFT(rshift)
 	int b;						\
 	b = NUM2INT(bits);				\
 	GetBN(self, bn);				\
-	if (!BN_##func(bn, bn, b))			\
+	if (BN_##func(bn, bn, b) <= 0)			\
 		ossl_raise(eBNError, NULL);		\
 	return self;					\
     }
@@ -780,7 +790,7 @@ BIGNUM_SELF_SHIFT(rshift)
 	if (!(result = BN_new())) {				\
 	    ossl_raise(eBNError, NULL);				\
 	}							\
-	if (!BN_##func(result, b, top, bottom)) {		\
+	if (BN_##func(result, b, top, bottom) <= 0) {		\
 	    BN_free(result);					\
 	    ossl_raise(eBNError, NULL);				\
 	}							\
@@ -809,7 +819,7 @@ BIGNUM_RAND(pseudo_rand)
 	if (!(result = BN_new())) {				\
 	    ossl_raise(eBNError, NULL);				\
 	}							\
-	if (!BN_##func##_range(result, bn)) {			\
+	if (BN_##func##_range(result, bn) <= 0) {		\
 	    BN_free(result);					\
 	    ossl_raise(eBNError, NULL);				\
 	}							\
-- 
cgit v1.2.3