[ruby/openssl] pkey: implement PKey#sign_raw, #verify_raw, and #verify_recover

Add a variant of PKey#sign and #verify that do not hash the data
automatically.

Sometimes the caller has the hashed data only, but not the plaintext
to be signed. In that case, users would have to use the low-level API
such as RSA#private_encrypt or #public_decrypt directly.

OpenSSL 1.0.0 and later supports EVP_PKEY_sign() and EVP_PKEY_verify()
which provide the same functionality as part of the EVP API. This patch
adds wrappers for them.

https://github.com/ruby/openssl/commit/16cca4e0c4

3 files changed, 93 insertions, 31 deletions

diff --git a/test/openssl/test_pkey_dsa.rb b/test/openssl/test_pkey_dsa.rb
index 85bb6ec0ae..147e50176b 100644
--- a/test/openssl/test_pkey_dsa.rb
+++ b/test/openssl/test_pkey_dsa.rb
@@ -48,12 +48,31 @@ class OpenSSL::TestPKeyDSA < OpenSSL::PKeyTestCase
     assert_equal false, dsa512.verify("SHA256", signature1, data)
   end
 
-  def test_sys_sign_verify
-    key = Fixtures.pkey("dsa256")
+  def test_sign_verify_raw
+    key = Fixtures.pkey("dsa512")
     data = 'Sign me!'
     digest = OpenSSL::Digest.digest('SHA1', data)
+
+    invalid_sig = key.sign_raw(nil, digest.succ)
+    malformed_sig = "*" * invalid_sig.bytesize
+
+    # Sign by #syssign
     sig = key.syssign(digest)
-    assert(key.sysverify(digest, sig))
+    assert_equal true, key.sysverify(digest, sig)
+    assert_equal false, key.sysverify(digest, invalid_sig)
+    assert_raise(OpenSSL::PKey::DSAError) { key.sysverify(digest, malformed_sig) }
+    assert_equal true, key.verify_raw(nil, sig, digest)
+    assert_equal false, key.verify_raw(nil, invalid_sig, digest)
+    assert_raise(OpenSSL::PKey::PKeyError) { key.verify_raw(nil, malformed_sig, digest) }
+
+    # Sign by #sign_raw
+    sig = key.sign_raw(nil, digest)
+    assert_equal true, key.sysverify(digest, sig)
+    assert_equal false, key.sysverify(digest, invalid_sig)
+    assert_raise(OpenSSL::PKey::DSAError) { key.sysverify(digest, malformed_sig) }
+    assert_equal true, key.verify_raw(nil, sig, digest)
+    assert_equal false, key.verify_raw(nil, invalid_sig, digest)
+    assert_raise(OpenSSL::PKey::PKeyError) { key.verify_raw(nil, malformed_sig, digest) }
   end
 
   def test_DSAPrivateKey
diff --git a/test/openssl/test_pkey_ec.rb b/test/openssl/test_pkey_ec.rb
index 80ae9ffdf1..0a460bd536 100644
--- a/test/openssl/test_pkey_ec.rb
+++ b/test/openssl/test_pkey_ec.rb
@@ -109,13 +109,30 @@ class OpenSSL::TestEC < OpenSSL::PKeyTestCase
     assert_equal a.derive(b), a.dh_compute_key(b.public_key)
   end
 
-  def test_dsa_sign_verify
+  def test_sign_verify_raw
+    key = Fixtures.pkey("p256")
     data1 = "foo"
     data2 = "bar"
-    key = OpenSSL::PKey::EC.new("prime256v1").generate_key!
+
+    malformed_sig = "*" * 30
+
+    # Sign by #dsa_sign_asn1
     sig = key.dsa_sign_asn1(data1)
     assert_equal true, key.dsa_verify_asn1(data1, sig)
     assert_equal false, key.dsa_verify_asn1(data2, sig)
+    assert_raise(OpenSSL::PKey::ECError) { key.dsa_verify_asn1(data1, malformed_sig) }
+    assert_equal true, key.verify_raw(nil, sig, data1)
+    assert_equal false, key.verify_raw(nil, sig, data2)
+    assert_raise(OpenSSL::PKey::PKeyError) { key.verify_raw(nil, malformed_sig, data1) }
+
+    # Sign by #sign_raw
+    sig = key.sign_raw(nil, data1)
+    assert_equal true, key.dsa_verify_asn1(data1, sig)
+    assert_equal false, key.dsa_verify_asn1(data2, sig)
+    assert_raise(OpenSSL::PKey::ECError) { key.dsa_verify_asn1(data1, malformed_sig) }
+    assert_equal true, key.verify_raw(nil, sig, data1)
+    assert_equal false, key.verify_raw(nil, sig, data2)
+    assert_raise(OpenSSL::PKey::PKeyError) { key.verify_raw(nil, malformed_sig, data1) }
   end
 
   def test_dsa_sign_asn1_FIPS186_3
diff --git a/test/openssl/test_pkey_rsa.rb b/test/openssl/test_pkey_rsa.rb
index d6bfca3ac5..5e127f5407 100644
--- a/test/openssl/test_pkey_rsa.rb
+++ b/test/openssl/test_pkey_rsa.rb
@@ -13,32 +13,6 @@ class OpenSSL::TestPKeyRSA < OpenSSL::PKeyTestCase
     assert_raise(OpenSSL::PKey::RSAError){ key.private_decrypt("foo") }
   end
 
-  def test_padding
-    key = OpenSSL::PKey::RSA.new(512, 3)
-
-    # Need right size for raw mode
-    plain0 = "x" * (512/8)
-    cipher = key.private_encrypt(plain0, OpenSSL::PKey::RSA::NO_PADDING)
-    plain1 = key.public_decrypt(cipher, OpenSSL::PKey::RSA::NO_PADDING)
-    assert_equal(plain0, plain1)
-
-    # Need smaller size for pkcs1 mode
-    plain0 = "x" * (512/8 - 11)
-    cipher1 = key.private_encrypt(plain0, OpenSSL::PKey::RSA::PKCS1_PADDING)
-    plain1 = key.public_decrypt(cipher1, OpenSSL::PKey::RSA::PKCS1_PADDING)
-    assert_equal(plain0, plain1)
-
-    cipherdef = key.private_encrypt(plain0) # PKCS1_PADDING is default
-    plain1 = key.public_decrypt(cipherdef)
-    assert_equal(plain0, plain1)
-    assert_equal(cipher1, cipherdef)
-
-    # Failure cases
-    assert_raise(ArgumentError){ key.private_encrypt() }
-    assert_raise(ArgumentError){ key.private_encrypt("hi", 1, nil) }
-    assert_raise(OpenSSL::PKey::RSAError){ key.private_encrypt(plain0, 666) }
-  end
-
   def test_private
     # Generated by key size and public exponent
     key = OpenSSL::PKey::RSA.new(512, 3)
@@ -133,6 +107,58 @@ class OpenSSL::TestPKeyRSA < OpenSSL::PKeyTestCase
     assert_equal false, key.verify("SHA256", sig_pss, data)
   end
 
+  def test_sign_verify_raw
+    key = Fixtures.pkey("rsa-1")
+    data = "Sign me!"
+    hash = OpenSSL::Digest.digest("SHA1", data)
+    signature = key.sign_raw("SHA1", hash)
+    assert_equal true, key.verify_raw("SHA1", signature, hash)
+    assert_equal true, key.verify("SHA1", signature, data)
+
+    # Too long data
+    assert_raise(OpenSSL::PKey::PKeyError) {
+      key.sign_raw("SHA1", "x" * (key.n.num_bytes + 1))
+    }
+
+    # With options
+    pssopts = {
+      "rsa_padding_mode" => "pss",
+      "rsa_pss_saltlen" => 20,
+      "rsa_mgf1_md" => "SHA256"
+    }
+    sig_pss = key.sign_raw("SHA1", hash, pssopts)
+    assert_equal true, key.verify("SHA1", sig_pss, data, pssopts)
+    assert_equal true, key.verify_raw("SHA1", sig_pss, hash, pssopts)
+  end
+
+  def test_sign_verify_raw_legacy
+    key = Fixtures.pkey("rsa-1")
+    bits = key.n.num_bits
+
+    # Need right size for raw mode
+    plain0 = "x" * (bits/8)
+    cipher = key.private_encrypt(plain0, OpenSSL::PKey::RSA::NO_PADDING)
+    plain1 = key.public_decrypt(cipher, OpenSSL::PKey::RSA::NO_PADDING)
+    assert_equal(plain0, plain1)
+
+    # Need smaller size for pkcs1 mode
+    plain0 = "x" * (bits/8 - 11)
+    cipher1 = key.private_encrypt(plain0, OpenSSL::PKey::RSA::PKCS1_PADDING)
+    plain1 = key.public_decrypt(cipher1, OpenSSL::PKey::RSA::PKCS1_PADDING)
+    assert_equal(plain0, plain1)
+
+    cipherdef = key.private_encrypt(plain0) # PKCS1_PADDING is default
+    plain1 = key.public_decrypt(cipherdef)
+    assert_equal(plain0, plain1)
+    assert_equal(cipher1, cipherdef)
+
+    # Failure cases
+    assert_raise(ArgumentError){ key.private_encrypt() }
+    assert_raise(ArgumentError){ key.private_encrypt("hi", 1, nil) }
+    assert_raise(OpenSSL::PKey::RSAError){ key.private_encrypt(plain0, 666) }
+  end
+
+
   def test_verify_empty_rsa
     rsa = OpenSSL::PKey::RSA.new
     assert_raise(OpenSSL::PKey::PKeyError, "[Bug #12783]") {