LibCrypto+LibTLS: Replace RSA_PKCS1-EMSA implementation

This commit replaces the old implementation of `EMSA_PKCS1_V1_5` with
one backed by OpenSSL. In doing so, the `sign` and `verify` methods of
RSA have been modified to behave like expected and not just be
encryption and decryption.

I was not able to split this commit because the changes to `verify` and
`sign` break pretty much everything.

Libraries/LibCrypto/PK/PK.h

@@ -235,8 +235,8 @@ public:
     virtual ErrorOr<ByteBuffer> encrypt(ReadonlyBytes in) = 0;
     virtual ErrorOr<ByteBuffer> decrypt(ReadonlyBytes in) = 0;
 
-    virtual ErrorOr<ByteBuffer> verify(ReadonlyBytes in) = 0;
+    virtual ErrorOr<bool> verify(ReadonlyBytes message, ReadonlyBytes signature) = 0;
-    virtual ErrorOr<ByteBuffer> sign(ReadonlyBytes in) = 0;
+    virtual ErrorOr<ByteBuffer> sign(ReadonlyBytes message) = 0;
 
     virtual ByteString class_name() const = 0;
 

Libraries/LibCrypto/PK/RSA.cpp

@@ -263,24 +263,39 @@ ErrorOr<ByteBuffer> RSA::decrypt(ReadonlyBytes in)
     return out.slice(0, out_size);
 }
 
-ErrorOr<ByteBuffer> RSA::sign(ReadonlyBytes in)
+ErrorOr<ByteBuffer> RSA::sign(ReadonlyBytes message)
 {
-    auto in_integer = UnsignedBigInteger::import_data(in.data(), in.size());
+    auto key = TRY(private_key_to_openssl_pkey(m_private_key));
-    auto exp = NumberTheory::ModularPower(in_integer, m_private_key.private_exponent(), m_private_key.modulus());
 
-    auto out = TRY(ByteBuffer::create_uninitialized(exp.byte_length()));
+    auto ctx = TRY(OpenSSL_PKEY_CTX::wrap(EVP_PKEY_CTX_new_from_pkey(nullptr, key.ptr(), nullptr)));
-    auto size = exp.export_data(out);
+
-    return out.slice(out.size() - size, size);
+    OPENSSL_TRY(EVP_PKEY_sign_init(ctx.ptr()));
+    TRY(configure(ctx));
+
+    size_t signature_size = 0;
+    OPENSSL_TRY(EVP_PKEY_sign(ctx.ptr(), nullptr, &signature_size, message.data(), message.size()));
+
+    auto signature = TRY(ByteBuffer::create_uninitialized(signature_size));
+    OPENSSL_TRY(EVP_PKEY_sign(ctx.ptr(), signature.data(), &signature_size, message.data(), message.size()));
+    return signature.slice(0, signature_size);
 }
 
-ErrorOr<ByteBuffer> RSA::verify(ReadonlyBytes in)
+ErrorOr<bool> RSA::verify(ReadonlyBytes message, ReadonlyBytes signature)
 {
-    auto in_integer = UnsignedBigInteger::import_data(in.data(), in.size());
+    auto key = TRY(public_key_to_openssl_pkey(m_public_key));
-    auto exp = NumberTheory::ModularPower(in_integer, m_public_key.public_exponent(), m_public_key.modulus());
 
-    auto out = TRY(ByteBuffer::create_uninitialized(exp.byte_length()));
+    auto ctx = TRY(OpenSSL_PKEY_CTX::wrap(EVP_PKEY_CTX_new_from_pkey(nullptr, key.ptr(), nullptr)));
-    auto size = exp.export_data(out);
+
-    return out.slice(out.size() - size, size);
+    OPENSSL_TRY(EVP_PKEY_verify_init(ctx.ptr()));
+    TRY(configure(ctx));
+
+    auto ret = EVP_PKEY_verify(ctx.ptr(), signature.data(), signature.size(), message.data(), message.size());
+    if (ret == 1)
+        return true;
+    if (ret == 0)
+        return false;
+    OPENSSL_TRY(ret);
+    VERIFY_NOT_REACHED();
 }
 
 void RSA::import_private_key(ReadonlyBytes bytes, bool pem)
@@ -405,7 +420,7 @@ ErrorOr<ByteBuffer> RSA_PKCS1_EME::decrypt(ReadonlyBytes in)
     if (offset - 3 < 8)
         return Error::from_string_literal("PS too small");
 
-    return out.slice(offset, out.size() - offset);;
+    return out.slice(offset, out.size() - offset);
 }
 
 ErrorOr<ByteBuffer> RSA_PKCS1_EME::sign(ReadonlyBytes)
@@ -413,8 +428,86 @@ ErrorOr<ByteBuffer> RSA_PKCS1_EME::sign(ReadonlyBytes)
     return Error::from_string_literal("FIXME: RSA_PKCS_EME::sign");
 }
 
-ErrorOr<ByteBuffer> RSA_PKCS1_EME::verify(ReadonlyBytes)
+ErrorOr<bool> RSA_PKCS1_EME::verify(ReadonlyBytes, ReadonlyBytes)
 {
     return Error::from_string_literal("FIXME: RSA_PKCS_EME::verify");
 }
+
+ErrorOr<EVP_MD const*> hash_kind_to_hash_type(Hash::HashKind hash_kind)
+{
+    switch (hash_kind) {
+    case Hash::HashKind::None:
+        return nullptr;
+    case Hash::HashKind::BLAKE2b:
+        return EVP_blake2b512();
+    case Hash::HashKind::MD5:
+        return EVP_md5();
+    case Hash::HashKind::SHA1:
+        return EVP_sha1();
+    case Hash::HashKind::SHA256:
+        return EVP_sha256();
+    case Hash::HashKind::SHA384:
+        return EVP_sha384();
+    case Hash::HashKind::SHA512:
+        return EVP_sha512();
+    default:
+        return Error::from_string_literal("Unsupported hash kind");
+    }
+}
+
+ErrorOr<bool> RSA_PKCS1_EMSA::verify(ReadonlyBytes message, ReadonlyBytes signature)
+{
+    auto key = TRY(public_key_to_openssl_pkey(m_public_key));
+    auto const* hash_type = TRY(hash_kind_to_hash_type(m_hash_kind));
+
+    auto ctx = TRY(OpenSSL_MD_CTX::create());
+
+    auto key_ctx = TRY(OpenSSL_PKEY_CTX::wrap(EVP_PKEY_CTX_new(key.ptr(), nullptr)));
+    EVP_MD_CTX_set_pkey_ctx(ctx.ptr(), key_ctx.ptr());
+
+    OPENSSL_TRY(EVP_DigestVerifyInit(ctx.ptr(), nullptr, hash_type, nullptr, key.ptr()));
+    TRY(configure(key_ctx));
+
+    auto res = EVP_DigestVerify(ctx.ptr(), signature.data(), signature.size(), message.data(), message.size());
+    if (res == 1)
+        return true;
+    if (res == 0)
+        return false;
+    OPENSSL_TRY(res);
+    VERIFY_NOT_REACHED();
+}
+
+ErrorOr<ByteBuffer> RSA_PKCS1_EMSA::sign(ReadonlyBytes message)
+{
+    auto key = TRY(private_key_to_openssl_pkey(m_private_key));
+    auto const* hash_type = TRY(hash_kind_to_hash_type(m_hash_kind));
+
+    auto ctx = TRY(OpenSSL_MD_CTX::create());
+
+    auto key_ctx = TRY(OpenSSL_PKEY_CTX::wrap(EVP_PKEY_CTX_new(key.ptr(), nullptr)));
+    EVP_MD_CTX_set_pkey_ctx(ctx.ptr(), key_ctx.ptr());
+
+    OPENSSL_TRY(EVP_DigestSignInit(ctx.ptr(), nullptr, hash_type, nullptr, key.ptr()));
+    TRY(configure(key_ctx));
+
+    size_t signature_size = 0;
+    OPENSSL_TRY(EVP_DigestSign(ctx.ptr(), nullptr, &signature_size, message.data(), message.size()));
+
+    auto signature = TRY(ByteBuffer::create_uninitialized(signature_size));
+    OPENSSL_TRY(EVP_DigestSign(ctx.ptr(), signature.data(), &signature_size, message.data(), message.size()));
+    return signature.slice(0, signature_size);
+}
+
+ErrorOr<void> RSA_PKCS1_EME::configure(OpenSSL_PKEY_CTX& ctx)
+{
+    OPENSSL_TRY(EVP_PKEY_CTX_set_rsa_padding(ctx.ptr(), RSA_PKCS1_PADDING));
+    return {};
+}
+
+ErrorOr<void> RSA_PKCS1_EMSA::configure(OpenSSL_PKEY_CTX& ctx)
+{
+    OPENSSL_TRY(EVP_PKEY_CTX_set_rsa_padding(ctx.ptr(), RSA_PKCS1_PADDING));
+    return {};
+}
+
 }

Libraries/LibCrypto/PK/RSA.h

@@ -9,6 +9,7 @@
 
 #include <LibCrypto/ASN1/DER.h>
 #include <LibCrypto/BigInt/UnsignedBigInteger.h>
+#include <LibCrypto/Hash/HashManager.h>
 #include <LibCrypto/NumberTheory/ModularFunctions.h>
 #include <LibCrypto/OpenSSL.h>
 #include <LibCrypto/PK/PK.h>
@@ -199,8 +200,8 @@ public:
     virtual ErrorOr<ByteBuffer> encrypt(ReadonlyBytes in) override;
     virtual ErrorOr<ByteBuffer> decrypt(ReadonlyBytes in) override;
 
-    virtual ErrorOr<ByteBuffer> verify(ReadonlyBytes in) override;
+    virtual ErrorOr<ByteBuffer> sign(ReadonlyBytes message) override;
-    virtual ErrorOr<ByteBuffer> sign(ReadonlyBytes in) override;
+    virtual ErrorOr<bool> verify(ReadonlyBytes message, ReadonlyBytes signature) override;
 
     virtual ByteString class_name() const override
     {
@@ -228,31 +229,119 @@ protected:
     static ErrorOr<OpenSSL_PKEY> private_key_to_openssl_pkey(PrivateKeyType const& private_key);
 };
 
-class RSA_PKCS1_EME : public RSA {
+ErrorOr<EVP_MD const*> hash_kind_to_hash_type(Hash::HashKind hash_kind);
+
+class RSA_EME : public RSA {
+public:
+    template<typename... Args>
+    RSA_EME(Hash::HashKind hash_kind, Args... args)
+        : RSA(args...)
+        , m_hash_kind(hash_kind)
+    {
+    }
+
+    ~RSA_EME() = default;
+
+    virtual ErrorOr<ByteBuffer> sign(ReadonlyBytes) override
+    {
+        return Error::from_string_literal("Signing is not supported");
+    }
+    virtual ErrorOr<bool> verify(ReadonlyBytes, ReadonlyBytes) override
+    {
+        return Error::from_string_literal("Verifying is not supported");
+    }
+
+protected:
+    Hash::HashKind m_hash_kind { Hash::HashKind::Unknown };
+};
+
+class RSA_EMSA : public RSA {
+public:
+    template<typename... Args>
+    RSA_EMSA(Hash::HashKind hash_kind, Args... args)
+        : RSA(args...)
+        , m_hash_kind(hash_kind)
+    {
+    }
+
+    ~RSA_EMSA() = default;
+
+    virtual ErrorOr<ByteBuffer> encrypt(ReadonlyBytes) override
+    {
+        return Error::from_string_literal("Encrypting is not supported");
+    }
+    virtual ErrorOr<ByteBuffer> decrypt(ReadonlyBytes) override
+    {
+        return Error::from_string_literal("Decrypting is not supported");
+    }
+
+    virtual ErrorOr<bool> verify(ReadonlyBytes message, ReadonlyBytes signature) override;
+    virtual ErrorOr<ByteBuffer> sign(ReadonlyBytes message) override;
+
+protected:
+    Hash::HashKind m_hash_kind { Hash::HashKind::Unknown };
+};
+
+class RSA_PKCS1_EME : public RSA_EME {
 public:
-    // forward all constructions to RSA
     template<typename... Args>
     RSA_PKCS1_EME(Args... args)
-        : RSA(args...)
+        : RSA_EME(Hash::HashKind::None, args...)
     {
     }
 
     ~RSA_PKCS1_EME() = default;
 
-    virtual ErrorOr<ByteBuffer> encrypt(ReadonlyBytes in) override;
-    virtual ErrorOr<ByteBuffer> decrypt(ReadonlyBytes in) override;
-
-    virtual ErrorOr<ByteBuffer> verify(ReadonlyBytes in) override;
-    virtual ErrorOr<ByteBuffer> sign(ReadonlyBytes in) override;
-
     virtual ByteString class_name() const override
     {
         return "RSA_PKCS1-EME";
     }
 
-    virtual size_t output_size() const override
+protected:
-    {
+    ErrorOr<void> configure(OpenSSL_PKEY_CTX& ctx) override;
-        return m_public_key.length();
-    }
 };
+
+class RSA_PKCS1_EMSA : public RSA_EMSA {
+public:
+    template<typename... Args>
+    RSA_PKCS1_EMSA(Hash::HashKind hash_kind, Args... args)
+        : RSA_EMSA(hash_kind, args...)
+    {
+    }
+
+    ~RSA_PKCS1_EMSA() = default;
+
+    virtual ByteString class_name() const override
+    {
+        return "RSA_PKCS1-EMSA";
+    }
+
+protected:
+    ErrorOr<void> configure(OpenSSL_PKEY_CTX& ctx) override;
+};
+
+class RSA_OAEP_EME : public RSA_EME {
+public:
+    template<typename... Args>
+    RSA_OAEP_EME(Hash::HashKind hash_kind, Args... args)
+        : RSA_EME(hash_kind, args...)
+    {
+    }
+
+    ~RSA_OAEP_EME() = default;
+
+    virtual ByteString class_name() const override
+    {
+        return "RSA_OAEP-EME";
+    }
+
+    void set_label(ReadonlyBytes label) { m_label = label; }
+
+protected:
+    ErrorOr<void> configure(OpenSSL_PKEY_CTX& ctx) override;
+
+private:
+    Optional<ReadonlyBytes> m_label {};
+};
+
 }

Libraries/LibTLS/HandshakeServer.cpp

@@ -16,7 +16,6 @@
 #include <LibCrypto/Curves/SECPxxxr1.h>
 #include <LibCrypto/Curves/X25519.h>
 #include <LibCrypto/Curves/X448.h>
-#include <LibCrypto/PK/Code/EMSA_PKCS1_V1_5.h>
 #include <LibTLS/TLSv12.h>
 
 namespace TLS {
@@ -377,11 +376,6 @@ ssize_t TLSv12::verify_rsa_server_key_exchange(ReadonlyBytes server_key_info_buf
         return (i8)Error::NotSafe;
     }
     // RFC5246 section 7.4.2: The sender's certificate MUST come first in the list.
-    auto certificate_public_key = m_context.certificates.first().public_key;
-    Crypto::PK::RSAPrivateKey dummy_private_key;
-    auto rsa = Crypto::PK::RSA(certificate_public_key.rsa, dummy_private_key);
-
-    auto signature_verify = MUST(rsa.verify(signature));
 
     auto message_result = ByteBuffer::create_uninitialized(64 + server_key_info_buffer.size());
     if (message_result.is_error()) {
@@ -412,10 +406,11 @@ ssize_t TLSv12::verify_rsa_server_key_exchange(ReadonlyBytes server_key_info_buf
         return (i8)Error::NotUnderstood;
     }
 
-    auto pkcs1 = Crypto::PK::EMSA_PKCS1_V1_5<Crypto::Hash::Manager>(hash_kind);
+    auto certificate_public_key = m_context.certificates.first().public_key;
-    auto verification = pkcs1.verify(message, signature_verify, signature_length * 8);
+    auto rsa = Crypto::PK::RSA_PKCS1_EMSA(hash_kind, certificate_public_key.rsa);
+    auto verification = MUST(rsa.verify(message, signature));
 
-    if (verification == Crypto::VerificationConsistency::Inconsistent) {
+    if (!verification) {
         dbgln("verify_rsa_server_key_exchange failed: Verification of signature inconsistent");
         return (i8)Error::NotSafe;
     }

Libraries/LibTLS/TLSv12.cpp

@@ -18,7 +18,6 @@
 #include <LibCrypto/Certificate/Certificate.h>
 #include <LibCrypto/Curves/Ed25519.h>
 #include <LibCrypto/Curves/SECPxxxr1.h>
-#include <LibCrypto/PK/Code/EMSA_PKCS1_V1_5.h>
 #include <LibFileSystem/FileSystem.h>
 #include <LibTLS/TLSv12.h>
 #include <errno.h>
@@ -342,15 +341,8 @@ bool Context::verify_certificate_pair(Certificate const& subject, Certificate co
     }
 
     if (is_rsa) {
-        Crypto::PK::RSAPrivateKey dummy_private_key;
+        auto rsa = Crypto::PK::RSA_PKCS1_EMSA(kind, issuer.public_key.rsa);
-        Crypto::PK::RSAPublicKey public_key_copy { issuer.public_key.rsa };
+        return MUST(rsa.verify(subject.tbs_asn1, subject.signature_value));
-        auto rsa = Crypto::PK::RSA(public_key_copy, dummy_private_key);
-        auto verification_bytes = MUST(rsa.verify(subject.signature_value));
-
-        ReadonlyBytes message = subject.tbs_asn1.bytes();
-        auto pkcs1 = Crypto::PK::EMSA_PKCS1_V1_5<Crypto::Hash::Manager>(kind);
-        auto verification = pkcs1.verify(message, verification_bytes, subject.signature_value.size() * 8);
-        return verification == Crypto::VerificationConsistency::Consistent;
     }
 
     // ECDSA hash verification: hash, then check signature against the specific curve

Tests/LibCrypto/TestRSA.cpp

@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2021, Peter Bocan  <me@pbocan.net>
+ * Copyright (c) 2025, Altomani Gianluca <altomanigianluca@gmail.com>
  *
  * SPDX-License-Identifier: BSD-2-Clause
  */
@@ -160,3 +161,19 @@ TEST_CASE(test_RSA_encrypt_decrypt)
 
     EXPECT(memcmp(dec.data(), "WellHelloFriendsWellHelloFriendsWellHelloFriendsWellHelloFriends", 64) == 0);
 }
+
+TEST_CASE(test_RSA_sign_verify)
+{
+    auto keypair = TRY_OR_FAIL(Crypto::PK::RSA::generate_key_pair(1024));
+    Crypto::PK::RSA rsa(keypair);
+
+    ByteBuffer msg_buffer = {};
+    msg_buffer.resize(rsa.output_size());
+
+    auto msg = msg_buffer.bytes();
+    msg.overwrite(0, "WellHelloFriendsWellHelloFriendsWellHelloFriendsWellHelloFriends", 64);
+
+    auto sig = TRY_OR_FAIL(rsa.sign(msg));
+    auto ok = TRY_OR_FAIL(rsa.verify(msg, sig));
+    EXPECT_EQ(ok, true);
+}