LibTLS: Allow using other hash algorithms for HMAC

The standard allows for ciphers to define which hash to use.
Fixes #7348

Userland/Libraries/LibTLS/Handshake.cpp

@@ -140,12 +140,17 @@ ByteBuffer TLSv12::build_handshake_finished()
     PacketBuilder builder { MessageType::Handshake, m_context.options.version, 12 + 64 };
     builder.append((u8)HandshakeType::Finished);
 
-    constexpr u32 out_size = 12;
+    // RFC 5246 section 7.4.9: "In previous versions of TLS, the verify_data was always 12 octets
+    //                          long.  In the current version of TLS, it depends on the cipher
+    //                          suite.  Any cipher suite which does not explicitly specify
+    //                          verify_data_length has a verify_data_length equal to 12."
+    // Simplification: Assume that verify_data_length is always 12.
+    constexpr u32 verify_data_length = 12;
 
-    builder.append_u24(out_size);
+    builder.append_u24(verify_data_length);
 
-    u8 out[out_size];
+    u8 out[verify_data_length];
-    auto outbuffer = Bytes { out, out_size };
+    auto outbuffer = Bytes { out, verify_data_length };
     auto dummy = ByteBuffer::create_zeroed(0);
 
     auto digest = m_context.handshake_hash.digest();

Userland/Libraries/LibTLS/HandshakeServer.cpp

@@ -81,8 +81,8 @@ ssize_t TLSv12::handle_server_hello(ReadonlyBytes buffer, WritePacketStage& writ
     m_context.cipher = cipher;
     dbgln_if(TLS_DEBUG, "Cipher: {}", (u16)cipher);
 
-    // The handshake hash function is _always_ SHA256
+    // Simplification: We only support handshake hash functions via HMAC
-    m_context.handshake_hash.initialize(Crypto::Hash::HashKind::SHA256);
+    m_context.handshake_hash.initialize(hmac_hash());
 
     // Compression method
     if (buffer.size() - res < 1)

Userland/Libraries/LibTLS/TLSv12.cpp

@@ -232,19 +232,14 @@ bool Context::verify_chain() const
     return true;
 }
 
-void TLSv12::pseudorandom_function(Bytes output, ReadonlyBytes secret, const u8* label, size_t label_length, ReadonlyBytes seed, ReadonlyBytes seed_b)
+template<typename HMACType>
+static void hmac_pseudorandom_function(Bytes output, ReadonlyBytes secret, const u8* label, size_t label_length, ReadonlyBytes seed, ReadonlyBytes seed_b)
 {
     if (!secret.size()) {
         dbgln("null secret");
         return;
     }
 
-    // RFC 5246: "In this section, we define one PRF, based on HMAC.  This PRF with the
-    //            SHA-256 hash function is used for all cipher suites defined in this
-    //            document and in TLS documents published prior to this document when
-    //            TLS 1.2 is negotiated."
-    // Apparently this PRF _always_ uses SHA256
-
     auto append_label_seed = [&](auto& hmac) {
         hmac.update(label, label_length);
         hmac.update(seed);
@@ -252,7 +247,7 @@ void TLSv12::pseudorandom_function(Bytes output, ReadonlyBytes secret, const u8*
             hmac.update(seed_b);
     };
 
-    Crypto::Authentication::HMAC<Crypto::Hash::SHA256> hmac(secret);
+    HMACType hmac(secret);
     append_label_seed(hmac);
 
     constexpr auto digest_size = hmac.digest_size();
@@ -276,6 +271,34 @@ void TLSv12::pseudorandom_function(Bytes output, ReadonlyBytes secret, const u8*
     }
 }
 
+void TLSv12::pseudorandom_function(Bytes output, ReadonlyBytes secret, const u8* label, size_t label_length, ReadonlyBytes seed, ReadonlyBytes seed_b)
+{
+    // Simplification: We only support the HMAC PRF with the hash function SHA-256 or stronger.
+
+    // RFC 5246: "In this section, we define one PRF, based on HMAC.  This PRF with the
+    //            SHA-256 hash function is used for all cipher suites defined in this
+    //            document and in TLS documents published prior to this document when
+    //            TLS 1.2 is negotiated.  New cipher suites MUST explicitly specify a
+    //            PRF and, in general, SHOULD use the TLS PRF with SHA-256 or a
+    //            stronger standard hash function."
+
+    switch (hmac_hash()) {
+    case Crypto::Hash::HashKind::SHA512:
+        hmac_pseudorandom_function<Crypto::Authentication::HMAC<Crypto::Hash::SHA512>>(output, secret, label, label_length, seed, seed_b);
+        break;
+    case Crypto::Hash::HashKind::SHA384:
+        hmac_pseudorandom_function<Crypto::Authentication::HMAC<Crypto::Hash::SHA384>>(output, secret, label, label_length, seed, seed_b);
+        break;
+    case Crypto::Hash::HashKind::SHA256:
+        hmac_pseudorandom_function<Crypto::Authentication::HMAC<Crypto::Hash::SHA256>>(output, secret, label, label_length, seed, seed_b);
+        break;
+    default:
+        dbgln("Failed to find a suitable HMAC hash");
+        VERIFY_NOT_REACHED();
+        break;
+    }
+}
+
 TLSv12::TLSv12(Core::Object* parent, Options options)
     : Core::Socket(Core::Socket::Type::TCP, parent)
 {

Userland/Libraries/LibTLS/TLSv12.h

@@ -447,6 +447,20 @@ private:
         }
     }
 
+    Crypto::Hash::HashKind hmac_hash() const
+    {
+        switch (mac_length()) {
+        case Crypto::Hash::SHA512::DigestSize:
+            return Crypto::Hash::HashKind::SHA512;
+        case Crypto::Hash::SHA384::DigestSize:
+            return Crypto::Hash::HashKind::SHA384;
+        case Crypto::Hash::SHA256::DigestSize:
+        case Crypto::Hash::SHA1::DigestSize:
+        default:
+            return Crypto::Hash::HashKind::SHA256;
+        }
+    }
+
     size_t iv_length() const
     {
         switch (m_context.cipher) {