ssl: accept TLS 1.2 rsa_pss_rsae in client SKE

Fix a TLS 1.2 client regression that caused valid ServerKeyExchange signatures using rsa_pss_rsae_* to be rejected.

Allow rsa_pss_rsae_* in the TLS 1.2 client ServerKeyExchange parse path when the algorithm is supported and was offered by the client. Add OpenSSL and GnuTLS interoperability coverage for TLS 1.2 servers that force rsa_pss_rsae_sha256.

Fixes #10668.

Signed-off-by: Viktor Sokolovskiy <maokaman@gmail.com>

ChangeLog.d/fix-tls12-rsa-pss-sigalgs.txt

@@ -0,0 +1,4 @@
+Bugfix
+   * Fix a TLS 1.2 regression that caused clients to reject valid
+     ServerKeyExchange signatures using RSA-PSS signature algorithms.
+     Fixes #10668.

library/ssl_tls12_client.c

@@ -12,6 +12,7 @@
 #include "mbedtls/platform.h"
 
 #include "mbedtls/ssl.h"
+#include "ssl_debug_helpers.h"
 #include "ssl_client.h"
 #include "debug_internal.h"
 #include "mbedtls/error.h"
@@ -1742,32 +1743,76 @@ static int ssl_parse_signature_algorithm(mbedtls_ssl_context *ssl,
 {
     if (mbedtls_ssl_get_pk_sigalg_and_md_alg_from_sig_alg(sig_alg, pk_alg, md_alg) != 0) {
         MBEDTLS_SSL_DEBUG_MSG(1,
-                              ("Server used unsupported value in SigAlg extension 0x%04x",
-                               sig_alg));
+                              ("Server used unsupported %s signature algorithm",
+                               mbedtls_ssl_sig_alg_to_str(sig_alg)));
         return MBEDTLS_SSL_ALERT_MSG_ILLEGAL_PARAMETER;
     }
 
     /*
-     * mbedtls_ssl_get_pk_sigalg_and_md_alg_from_sig_alg() understands sig_alg code points across
-     * TLS versions. Make sure that the received sig_alg extension is valid in TLS 1.2.
+     * mbedtls_ssl_get_pk_sigalg_and_md_alg_from_sig_alg() understands
+     * signature algorithm code points from both TLS 1.2 and TLS 1.3. Make sure
+     * that the selected signature algorithm is acceptable when TLS 1.2 is
+     * negotiated.
+     *
+     * In TLS 1.2, RSA-PSS signature algorithms (rsa_pss_rsae_*) are not
+     * defined by RFC 5246. However, RFC 8446 Section 4.2.3 requires that
+     * implementations which advertise support for RSASSA-PSS must be
+     * prepared to accept such signatures even when TLS 1.2 is negotiated,
+     * provided they were offered in the signature_algorithms extension.
+     *
+     * Therefore, we allow rsa_pss_rsae_* here if:
+     *  - the implementation supports them, and
+     *  - they were offered in the signature_algorithms extension (checked by
+     *    `mbedtls_ssl_sig_alg_is_offered()` below).
+     *
+     * If we were to add full support for rsa_pss_rsae_* signature algorithms
+     * in TLS 1.2 (not defined by RFC 5246; RFC 8446 requires implementations
+     * that advertise RSASSA-PSS to accept such signatures even when TLS 1.2
+     * is negotiated; in practice, several TLS implementations also offer and
+     * use these algorithms in TLS 1.2-only configurations), we should then
+     * integrate RSA-PSS into the TLS 1.2 signature algorithm support logic
+     * (`mbedtls_ssl_tls12_sig_alg_is_supported()`) instead of handling it as a
+     * special case here.
      */
     if (!mbedtls_ssl_sig_alg_is_supported(ssl, sig_alg)) {
-        MBEDTLS_SSL_DEBUG_MSG(1,
-                              ("Server used unsupported value in SigAlg extension 0x%04x",
-                               sig_alg));
-        return MBEDTLS_SSL_ALERT_MSG_ILLEGAL_PARAMETER;
+        switch (sig_alg) {
+#if defined(PSA_WANT_ALG_RSA_PSS)
+#if defined(PSA_WANT_ALG_SHA_256)
+            case MBEDTLS_TLS1_3_SIG_RSA_PSS_RSAE_SHA256:
+#endif
+#if defined(PSA_WANT_ALG_SHA_384)
+            case MBEDTLS_TLS1_3_SIG_RSA_PSS_RSAE_SHA384:
+#endif
+#if defined(PSA_WANT_ALG_SHA_512)
+            case MBEDTLS_TLS1_3_SIG_RSA_PSS_RSAE_SHA512:
+#endif
+#if defined(PSA_WANT_ALG_SHA_256) || defined(PSA_WANT_ALG_SHA_384) || defined(PSA_WANT_ALG_SHA_512)
+                MBEDTLS_SSL_DEBUG_MSG(3,
+                                      ("Accepting TLS 1.2 RSA-PSS signature algorithm %s via compatibility exception",
+                                       mbedtls_ssl_sig_alg_to_str(sig_alg)));
+                break;
+#endif
+#endif /* PSA_WANT_ALG_RSA_PSS */
+            default:
+                MBEDTLS_SSL_DEBUG_MSG(1,
+                                      ("Server used unsupported %s signature algorithm",
+                                       mbedtls_ssl_sig_alg_to_str(sig_alg)));
+                return MBEDTLS_SSL_ALERT_MSG_ILLEGAL_PARAMETER;
+        }
     }
 
     /*
      * Check if the signature algorithm is acceptable
      */
     if (!mbedtls_ssl_sig_alg_is_offered(ssl, sig_alg)) {
-        MBEDTLS_SSL_DEBUG_MSG(1, ("Server used SigAlg value 0x%04x that was not offered", sig_alg));
+        MBEDTLS_SSL_DEBUG_MSG(1,
+                              ("Server used the signature algorithm %s that was not offered",
+                               mbedtls_ssl_sig_alg_to_str(sig_alg)));
         return MBEDTLS_SSL_ALERT_MSG_ILLEGAL_PARAMETER;
     }
 
-    MBEDTLS_SSL_DEBUG_MSG(2, ("Server used SignatureAlgorithm %d", sig_alg & 0x00FF));
-    MBEDTLS_SSL_DEBUG_MSG(2, ("Server used HashAlgorithm %d", sig_alg >> 8));
+    MBEDTLS_SSL_DEBUG_MSG(2, ("Server used the signature algorithm %s",
+                              mbedtls_ssl_sig_alg_to_str(sig_alg)));
 
     return 0;
 }

tests/ssl-opt.sh

@@ -13954,6 +13954,43 @@ run_test    "TLS 1.2: Check rsa_pss_rsae compatibility issue, m->G" \
             -c "Protocol is TLSv1.2" \
             -c "HTTP/1.0 200 [Oo][Kk]"
 
+requires_openssl_tls1_3_with_compatible_ephemeral
+requires_config_enabled MBEDTLS_SSL_PROTO_TLS1_2
+requires_config_enabled MBEDTLS_SSL_PROTO_TLS1_3
+requires_config_enabled MBEDTLS_DEBUG_C
+requires_config_enabled MBEDTLS_SSL_CLI_C
+requires_config_enabled PSA_WANT_ALG_RSA_PSS
+requires_config_enabled PSA_WANT_ALG_SHA_256
+run_test    "TLS 1.2: Server forces TLS 1.2 and rsa_pss_rsae_sha256, m->O" \
+            "$O_NEXT_SRV_NO_CERT -cert $DATA_FILES_PATH/server2-sha256.crt -key $DATA_FILES_PATH/server2.key
+                                 -tls1_2 -sigalgs rsa_pss_rsae_sha256 " \
+            "$P_CLI debug_level=3" \
+            0 \
+            -c "sent signature scheme \\[804\\] rsa_pss_rsae_sha256" \
+            -c "Perform .* computation of digest of ServerKeyExchange" \
+            -c "Server used the signature algorithm rsa_pss_rsae_sha256" \
+            -c "Protocol is TLSv1.2" \
+            -c "HTTP/1.0 200 [Oo][Kk]"
+
+requires_gnutls_tls1_3
+requires_config_enabled MBEDTLS_SSL_PROTO_TLS1_2
+requires_config_enabled MBEDTLS_SSL_PROTO_TLS1_3
+requires_config_enabled MBEDTLS_DEBUG_C
+requires_config_enabled MBEDTLS_SSL_CLI_C
+requires_config_enabled PSA_WANT_ALG_RSA_PSS
+requires_config_enabled PSA_WANT_ALG_SHA_256
+run_test    "TLS 1.2: Server forces TLS 1.2 and rsa_pss_rsae_sha256, m->G" \
+            "$G_NEXT_SRV_NO_CERT --x509certfile $DATA_FILES_PATH/server2-sha256.crt --x509keyfile $DATA_FILES_PATH/server2.key
+                    --disable-client-cert
+                    --priority=NORMAL:-VERS-ALL:+VERS-TLS1.2:-SIGN-ALL:+SIGN-RSA-PSS-RSAE-SHA256" \
+            "$P_CLI debug_level=3" \
+            0 \
+            -c "sent signature scheme \\[804\\] rsa_pss_rsae_sha256" \
+            -c "Perform .* computation of digest of ServerKeyExchange" \
+            -c "Server used the signature algorithm rsa_pss_rsae_sha256" \
+            -c "Protocol is TLSv1.2" \
+            -c "HTTP/1.0 200 [Oo][Kk]"
+
 requires_config_enabled MBEDTLS_SSL_SRV_C
 requires_config_enabled MBEDTLS_DEBUG_C
 requires_config_enabled MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_EPHEMERAL_ENABLED