From 619f1acd75e3daa652659a0d0b6046b54be2855e Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sun, 5 Apr 2026 20:41:34 +0200 Subject: [PATCH 1/9] Update framework with UNCOVERED_TESTS in outcome analysis Signed-off-by: Gilles Peskine --- framework | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/framework b/framework index dff9da0443..80a0ea93f0 160000 --- a/framework +++ b/framework @@ -1 +1 @@ -Subproject commit dff9da04438d712f7647fd995bc90fadd0c0e2ce +Subproject commit 80a0ea93f0215bcd9030734904b4b54fb8306f07 From 68d6b072877fb99aab7c5373289912e68ef6bd46 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sun, 5 Apr 2026 20:41:58 +0200 Subject: [PATCH 2/9] Rename IGNORED_TESTS to UNCOVERED_TESTS Signed-off-by: Gilles Peskine --- tests/scripts/analyze_outcomes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index b6f18c5b88..7ca1f760cb 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -33,7 +33,7 @@ class CoverageTask(outcome_analysis.CoverageTask): r'.*\b(?:' + r'|'.join(words) + r')\b.*', re.DOTALL) - IGNORED_TESTS = { + UNCOVERED_TESTS = { 'ssl-opt': [ # We don't run ssl-opt.sh with Valgrind on the CI because # it's extremely slow. We don't intend to change this. From bb5cfbbdec43242901fe8a9d74291ed3300a622c Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sun, 5 Apr 2026 20:43:52 +0200 Subject: [PATCH 3/9] Move _has_word_re to the framework Signed-off-by: Gilles Peskine --- tests/scripts/analyze_outcomes.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 7ca1f760cb..5a9b343034 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -16,23 +16,6 @@ from mbedtls_framework import outcome_analysis class CoverageTask(outcome_analysis.CoverageTask): """Justify test cases that are never executed.""" - @staticmethod - def _has_word_re(words: typing.Iterable[str], - exclude: typing.Optional[str] = None) -> typing.Pattern: - """Construct a regex that matches if any of the words appears. - - The occurrence must start and end at a word boundary. - - If exclude is specified, strings containing a match for that - regular expression will not match the returned pattern. - """ - exclude_clause = r'' - if exclude: - exclude_clause = r'(?!.*' + exclude + ')' - return re.compile(exclude_clause + - r'.*\b(?:' + r'|'.join(words) + r')\b.*', - re.DOTALL) - UNCOVERED_TESTS = { 'ssl-opt': [ # We don't run ssl-opt.sh with Valgrind on the CI because From 1978e1bd6b2181021a839c42ab7e4b0157faffdb Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 6 Apr 2026 21:51:25 +0200 Subject: [PATCH 4/9] Ignore test cases that TF-PSA-Crypto tells us to ignore If the `tf-psa-crypto` submodule has `tests/scripts/analyze_outcomes.py`, require it to define a global variable `INTERNAL_TEST_CASES`. Those test cases will be ignored in Mbed TLS's coverage analysis. Signed-off-by: Gilles Peskine --- tests/scripts/analyze_outcomes.py | 50 +++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 5a9b343034..1e5ab43634 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -6,11 +6,28 @@ This script can also run on outcomes from a partial run, but the results are less likely to be useful. """ +import importlib +import importlib.machinery +import importlib.util +import os import re import typing import scripts_path # pylint: disable=unused-import from mbedtls_framework import outcome_analysis +from mbedtls_framework import typing_util + + +class CryptoAnalyzeOutcomesType(typing_util.Protocol): + """Our expectations on tf-psa-crypto/tests/scripts/analyze_outcomes.py. + + See CoverageTask_load_crypto_module(). + """ + #pylint: disable=too-few-public-methods + + # Test cases that are about internal aspects of TF-PSA-Crypto, + # which Mbed TLS is therefore not required to cover. + INTERNAL_TEST_CASES: outcome_analysis.TestCaseSetDescription class CoverageTask(outcome_analysis.CoverageTask): @@ -210,6 +227,39 @@ class CoverageTask(outcome_analysis.CoverageTask): ], } + def _load_crypto_module(self) -> None: + """Try to load the tf-psa-crypto submodule's outcome analysis Python module.""" + if self.crypto_module is not None: + return + crypto_script_path = 'tf-psa-crypto/tests/scripts/analyze_outcomes.py' + if not os.path.exists(crypto_script_path): + # During a transition period, while the crypto script is not + # yet present in all branches we care about, allow it not to + # exist. + return + crypto_spec = importlib.util.spec_from_file_location( + 'tf_psa_crypto.analyze_outcomes', + crypto_script_path) + # Assertions to help mypy. + assert crypto_spec is not None + assert crypto_spec.loader is not None + self.crypto_module: typing.Optional[CryptoAnalyzeOutcomesType] = \ + importlib.util.module_from_spec(crypto_spec) + crypto_spec.loader.exec_module(self.crypto_module) + + def _load_crypto_instructions(self) -> None: + """Try to load instructions from the tf-psa-crypto submodule's outcome analysis.""" + self._load_crypto_module() + if self.crypto_module is not None: + crypto_internal_test_cases = self.crypto_module.INTERNAL_TEST_CASES + self.ignored_tests.extend(crypto_internal_test_cases) + + def __init__(self, options) -> None: + super().__init__(options) + self.crypto_module = None # declared with a type in _load_crypto_module above + self._load_crypto_instructions() + + # List of tasks with a function that can handle this task and additional arguments if required KNOWN_TASKS: typing.Dict[str, typing.Type[outcome_analysis.Task]] = { 'analyze_coverage': CoverageTask, From 667a3f6442d55231b64106d5974fbd537ed94849 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 7 Apr 2026 11:47:24 +0200 Subject: [PATCH 5/9] Move test currently covered by crypto from uncovered list to ignored list If we can't read `INTERNAL_TEST_CASES` from `tf-psa-crypto/tests/scripts/analyze_outcomes.py` because the script doesn't exist, hard-code the legacy value of that information. Signed-off-by: Gilles Peskine --- tests/scripts/analyze_outcomes.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 1e5ab43634..48f00f03c4 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -71,12 +71,6 @@ class CoverageTask(outcome_analysis.CoverageTask): # https://github.com/Mbed-TLS/mbedtls/issues/9586 'Config: !MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_PSK_ENABLED', ], - 'test_suite_config.crypto_combinations': [ - # New thing in crypto. Not intended to be tested separately - # in mbedtls. - # https://github.com/Mbed-TLS/mbedtls/issues/10300 - 'Config: entropy: NV seed only', - ], 'test_suite_config.psa_boolean': [ # We don't test with HMAC disabled. # https://github.com/Mbed-TLS/mbedtls/issues/9591 @@ -252,7 +246,17 @@ class CoverageTask(outcome_analysis.CoverageTask): self._load_crypto_module() if self.crypto_module is not None: crypto_internal_test_cases = self.crypto_module.INTERNAL_TEST_CASES - self.ignored_tests.extend(crypto_internal_test_cases) + else: + # Legacy set of tests covered by TF-PSA-Crypto only, + # from before Mbed TLS's outcome analysis read that information + # from TF-PSA-Crypto. This branch can be removed once + # the presence of the crypto module becomes mandatory. + crypto_internal_test_cases = { + 'test_suite_config.crypto_combinations': [ + 'Config: entropy: NV seed only', + ], + } + self.ignored_tests.extend(crypto_internal_test_cases) def __init__(self, options) -> None: super().__init__(options) From 16a90a556e7ee045cc2633545de799ebb729e795 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 8 Apr 2026 15:31:52 +0200 Subject: [PATCH 6/9] Add copyright line Signed-off-by: Gilles Peskine --- tests/scripts/analyze_outcomes.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 48f00f03c4..226ca54c14 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -6,6 +6,9 @@ This script can also run on outcomes from a partial run, but the results are less likely to be useful. """ +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later + import importlib import importlib.machinery import importlib.util From d25f03919a3de57dc7e3492804ad25dcbf597eb2 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 8 Apr 2026 15:47:49 +0200 Subject: [PATCH 7/9] INTERNAL_TEST_CASES moved to a separate data-only module This way, when Mbed TLS's `analyze_outcomes.py` loads the python module from TF-PSA-Crypto (because it needs to know the value of `INTERNAL_TEST_CASES`), there's no risk that the subproject and the superproject will have different requirements on auxiliary modules such as `mbedtls_framework.outcome_analysis`. Signed-off-by: Gilles Peskine --- tests/scripts/analyze_outcomes.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 226ca54c14..7ce88f9921 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -22,7 +22,7 @@ from mbedtls_framework import typing_util class CryptoAnalyzeOutcomesType(typing_util.Protocol): - """Our expectations on tf-psa-crypto/tests/scripts/analyze_outcomes.py. + """Our expectations on tf-psa-crypto/tests/scripts/tf_psa_crypto_test_case_info.py. See CoverageTask_load_crypto_module(). """ @@ -225,17 +225,19 @@ class CoverageTask(outcome_analysis.CoverageTask): } def _load_crypto_module(self) -> None: - """Try to load the tf-psa-crypto submodule's outcome analysis Python module.""" + """Try to load the information about test cases from the tf-psa-crypto submodule..""" + # All this complexity is because we don't want to add the directory + # to the import path. if self.crypto_module is not None: return - crypto_script_path = 'tf-psa-crypto/tests/scripts/analyze_outcomes.py' + crypto_script_path = 'tf-psa-crypto/tests/scripts/tf_psa_crypto_test_case_info.py' if not os.path.exists(crypto_script_path): # During a transition period, while the crypto script is not # yet present in all branches we care about, allow it not to # exist. return crypto_spec = importlib.util.spec_from_file_location( - 'tf_psa_crypto.analyze_outcomes', + 'tf_psa_crypto_test_case_info', crypto_script_path) # Assertions to help mypy. assert crypto_spec is not None From 806e1d365b254c12ed88af6ec94f71c932aaeedb Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 8 Apr 2026 17:22:10 +0200 Subject: [PATCH 8/9] Documentation improvements Signed-off-by: Gilles Peskine --- tests/scripts/analyze_outcomes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 7ce88f9921..1a73a2a619 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -24,7 +24,7 @@ from mbedtls_framework import typing_util class CryptoAnalyzeOutcomesType(typing_util.Protocol): """Our expectations on tf-psa-crypto/tests/scripts/tf_psa_crypto_test_case_info.py. - See CoverageTask_load_crypto_module(). + See CoverageTask._load_crypto_module(). """ #pylint: disable=too-few-public-methods @@ -239,7 +239,7 @@ class CoverageTask(outcome_analysis.CoverageTask): crypto_spec = importlib.util.spec_from_file_location( 'tf_psa_crypto_test_case_info', crypto_script_path) - # Assertions to help mypy. + # Assertions and type annotation to help mypy. assert crypto_spec is not None assert crypto_spec.loader is not None self.crypto_module: typing.Optional[CryptoAnalyzeOutcomesType] = \ From cc134b0b94836a6044c76c2d7d362848811d0855 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 10 Apr 2026 14:49:14 +0200 Subject: [PATCH 9/9] Update crypto submodule with analyze_outcomes.py Update framework to match. Signed-off-by: Gilles Peskine --- framework | 2 +- tf-psa-crypto | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/framework b/framework index 80a0ea93f0..c6610dde67 160000 --- a/framework +++ b/framework @@ -1 +1 @@ -Subproject commit 80a0ea93f0215bcd9030734904b4b54fb8306f07 +Subproject commit c6610dde67ffd2a3a81cc204a73572b9c31a5775 diff --git a/tf-psa-crypto b/tf-psa-crypto index 426f86031a..8c29e401e9 160000 --- a/tf-psa-crypto +++ b/tf-psa-crypto @@ -1 +1 @@ -Subproject commit 426f86031a37bf317fbf0fee9251eb6e612ae58e +Subproject commit 8c29e401e9c1a3180a1eca6aed13958453276550