diff --git a/framework b/framework index dff9da0443..c6610dde67 160000 --- a/framework +++ b/framework @@ -1 +1 @@ -Subproject commit dff9da04438d712f7647fd995bc90fadd0c0e2ce +Subproject commit c6610dde67ffd2a3a81cc204a73572b9c31a5775 diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index b6f18c5b88..1a73a2a619 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -6,34 +6,37 @@ This script can also run on outcomes from a partial run, but the results are less likely to be useful. """ +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later + +import importlib +import importlib.machinery +import importlib.util +import os import re import typing import scripts_path # pylint: disable=unused-import from mbedtls_framework import outcome_analysis +from mbedtls_framework import typing_util + + +class CryptoAnalyzeOutcomesType(typing_util.Protocol): + """Our expectations on tf-psa-crypto/tests/scripts/tf_psa_crypto_test_case_info.py. + + See CoverageTask._load_crypto_module(). + """ + #pylint: disable=too-few-public-methods + + # Test cases that are about internal aspects of TF-PSA-Crypto, + # which Mbed TLS is therefore not required to cover. + INTERNAL_TEST_CASES: outcome_analysis.TestCaseSetDescription class CoverageTask(outcome_analysis.CoverageTask): """Justify test cases that are never executed.""" - @staticmethod - def _has_word_re(words: typing.Iterable[str], - exclude: typing.Optional[str] = None) -> typing.Pattern: - """Construct a regex that matches if any of the words appears. - - The occurrence must start and end at a word boundary. - - If exclude is specified, strings containing a match for that - regular expression will not match the returned pattern. - """ - exclude_clause = r'' - if exclude: - exclude_clause = r'(?!.*' + exclude + ')' - return re.compile(exclude_clause + - r'.*\b(?:' + r'|'.join(words) + r')\b.*', - re.DOTALL) - - IGNORED_TESTS = { + UNCOVERED_TESTS = { 'ssl-opt': [ # We don't run ssl-opt.sh with Valgrind on the CI because # it's extremely slow. We don't intend to change this. @@ -71,12 +74,6 @@ class CoverageTask(outcome_analysis.CoverageTask): # https://github.com/Mbed-TLS/mbedtls/issues/9586 'Config: !MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_PSK_ENABLED', ], - 'test_suite_config.crypto_combinations': [ - # New thing in crypto. Not intended to be tested separately - # in mbedtls. - # https://github.com/Mbed-TLS/mbedtls/issues/10300 - 'Config: entropy: NV seed only', - ], 'test_suite_config.psa_boolean': [ # We don't test with HMAC disabled. # https://github.com/Mbed-TLS/mbedtls/issues/9591 @@ -227,6 +224,51 @@ class CoverageTask(outcome_analysis.CoverageTask): ], } + def _load_crypto_module(self) -> None: + """Try to load the information about test cases from the tf-psa-crypto submodule..""" + # All this complexity is because we don't want to add the directory + # to the import path. + if self.crypto_module is not None: + return + crypto_script_path = 'tf-psa-crypto/tests/scripts/tf_psa_crypto_test_case_info.py' + if not os.path.exists(crypto_script_path): + # During a transition period, while the crypto script is not + # yet present in all branches we care about, allow it not to + # exist. + return + crypto_spec = importlib.util.spec_from_file_location( + 'tf_psa_crypto_test_case_info', + crypto_script_path) + # Assertions and type annotation to help mypy. + assert crypto_spec is not None + assert crypto_spec.loader is not None + self.crypto_module: typing.Optional[CryptoAnalyzeOutcomesType] = \ + importlib.util.module_from_spec(crypto_spec) + crypto_spec.loader.exec_module(self.crypto_module) + + def _load_crypto_instructions(self) -> None: + """Try to load instructions from the tf-psa-crypto submodule's outcome analysis.""" + self._load_crypto_module() + if self.crypto_module is not None: + crypto_internal_test_cases = self.crypto_module.INTERNAL_TEST_CASES + else: + # Legacy set of tests covered by TF-PSA-Crypto only, + # from before Mbed TLS's outcome analysis read that information + # from TF-PSA-Crypto. This branch can be removed once + # the presence of the crypto module becomes mandatory. + crypto_internal_test_cases = { + 'test_suite_config.crypto_combinations': [ + 'Config: entropy: NV seed only', + ], + } + self.ignored_tests.extend(crypto_internal_test_cases) + + def __init__(self, options) -> None: + super().__init__(options) + self.crypto_module = None # declared with a type in _load_crypto_module above + self._load_crypto_instructions() + + # List of tasks with a function that can handle this task and additional arguments if required KNOWN_TASKS: typing.Dict[str, typing.Type[outcome_analysis.Task]] = { 'analyze_coverage': CoverageTask, diff --git a/tf-psa-crypto b/tf-psa-crypto index 426f86031a..8c29e401e9 160000 --- a/tf-psa-crypto +++ b/tf-psa-crypto @@ -1 +1 @@ -Subproject commit 426f86031a37bf317fbf0fee9251eb6e612ae58e +Subproject commit 8c29e401e9c1a3180a1eca6aed13958453276550