From 9ba9c21c616e9dee97885cea78dc3d685fe2c872 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 23 May 2024 15:03:43 +0200 Subject: [PATCH 1/9] Recognize that a double-inclusion guard is not a config setting Fix PSA_CRYPTO_CONFIG_H being treated as a configuration setting in include/psa/crypto_config.h. Signed-off-by: Gilles Peskine --- scripts/config.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scripts/config.py b/scripts/config.py index c53f9e7fe200..8704bdb51ea3 100755 --- a/scripts/config.py +++ b/scripts/config.py @@ -396,6 +396,7 @@ def __init__(self, filename=None): self.default_path) super().__init__() self.filename = filename + self.inclusion_guard = None self.current_section = 'header' with open(filename, 'r', encoding='utf-8') as file: self.templates = [self._parse_line(line) for line in file] @@ -413,9 +414,11 @@ def set(self, name, value=None): r'(?P(?:\((?:\w|\s|,)*\))?)' + r'(?P\s*)' + r'(?P.*)') + _ifndef_line_regexp = r'#ifndef (?P\w+)' _section_line_regexp = (r'\s*/?\*+\s*[\\@]name\s+SECTION:\s*' + r'(?P
.*)[ */]*') _config_line_regexp = re.compile(r'|'.join([_define_line_regexp, + _ifndef_line_regexp, _section_line_regexp])) def _parse_line(self, line): """Parse a line in mbedtls_config.h and return the corresponding template.""" @@ -426,10 +429,16 @@ def _parse_line(self, line): elif m.group('section'): self.current_section = m.group('section') return line + elif m.group('inclusion_guard') and self.inclusion_guard is None: + self.inclusion_guard = m.group('inclusion_guard') + return line else: active = not m.group('commented_out') name = m.group('name') value = m.group('value') + if name == self.inclusion_guard and value == '': + # The file double-inclusion guard is not an option. + return line template = (name, m.group('indentation'), m.group('define') + name + From b0aa75e7cc8f9d3bff828d827240dce1ef9273fe Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 23 May 2024 16:38:07 +0200 Subject: [PATCH 2/9] Clean up generated files enumeration Avoid having to list multiple generation scripts on the same line. No intended semantic change. Signed-off-by: Gilles Peskine --- tests/Makefile | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index d1d5ed97212c..dcbd066a3342 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -17,7 +17,6 @@ ifdef RECORD_PSA_STATUS_COVERAGE_LOG LOCAL_CFLAGS += -Werror -DRECORD_PSA_STATUS_COVERAGE_LOG endif -.PHONY: generated_files GENERATED_BIGNUM_DATA_FILES := $(patsubst tests/%,%,$(shell \ $(PYTHON) ../framework/scripts/generate_bignum_tests.py --list || \ echo FAILED \ @@ -25,6 +24,8 @@ GENERATED_BIGNUM_DATA_FILES := $(patsubst tests/%,%,$(shell \ ifeq ($(GENERATED_BIGNUM_DATA_FILES),FAILED) $(error "$(PYTHON) ../framework/scripts/generate_bignum_tests.py --list" failed) endif +GENERATED_DATA_FILES += $(GENERATED_BIGNUM_DATA_FILES) + GENERATED_ECP_DATA_FILES := $(patsubst tests/%,%,$(shell \ $(PYTHON) ../framework/scripts/generate_ecp_tests.py --list || \ echo FAILED \ @@ -32,6 +33,8 @@ GENERATED_ECP_DATA_FILES := $(patsubst tests/%,%,$(shell \ ifeq ($(GENERATED_ECP_DATA_FILES),FAILED) $(error "$(PYTHON) ../framework/scripts/generate_ecp_tests.py --list" failed) endif +GENERATED_DATA_FILES += $(GENERATED_ECP_DATA_FILES) + GENERATED_PSA_DATA_FILES := $(patsubst tests/%,%,$(shell \ $(PYTHON) ../framework/scripts/generate_psa_tests.py --list || \ echo FAILED \ @@ -39,8 +42,13 @@ GENERATED_PSA_DATA_FILES := $(patsubst tests/%,%,$(shell \ ifeq ($(GENERATED_PSA_DATA_FILES),FAILED) $(error "$(PYTHON) ../framework/scripts/generate_psa_tests.py --list" failed) endif -GENERATED_FILES := $(GENERATED_PSA_DATA_FILES) $(GENERATED_ECP_DATA_FILES) $(GENERATED_BIGNUM_DATA_FILES) -generated_files: $(GENERATED_FILES) src/test_keys.h src/test_certs.h +GENERATED_DATA_FILES += $(GENERATED_PSA_DATA_FILES) + +GENERATED_FILES = $(GENERATED_DATA_FILES) +GENERATED_FILES += src/test_keys.h src/test_certs.h + +.PHONY: generated_files +generated_files: $(GENERATED_FILES) # generate_bignum_tests.py and generate_psa_tests.py spend more time analyzing # inputs than generating outputs. Its inputs are the same no matter which files @@ -48,7 +56,6 @@ generated_files: $(GENERATED_FILES) src/test_keys.h src/test_certs.h # It's rare not to want all the outputs. So always generate all of its outputs. # Use an intermediate phony dependency so that parallel builds don't run # a separate instance of the recipe for each output file. -.SECONDARY: generated_bignum_test_data generated_ecp_test_data generated_psa_test_data $(GENERATED_BIGNUM_DATA_FILES): $(gen_file_dep) generated_bignum_test_data generated_bignum_test_data: ../framework/scripts/generate_bignum_tests.py generated_bignum_test_data: ../framework/scripts/mbedtls_framework/bignum_common.py @@ -60,6 +67,7 @@ generated_bignum_test_data: ../framework/scripts/mbedtls_framework/test_data_gen generated_bignum_test_data: echo " Gen $(GENERATED_BIGNUM_DATA_FILES)" $(PYTHON) ../framework/scripts/generate_bignum_tests.py +.SECONDARY: generated_bignum_test_data $(GENERATED_ECP_DATA_FILES): $(gen_file_dep) generated_ecp_test_data generated_ecp_test_data: ../framework/scripts/generate_ecp_tests.py @@ -70,6 +78,7 @@ generated_ecp_test_data: ../framework/scripts/mbedtls_framework/test_data_genera generated_ecp_test_data: echo " Gen $(GENERATED_ECP_DATA_FILES)" $(PYTHON) ../framework/scripts/generate_ecp_tests.py +.SECONDARY: generated_ecp_test_data $(GENERATED_PSA_DATA_FILES): $(gen_file_dep) generated_psa_test_data generated_psa_test_data: ../framework/scripts/generate_psa_tests.py @@ -92,6 +101,7 @@ generated_psa_test_data: suites/test_suite_psa_crypto_metadata.data generated_psa_test_data: echo " Gen $(GENERATED_PSA_DATA_FILES) ..." $(PYTHON) ../framework/scripts/generate_psa_tests.py +.SECONDARY: generated_psa_test_data # A test application is built for each suites/test_suite_*.data file. # Application name is same as .data file's base name and can be @@ -99,7 +109,7 @@ generated_psa_test_data: DATA_FILES := $(wildcard suites/test_suite_*.data) # Make sure that generated data files are included even if they don't # exist yet when the makefile is parsed. -DATA_FILES += $(filter-out $(DATA_FILES),$(GENERATED_FILES)) +DATA_FILES += $(filter-out $(DATA_FILES),$(GENERATED_DATA_FILES)) APPS = $(basename $(subst suites/,,$(DATA_FILES))) # Construct executable name by adding OS specific suffix $(EXEXT). From ada30fe650407c3fee80adde3cfe2801ec4b3e2e Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 23 May 2024 15:50:44 +0200 Subject: [PATCH 3/9] New test suite to report configuration options Add a test suite intended to report configuration options in the outcome file: we're only interested in SKIP vs PASS. Add a few test cases for some interesting combinations of options. The selection here is just for illustration purposes, more will be added later. A subsequent commit will automatically generate test cases for single options. Signed-off-by: Gilles Peskine --- .../test_suite_config.crypto_combinations.data | 9 +++++++++ tests/suites/test_suite_config.function | 14 ++++++++++++++ .../suites/test_suite_config.psa_combinations.data | 9 +++++++++ .../suites/test_suite_config.tls_combinations.data | 9 +++++++++ 4 files changed, 41 insertions(+) create mode 100644 tests/suites/test_suite_config.crypto_combinations.data create mode 100644 tests/suites/test_suite_config.function create mode 100644 tests/suites/test_suite_config.psa_combinations.data create mode 100644 tests/suites/test_suite_config.tls_combinations.data diff --git a/tests/suites/test_suite_config.crypto_combinations.data b/tests/suites/test_suite_config.crypto_combinations.data new file mode 100644 index 000000000000..d3287d266a3b --- /dev/null +++ b/tests/suites/test_suite_config.crypto_combinations.data @@ -0,0 +1,9 @@ +# Interesting combinations of low-level crypto options + +Config: ECC: Weierstrass curves only +depends_on:MBEDTLS_ECP_SHORT_WEIERSTRASS_ENABLED:!MBEDTLS_ECP_MONTGOMERY_ENABLED +pass: + +Config: ECC: Montgomery curves only +depends_on:!MBEDTLS_ECP_SHORT_WEIERSTRASS_ENABLED:MBEDTLS_ECP_MONTGOMERY_ENABLED +pass: diff --git a/tests/suites/test_suite_config.function b/tests/suites/test_suite_config.function new file mode 100644 index 000000000000..9e9dd019906c --- /dev/null +++ b/tests/suites/test_suite_config.function @@ -0,0 +1,14 @@ +/* BEGIN_HEADER */ + +/* END_HEADER */ + +/* BEGIN_CASE */ +/* This test case always passes. It is intended solely for configuration + * reporting in the outcome file. Write test cases using this function + * with dependencies to record in which configurations the dependencies + * are met. */ +void pass() +{ + goto exit; +} +/* END_CASE */ diff --git a/tests/suites/test_suite_config.psa_combinations.data b/tests/suites/test_suite_config.psa_combinations.data new file mode 100644 index 000000000000..1035af248702 --- /dev/null +++ b/tests/suites/test_suite_config.psa_combinations.data @@ -0,0 +1,9 @@ +# Interesting combinations of PSA options + +Config: PSA_WANT_ALG_ECDSA without PSA_WANT_ALG_DETERMINISTIC_ECDSA +depends_on:PSA_WANT_ALG_ECDSA:!PSA_WANT_ALG_DETERMINISTIC_ECDSA +pass: + +Config: PSA_WANT_ALG_DETERMINSTIC_ECDSA without PSA_WANT_ALG_ECDSA +depends_on:PSA_WANT_ALG_DETERMINISTIC_ECDSA:!PSA_WANT_ALG_ECDSA +pass: diff --git a/tests/suites/test_suite_config.tls_combinations.data b/tests/suites/test_suite_config.tls_combinations.data new file mode 100644 index 000000000000..cbc57d6cd3c6 --- /dev/null +++ b/tests/suites/test_suite_config.tls_combinations.data @@ -0,0 +1,9 @@ +# Interesting combinations of TLS options + +Config: TLS 1.2 without TLS 1.3 +depends_on:MBEDTLS_SSL_PROTO_TLS1_2:!MBEDTLS_SSL_PROTO_TLS1_3 +pass: + +Config: TLS 1.3 without TLS 1.2 +depends_on:MBEDTLS_SSL_PROTO_TLS1_3:!MBEDTLS_SSL_PROTO_TLS1_2 +pass: From e154e6fe51ab4245561e26934059fba3da924f99 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 23 May 2024 16:31:22 +0200 Subject: [PATCH 4/9] Generate config test cases for single options Generate option-on and option-off cases for test_suite_config, for all boolean options (MBEDTLS_xxx and PSA_WANT_xxx, collected from the mbedtls and PSA config files). Signed-off-by: Gilles Peskine --- framework | 2 +- scripts/make_generated_files.bat | 1 + tests/.gitignore | 2 ++ tests/CMakeLists.txt | 31 ++++++++++++++++++++++++++ tests/Makefile | 25 +++++++++++++++++++++ tests/scripts/check-generated-files.sh | 1 + 6 files changed, 61 insertions(+), 1 deletion(-) diff --git a/framework b/framework index 04847216ab96..1893bfb1ee4b 160000 --- a/framework +++ b/framework @@ -1 +1 @@ -Subproject commit 04847216ab964b9bdce41f1e61ccc6d8f5d2a139 +Subproject commit 1893bfb1ee4b1d971db38bf3ea1f8005fe3c8d9a diff --git a/scripts/make_generated_files.bat b/scripts/make_generated_files.bat index f04f6b72a996..b03bce2ade5d 100644 --- a/scripts/make_generated_files.bat +++ b/scripts/make_generated_files.bat @@ -11,6 +11,7 @@ python scripts\generate_ssl_debug_helpers.py || exit /b 1 perl scripts\generate_visualc_files.pl || exit /b 1 python scripts\generate_psa_constants.py || exit /b 1 python framework\scripts\generate_bignum_tests.py || exit /b 1 +python framework\scripts\generate_config_tests.py || exit /b 1 python framework\scripts\generate_ecp_tests.py || exit /b 1 python framework\scripts\generate_psa_tests.py || exit /b 1 python framework\scripts\generate_test_keys.py --output tests\src\test_keys.h || exit /b 1 diff --git a/tests/.gitignore b/tests/.gitignore index 838ea699fc53..635dd6257df6 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -19,6 +19,8 @@ libtestdriver1/* ###START_GENERATED_FILES### # Generated source files /suites/*.generated.data +suites/test_suite_config.mbedtls_boolean.data +suites/test_suite_config.psa_boolean.data /suites/test_suite_psa_crypto_storage_format.v[0-9]*.data /suites/test_suite_psa_crypto_storage_format.current.data /src/test_keys.h diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 62be14e5339b..060a928ab702 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -30,6 +30,18 @@ execute_process( string(REGEX REPLACE "[^;]*/" "" base_bignum_generated_data_files "${base_bignum_generated_data_files}") +execute_process( + COMMAND + ${MBEDTLS_PYTHON_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/generate_config_tests.py + --list-for-cmake + WORKING_DIRECTORY + ${CMAKE_CURRENT_SOURCE_DIR}/.. + OUTPUT_VARIABLE + base_config_generated_data_files) +string(REGEX REPLACE "[^;]*/" "" + base_config_generated_data_files "${base_config_generated_data_files}") + execute_process( COMMAND ${MBEDTLS_PYTHON_EXECUTABLE} @@ -61,11 +73,15 @@ set(base_generated_data_files string(REGEX REPLACE "([^;]+)" "suites/\\1" all_generated_data_files "${base_generated_data_files}") set(bignum_generated_data_files "") +set(config_generated_data_files "") set(ecp_generated_data_files "") set(psa_generated_data_files "") foreach(file ${base_bignum_generated_data_files}) list(APPEND bignum_generated_data_files ${CMAKE_CURRENT_BINARY_DIR}/suites/${file}) endforeach() +foreach(file ${base_config_generated_data_files}) + list(APPEND config_generated_data_files ${CMAKE_CURRENT_BINARY_DIR}/suites/${file}) +endforeach() foreach(file ${base_ecp_generated_data_files}) list(APPEND ecp_generated_data_files ${CMAKE_CURRENT_BINARY_DIR}/suites/${file}) endforeach() @@ -92,6 +108,21 @@ if(GEN_FILES) ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/mbedtls_framework/test_case.py ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/mbedtls_framework/test_data_generation.py ) + add_custom_command( + OUTPUT + ${config_generated_data_files} + WORKING_DIRECTORY + ${CMAKE_CURRENT_SOURCE_DIR}/.. + COMMAND + ${MBEDTLS_PYTHON_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/generate_config_tests.py + --directory ${CMAKE_CURRENT_BINARY_DIR}/suites + DEPENDS + ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/generate_config_tests.py + # Do not declare the configuration files as dependencies: they + # change too often in ways that don't affect the result + # ((un)commenting some options). + ) add_custom_command( OUTPUT ${ecp_generated_data_files} diff --git a/tests/Makefile b/tests/Makefile index dcbd066a3342..b5637f37d334 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -26,6 +26,15 @@ $(error "$(PYTHON) ../framework/scripts/generate_bignum_tests.py --list" failed) endif GENERATED_DATA_FILES += $(GENERATED_BIGNUM_DATA_FILES) +GENERATED_CONFIG_DATA_FILES := $(patsubst tests/%,%,$(shell \ + $(PYTHON) ../framework/scripts/generate_config_tests.py --list || \ + echo FAILED \ +)) +ifeq ($(GENERATED_CONFIG_DATA_FILES),FAILED) +$(error "$(PYTHON) ../framework/scripts/generate_config_tests.py --list" failed) +endif +GENERATED_DATA_FILES += $(GENERATED_CONFIG_DATA_FILES) + GENERATED_ECP_DATA_FILES := $(patsubst tests/%,%,$(shell \ $(PYTHON) ../framework/scripts/generate_ecp_tests.py --list || \ echo FAILED \ @@ -69,6 +78,22 @@ generated_bignum_test_data: $(PYTHON) ../framework/scripts/generate_bignum_tests.py .SECONDARY: generated_bignum_test_data +# We deliberately omit the configuration files (mbedtls_config.h, +# crypto_config.h) from the depenency list because during development +# and on the CI, we often edit those in a way that doesn't change the +# output, to comment out certain options, or even to remove certain +# lines which do affect the output negatively (it will miss the +# corresponding test cases). +$(GENERATED_CONFIG_DATA_FILES): $(gen_file_dep) generated_config_test_data +generated_config_test_data: ../framework/scripts/generate_config_tests.py +generated_config_test_data: ../scripts/config.py +generated_config_test_data: ../framework/scripts/mbedtls_framework/test_case.py +generated_config_test_data: ../framework/scripts/mbedtls_framework/test_data_generation.py +generated_config_test_data: + echo " Gen $(GENERATED_CONFIG_DATA_FILES)" + $(PYTHON) ../framework/scripts/generate_config_tests.py +.SECONDARY: generated_config_test_data + $(GENERATED_ECP_DATA_FILES): $(gen_file_dep) generated_ecp_test_data generated_ecp_test_data: ../framework/scripts/generate_ecp_tests.py generated_ecp_test_data: ../framework/scripts/mbedtls_framework/bignum_common.py diff --git a/tests/scripts/check-generated-files.sh b/tests/scripts/check-generated-files.sh index e740f33865a1..09c850af7ad6 100755 --- a/tests/scripts/check-generated-files.sh +++ b/tests/scripts/check-generated-files.sh @@ -129,6 +129,7 @@ check() # These checks are common to Mbed TLS and TF-PSA-Crypto check scripts/generate_psa_constants.py programs/psa/psa_constant_names_generated.c check framework/scripts/generate_bignum_tests.py $(framework/scripts/generate_bignum_tests.py --list) +check framework/scripts/generate_config_tests.py $(framework/scripts/generate_config_tests.py --list) check framework/scripts/generate_ecp_tests.py $(framework/scripts/generate_ecp_tests.py --list) check framework/scripts/generate_psa_tests.py $(framework/scripts/generate_psa_tests.py --list) check framework/scripts/generate_test_keys.py tests/src/test_keys.h From a7469d3e8ce8f454b2141fd0e6d4705f4e95f6c2 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 24 May 2024 09:18:25 +0200 Subject: [PATCH 5/9] Driver vs referenee: ignore relevant configuration differences The driver-vs-reference checks compare test results in different configurations. Ignore the test results that report differences in configurations that were the point of the comparison. Do compare other configuration reports: this will let us know if the configurations diverge in an unexpected way. Signed-off-by: Gilles Peskine --- tests/scripts/analyze_outcomes.py | 60 ++++++++++++++++++++++++++++--- 1 file changed, 56 insertions(+), 4 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index eb2469495e9a..f8147d1dc11e 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -160,10 +160,10 @@ def analyze_driver_vs_reference(results: Results, outcomes: Outcomes, # don't issue an error if they're skipped with drivers, # but issue an error if they're not (means we have a bad entry). ignored = False - if full_test_suite in ignored_tests: - for str_or_re in ignored_tests[full_test_suite]: - if name_matches_pattern(test_string, str_or_re): - ignored = True + for str_or_re in (ignored_tests.get(full_test_suite, []) + + ignored_tests.get(test_suite, [])): + if name_matches_pattern(test_string, str_or_re): + ignored = True if not ignored and not suite_case in driver_outcomes.successes: results.error("PASS -> SKIP/FAIL: {}", suite_case) @@ -242,6 +242,9 @@ def do_analyze_driver_vs_reference(results: Results, outcomes: Outcomes, args) - 'psa_crypto_low_hash.generated', # testing the builtins ], 'ignored_tests': { + 'test_suite_config': [ + re.compile(r'.*\bMBEDTLS_(MD5|RIPEMD160|SHA[0-9]+)_.*'), + ], 'test_suite_platform': [ # Incompatible with sanitizers (e.g. ASan). If the driver # component uses a sanitizer but the reference component @@ -265,6 +268,10 @@ def do_analyze_driver_vs_reference(results: Results, outcomes: Outcomes, args) - 'psa_crypto_low_hash.generated', ], 'ignored_tests': { + 'test_suite_config': [ + re.compile(r'.*\bMBEDTLS_(MD5|RIPEMD160|SHA[0-9]+)_.*'), + re.compile(r'.*\bMBEDTLS_MD_C\b') + ], 'test_suite_md': [ # Builtin HMAC is not supported in the accelerate component. re.compile('.*HMAC.*'), @@ -304,6 +311,12 @@ def do_analyze_driver_vs_reference(results: Results, outcomes: Outcomes, args) - 'cipher', ], 'ignored_tests': { + 'test_suite_config': [ + re.compile(r'.*\bMBEDTLS_(AES|ARIA|CAMELLIA|CHACHA20|DES)_.*'), + re.compile(r'.*\bMBEDTLS_(CCM|CHACHAPOLY|CMAC|GCM)_.*'), + re.compile(r'.*\bMBEDTLS_AES(\w+)_C\b.*'), + re.compile(r'.*\bMBEDTLS_CIPHER_.*'), + ], # PEM decryption is not supported so far. # The rest of PEM (write, unencrypted read) works though. 'test_suite_pem': [ @@ -357,6 +370,9 @@ def do_analyze_driver_vs_reference(results: Results, outcomes: Outcomes, args) - 'ecdsa', 'ecdh', 'ecjpake', ], 'ignored_tests': { + 'test_suite_config': [ + re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'), + ], 'test_suite_platform': [ # Incompatible with sanitizers (e.g. ASan). If the driver # component uses a sanitizer but the reference component @@ -397,6 +413,10 @@ def do_analyze_driver_vs_reference(results: Results, outcomes: Outcomes, args) - 'ecp', 'ecdsa', 'ecdh', 'ecjpake', ], 'ignored_tests': { + 'test_suite_config': [ + re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'), + re.compile(r'.*\bMBEDTLS_PK_PARSE_EC_COMPRESSED\b.*'), + ], 'test_suite_platform': [ # Incompatible with sanitizers (e.g. ASan). If the driver # component uses a sanitizer but the reference component @@ -436,6 +456,11 @@ def do_analyze_driver_vs_reference(results: Results, outcomes: Outcomes, args) - 'bignum.generated', 'bignum.misc', ], 'ignored_tests': { + 'test_suite_config': [ + re.compile(r'.*\bMBEDTLS_BIGNUM_C\b.*'), + re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'), + re.compile(r'.*\bMBEDTLS_PK_PARSE_EC_COMPRESSED\b.*'), + ], 'test_suite_platform': [ # Incompatible with sanitizers (e.g. ASan). If the driver # component uses a sanitizer but the reference component @@ -485,6 +510,13 @@ def do_analyze_driver_vs_reference(results: Results, outcomes: Outcomes, args) - # provide), even with MBEDTLS_USE_PSA_CRYPTO. re.compile(r'PSK callback:.*\bdhe-psk\b.*'), ], + 'test_suite_config': [ + re.compile(r'.*\bMBEDTLS_BIGNUM_C\b.*'), + re.compile(r'.*\bMBEDTLS_DHM_C\b.*'), + re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'), + re.compile(r'.*\bMBEDTLS_KEY_EXCHANGE_DHE_PSK_ENABLED\b.*'), + re.compile(r'.*\bMBEDTLS_PK_PARSE_EC_COMPRESSED\b.*'), + ], 'test_suite_platform': [ # Incompatible with sanitizers (e.g. ASan). If the driver # component uses a sanitizer but the reference component @@ -523,6 +555,9 @@ def do_analyze_driver_vs_reference(results: Results, outcomes: Outcomes, args) - 'component_driver': 'test_psa_crypto_config_accel_ffdh', 'ignored_suites': ['dhm'], 'ignored_tests': { + 'test_suite_config': [ + re.compile(r'.*\bMBEDTLS_DHM_C\b.*'), + ], 'test_suite_platform': [ # Incompatible with sanitizers (e.g. ASan). If the driver # component uses a sanitizer but the reference component @@ -545,6 +580,15 @@ def do_analyze_driver_vs_reference(results: Results, outcomes: Outcomes, args) - 'bignum.generated', 'bignum.misc', ], 'ignored_tests': { + 'test_suite_config': [ + re.compile(r'.*\bMBEDTLS_BIGNUM_C\b.*'), + re.compile(r'.*\bMBEDTLS_(ASN1\w+)_C\b.*'), + re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECP)_.*'), + re.compile(r'.*\bMBEDTLS_PSA_P256M_DRIVER_ENABLED\b.*') + ], + 'test_suite_config.crypto_combinations': [ + 'Config: ECC: Weierstrass curves only', + ], 'test_suite_platform': [ # Incompatible with sanitizers (e.g. ASan). If the driver # component uses a sanitizer but the reference component @@ -570,6 +614,10 @@ def do_analyze_driver_vs_reference(results: Results, outcomes: Outcomes, args) - 'pk', 'pkwrite', 'pkparse' ], 'ignored_tests': { + 'test_suite_config': [ + re.compile(r'.*\bMBEDTLS_(PKCS1|RSA)_.*'), + re.compile(r'.*\bMBEDTLS_GENPRIME\b.*') + ], 'test_suite_platform': [ # Incompatible with sanitizers (e.g. ASan). If the driver # component uses a sanitizer but the reference component @@ -611,6 +659,10 @@ def do_analyze_driver_vs_reference(results: Results, outcomes: Outcomes, args) - 'cipher.camellia', ], 'ignored_tests': { + 'test_suite_config': [ + re.compile(r'.*\bMBEDTLS_(AES|ARIA|CAMELLIA)_.*'), + re.compile(r'.*\bMBEDTLS_AES(\w+)_C\b.*'), + ], 'test_suite_cmac': [ # Following tests require AES_C/ARIA_C/CAMELLIA_C to be enabled, # but these are not available in the accelerated component. From 863705838407fcee5544919390c2980215f5f204 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 28 May 2024 18:54:55 +0200 Subject: [PATCH 6/9] Anchor relative paths Signed-off-by: Gilles Peskine --- tests/.gitignore | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/.gitignore b/tests/.gitignore index 635dd6257df6..870fa79808c9 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -3,24 +3,24 @@ *.log /test_suite* -data_files/mpi_write -data_files/hmac_drbg_seed -data_files/ctr_drbg_seed -data_files/entropy_seed +/data_files/mpi_write +/data_files/hmac_drbg_seed +/data_files/ctr_drbg_seed +/data_files/entropy_seed -include/alt-extra/psa/crypto_platform_alt.h -include/alt-extra/psa/crypto_struct_alt.h -include/test/instrument_record_status.h +/include/alt-extra/psa/crypto_platform_alt.h +/include/alt-extra/psa/crypto_struct_alt.h +/include/test/instrument_record_status.h -src/libmbed* +/src/libmbed* -libtestdriver1/* +/libtestdriver1/* ###START_GENERATED_FILES### # Generated source files /suites/*.generated.data -suites/test_suite_config.mbedtls_boolean.data -suites/test_suite_config.psa_boolean.data +/suites/test_suite_config.mbedtls_boolean.data +/suites/test_suite_config.psa_boolean.data /suites/test_suite_psa_crypto_storage_format.v[0-9]*.data /suites/test_suite_psa_crypto_storage_format.current.data /src/test_keys.h From ef822c16563ff0477cb13a6d305cde418d89e48e Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 28 May 2024 19:07:24 +0200 Subject: [PATCH 7/9] Add some missing handling for generated test_suite_config.*.data Fixes the files not being generated in the build tree. Signed-off-by: Gilles Peskine --- tests/CMakeLists.txt | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 060a928ab702..213578d370ee 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -173,6 +173,7 @@ endif() # With this line, only 4 sub-makefiles include the above command, that reduces # the risk of a race. add_custom_target(test_suite_bignum_generated_data DEPENDS ${bignum_generated_data_files}) +add_custom_target(test_suite_config_generated_data DEPENDS ${config_generated_data_files}) add_custom_target(test_suite_ecp_generated_data DEPENDS ${ecp_generated_data_files}) add_custom_target(test_suite_psa_generated_data DEPENDS ${psa_generated_data_files}) # If SKIP_TEST_SUITES is not defined with -D, get it from the environment. @@ -230,6 +231,10 @@ function(add_test_suite suite_name) set(data_file ${CMAKE_CURRENT_BINARY_DIR}/suites/test_suite_${data_name}.data) set(dependency test_suite_bignum_generated_data) + elseif(";${config_generated_data_names};" MATCHES ";${data_name};") + set(data_file + ${CMAKE_CURRENT_BINARY_DIR}/suites/test_suite_${data_name}.data) + set(dependency test_suite_bignum_generated_data) elseif(";${ecp_generated_data_names};" MATCHES ";${data_name};") set(data_file ${CMAKE_CURRENT_BINARY_DIR}/suites/test_suite_${data_name}.data) @@ -241,7 +246,11 @@ function(add_test_suite suite_name) else() set(data_file ${CMAKE_CURRENT_SOURCE_DIR}/suites/test_suite_${data_name}.data) - set(dependency test_suite_bignum_generated_data test_suite_ecp_generated_data test_suite_psa_generated_data) + set(dependency + test_suite_bignum_generated_data + test_suite_config_generated_data + test_suite_ecp_generated_data + test_suite_psa_generated_data) endif() add_custom_command( From 7f900690e08524a478e56f8b2f5646050bb925b0 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 12 Jun 2024 17:53:06 +0200 Subject: [PATCH 8/9] Update generate_config_tests.py Signed-off-by: Gilles Peskine --- framework | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/framework b/framework index 1893bfb1ee4b..558804797e61 160000 --- a/framework +++ b/framework @@ -1 +1 @@ -Subproject commit 1893bfb1ee4b1d971db38bf3ea1f8005fe3c8d9a +Subproject commit 558804797e617af23957bbe94a5e74af8ae83e38 From 3bf375cf255b58208d54373efe2360ef22b13f4d Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 1 Jul 2024 15:33:33 +0200 Subject: [PATCH 9/9] Update framework after merge of #28 Signed-off-by: Gilles Peskine --- framework | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/framework b/framework index 558804797e61..29e8dce54a10 160000 --- a/framework +++ b/framework @@ -1 +1 @@ -Subproject commit 558804797e617af23957bbe94a5e74af8ae83e38 +Subproject commit 29e8dce54a1041e22489f713cc8c44f700fafcec