From 469f7811fac98617e544aac2b1b2db56040c95db Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 29 Feb 2024 18:19:56 +0100 Subject: [PATCH 01/11] Require framework directory to exist when building The framework directory will be provided by a submodule. Signed-off-by: Gilles Peskine --- CMakeLists.txt | 2 ++ Makefile | 2 ++ library/Makefile | 1 + scripts/common.make | 2 ++ 4 files changed, 7 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5585c78fa7..154c84af64 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -277,6 +277,8 @@ if(LIB_INSTALL_DIR) set(CMAKE_INSTALL_LIBDIR "${LIB_INSTALL_DIR}") endif() +add_subdirectory(framework) + add_subdirectory(include) add_subdirectory(3rdparty) diff --git a/Makefile b/Makefile index 885948c112..2e41abaec4 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,8 @@ DESTDIR=/usr/local PREFIX=mbedtls_ PERL ?= perl +include framework/exported.make + .SILENT: .PHONY: all no_test programs lib tests install uninstall clean test check lcov apidoc apidoc_clean diff --git a/library/Makefile b/library/Makefile index d11a98df01..d5e4f64cea 100644 --- a/library/Makefile +++ b/library/Makefile @@ -1,3 +1,4 @@ +include ../framework/exported.make # Also see "include/mbedtls/mbedtls_config.h" diff --git a/scripts/common.make b/scripts/common.make index 2714bcd327..2da58d00e3 100644 --- a/scripts/common.make +++ b/scripts/common.make @@ -4,6 +4,8 @@ ifndef MBEDTLS_PATH MBEDTLS_PATH := .. endif +include $(MBEDTLS_PATH)/framework/exported.make + CFLAGS ?= -O2 WARNING_CFLAGS ?= -Wall -Wextra -Wformat=2 -Wno-format-nonliteral WARNING_CXXFLAGS ?= -Wall -Wextra -Wformat=2 -Wno-format-nonliteral From 456a54da8ef44c8b725b524567c08cffec6a7214 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 29 Feb 2024 18:38:12 +0100 Subject: [PATCH 02/11] Add framework submodule with makefiles Signed-off-by: Gilles Peskine --- .gitmodules | 3 +++ framework | 1 + 2 files changed, 4 insertions(+) create mode 100644 .gitmodules create mode 160000 framework diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000..4fb26b555e --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "framework"] + path = framework + url = https://github.com/Mbed-TLS/mbedtls-framework diff --git a/framework b/framework new file mode 160000 index 0000000000..2c29c56c10 --- /dev/null +++ b/framework @@ -0,0 +1 @@ +Subproject commit 2c29c56c1095044dd78d6ea5d804fe1f61f946bf From 1c13aa78c2938bebe3c8a7a7f2b0ca945ddbaebe Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 4 Mar 2024 11:06:56 +0100 Subject: [PATCH 03/11] Framework submodule: fix the libtestdriver1 build `make -C tests libtestdriver1` copies `library/Makefile` to `tests/libtestdriver1/library/Makefile`, where `../framework` does not point to the framework submodule. Signed-off-by: Gilles Peskine --- library/Makefile | 6 +++++- tests/Makefile | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/library/Makefile b/library/Makefile index d5e4f64cea..67f91f6b42 100644 --- a/library/Makefile +++ b/library/Makefile @@ -1,4 +1,8 @@ -include ../framework/exported.make +ifndef MBEDTLS_PATH +MBEDTLS_PATH := .. +endif + +include $(MBEDTLS_PATH)/framework/exported.make # Also see "include/mbedtls/mbedtls_config.h" diff --git a/tests/Makefile b/tests/Makefile index f82c267ac5..c2a0b84f07 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -208,6 +208,7 @@ define libtestdriver1_rewrite := s/\b(?=mbedtls_|psa_)/libtestdriver1_/g; endef +libtestdriver1.a: export MBEDTLS_PATH := $(patsubst ../..//%,/%,../../$(MBEDTLS_PATH)) libtestdriver1.a: # Copy the library and fake a 3rdparty Makefile include. rm -Rf ./libtestdriver1 From 2aa63ea48c1acb8b811aac167aa54d60941ee15a Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 4 Mar 2024 11:08:19 +0100 Subject: [PATCH 04/11] Support Git submodules Signed-off-by: Gilles Peskine --- tests/scripts/check_files.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/scripts/check_files.py b/tests/scripts/check_files.py index 65fbc9f070..5e4a7247ec 100755 --- a/tests/scripts/check_files.py +++ b/tests/scripts/check_files.py @@ -321,6 +321,7 @@ class TabIssueTracker(LineIssueTracker): ".make", ".pem", # some openssl dumps have tabs ".sln", + "/.gitmodules", "/Makefile", "/Makefile.inc", "/generate_visualc_files.pl", @@ -481,6 +482,12 @@ class IntegrityChecker: bytes_output = subprocess.check_output(['git', 'ls-files', '-z']) bytes_filepaths = bytes_output.split(b'\0')[:-1] ascii_filepaths = map(lambda fp: fp.decode('ascii'), bytes_filepaths) + # Filter out directories. Normally Git doesn't list directories + # (it only knows about the files inside them), but there is + # at least one case where 'git ls-files' includes a directory: + # submodules. Just skip submodules (and any other directories). + ascii_filepaths = [fp for fp in ascii_filepaths + if os.path.isfile(fp)] # Prepend './' to files in the top-level directory so that # something like `'/Makefile' in fp` matches in the top-level # directory as well as in subdirectories. From 8cbbc5f4e6cf3578c082fecfbdd68429f0689ffa Mon Sep 17 00:00:00 2001 From: David Horstmann Date: Mon, 4 Mar 2024 14:52:06 +0000 Subject: [PATCH 05/11] Tell ReadTheDocs to include framework submodule Signed-off-by: David Horstmann --- .readthedocs.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 72f126fa20..2b10f863f2 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -5,6 +5,11 @@ # Required version: 2 +# Include the framework submodule in the build +submodules: + include: + - framework + # Set the version of Python and other tools you might need build: os: ubuntu-20.04 From de047b09fe1f7c2ee878a0cc7b6f75f4cf429649 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 4 Mar 2024 11:51:31 +0100 Subject: [PATCH 06/11] Add docstrings to pacify pylint Signed-off-by: Gilles Peskine --- tests/scripts/check_files.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/scripts/check_files.py b/tests/scripts/check_files.py index 5e4a7247ec..a15a058f0e 100755 --- a/tests/scripts/check_files.py +++ b/tests/scripts/check_files.py @@ -468,6 +468,7 @@ class IntegrityChecker: ] def setup_logger(self, log_file, level=logging.INFO): + """Log to log_file if provided, or to stderr if None.""" self.logger = logging.getLogger() self.logger.setLevel(level) if log_file: @@ -479,6 +480,10 @@ class IntegrityChecker: @staticmethod def collect_files(): + """Return the list of files to check. + + These are the regular files commited into Git. + """ bytes_output = subprocess.check_output(['git', 'ls-files', '-z']) bytes_filepaths = bytes_output.split(b'\0')[:-1] ascii_filepaths = map(lambda fp: fp.decode('ascii'), bytes_filepaths) @@ -495,12 +500,17 @@ class IntegrityChecker: for fp in ascii_filepaths] def check_files(self): + """Check all files for all issues.""" for issue_to_check in self.issues_to_check: for filepath in self.collect_files(): if issue_to_check.should_check_file(filepath): issue_to_check.check_file_for_issue(filepath) def output_issues(self): + """Log the issues found and their locations. + + Return 1 if there were issues, 0 otherwise. + """ integrity_return_code = 0 for issue_to_check in self.issues_to_check: if issue_to_check.files_with_issues: From 0c3f0e998d584ac4ca01caa398bcf777ee802f5e Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 4 Mar 2024 15:54:54 +0100 Subject: [PATCH 07/11] Note the need to tell git to set up the submodule Signed-off-by: Gilles Peskine --- README.md | 4 ++++ framework | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2505d8fd9c..7d3894e8b2 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,10 @@ You need the following tools to build the library with the provided makefiles: * Microsoft Visual Studio 2013 or later (if using Visual Studio). * Doxygen 1.8.11 or later (if building the documentation; slightly older versions should work). +### Git usage + +The `development` branch and the `mbedtls-3.6` long-term support of Mbed TLS use a [Git submodule](https://git-scm.com/book/en/v2/Git-Tools-Submodules#_cloning_submodules) ([framework](https://github.com/Mbed-TLS/mbedtls-framework)). This is not needed to merely compile the library at a release tag. This is not needed to consume a release archive (zip or tar). + ### Generated source files in the development branch The source code of Mbed TLS includes some files that are automatically generated by scripts and whose content depends only on the Mbed TLS source, not on the platform or on the library configuration. These files are not included in the development branch of Mbed TLS, but the generated files are included in official releases. This section explains how to generate the missing files in the development branch. diff --git a/framework b/framework index 2c29c56c10..26ab40de58 160000 --- a/framework +++ b/framework @@ -1 +1 @@ -Subproject commit 2c29c56c1095044dd78d6ea5d804fe1f61f946bf +Subproject commit 26ab40de5887c872b5108981fece907104e8409c From f9bbe0de4c97ca532d052106bb471756e55742ae Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 4 Mar 2024 16:25:14 +0100 Subject: [PATCH 08/11] Show guidance if the framework is not found Signed-off-by: Gilles Peskine --- CMakeLists.txt | 3 +++ library/Makefile | 8 ++++++++ scripts/common.make | 10 ++++++++++ 3 files changed, 21 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 154c84af64..174a99f6a1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -277,6 +277,9 @@ if(LIB_INSTALL_DIR) set(CMAKE_INSTALL_LIBDIR "${LIB_INSTALL_DIR}") endif() +if (NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/framework/CMakeLists.txt") + message(FATAL_ERROR "${CMAKE_CURRENT_SOURCE_DIR}/framework/CMakeLists.txt not found. Run `git submodule update --init` from the source tree to fetch the submodule contents.") +endif() add_subdirectory(framework) add_subdirectory(include) diff --git a/library/Makefile b/library/Makefile index 67f91f6b42..2f42124742 100644 --- a/library/Makefile +++ b/library/Makefile @@ -2,6 +2,14 @@ ifndef MBEDTLS_PATH MBEDTLS_PATH := .. endif +ifeq (,$(wildcard $(MBEDTLS_PATH)/framework/exported.make)) +define error_message +$(MBEDTLS_PATH)/framework/exported.make not found. +Run `git submodule update --init` to fetch the submodule contents. +This is a fatal error +endef +$(error $(error_message)) +endif include $(MBEDTLS_PATH)/framework/exported.make # Also see "include/mbedtls/mbedtls_config.h" diff --git a/scripts/common.make b/scripts/common.make index 2da58d00e3..9908a3c265 100644 --- a/scripts/common.make +++ b/scripts/common.make @@ -4,6 +4,16 @@ ifndef MBEDTLS_PATH MBEDTLS_PATH := .. endif +ifeq (,$(wildcard $(MBEDTLS_PATH)/framework/exported.make)) + # Use the define keyword to get a multi-line message. + # GNU make appends ". Stop.", so tweak the ending of our message accordingly. + define error_message +$(MBEDTLS_PATH)/framework/exported.make not found. +Run `git submodule update --init` to fetch the submodule contents. +This is a fatal error + endef + $(error $(error_message)) +endif include $(MBEDTLS_PATH)/framework/exported.make CFLAGS ?= -O2 From 48230e84cb36b0d413f28122b153094f0f04574e Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 4 Mar 2024 16:42:54 +0100 Subject: [PATCH 09/11] In library, with make, only require the framework for generated files This way, `make lib` will work in the absence of the framework, as long as generated files are present. Signed-off-by: Gilles Peskine --- Makefile | 14 +++++++++++++- library/Makefile | 25 +++++++++++++++---------- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 2e41abaec4..47a3895b8a 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,19 @@ DESTDIR=/usr/local PREFIX=mbedtls_ PERL ?= perl -include framework/exported.make +ifneq (,$(filter-out lib library/%,$(or $(MAKECMDGOALS),all))) + ifeq (,$(wildcard framework/exported.make)) + # Use the define keyword to get a multi-line message. + # GNU make appends ". Stop.", so tweak the ending of our message accordingly. + define error_message +$(MBEDTLS_PATH)/framework/exported.make not found. +Run `git submodule update --init` to fetch the submodule contents. +This is a fatal error + endef + $(error $(error_message)) + endif + include framework/exported.make +endif .SILENT: diff --git a/library/Makefile b/library/Makefile index 2f42124742..77cfe077ac 100644 --- a/library/Makefile +++ b/library/Makefile @@ -2,15 +2,25 @@ ifndef MBEDTLS_PATH MBEDTLS_PATH := .. endif -ifeq (,$(wildcard $(MBEDTLS_PATH)/framework/exported.make)) -define error_message +GENERATED_FILES := \ + error.c version_features.c \ + ssl_debug_helpers_generated.c \ + psa_crypto_driver_wrappers.h \ + psa_crypto_driver_wrappers_no_static.c + +ifneq ($(GENERATED_FILES),$(wildcard $(GENERATED_FILES))) + ifeq (,$(wildcard $(MBEDTLS_PATH)/framework/exported.make)) + # Use the define keyword to get a multi-line message. + # GNU make appends ". Stop.", so tweak the ending of our message accordingly. + define error_message $(MBEDTLS_PATH)/framework/exported.make not found. Run `git submodule update --init` to fetch the submodule contents. This is a fatal error -endef -$(error $(error_message)) + endef + $(error $(error_message)) + endif + include $(MBEDTLS_PATH)/framework/exported.make endif -include $(MBEDTLS_PATH)/framework/exported.make # Also see "include/mbedtls/mbedtls_config.h" @@ -327,11 +337,6 @@ libmbedcrypto.dll: $(OBJS_CRYPTO) $(CC) $(LOCAL_CFLAGS) $(CFLAGS) -o $@ -c $< .PHONY: generated_files -GENERATED_FILES = \ - error.c version_features.c \ - ssl_debug_helpers_generated.c \ - psa_crypto_driver_wrappers.h \ - psa_crypto_driver_wrappers_no_static.c generated_files: $(GENERATED_FILES) # See root Makefile From 4ba34c0d888b57db1f9470f8e90c0281c067cb0a Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 4 Mar 2024 16:57:58 +0100 Subject: [PATCH 10/11] Update submodule after PR merge Signed-off-by: Gilles Peskine --- framework | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/framework b/framework index 26ab40de58..750634d3a5 160000 --- a/framework +++ b/framework @@ -1 +1 @@ -Subproject commit 26ab40de5887c872b5108981fece907104e8409c +Subproject commit 750634d3a51eb9d61b59fd5d801546927c946588 From 93b282232fc777807f21cb7db8804759989bc2c7 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 13 Mar 2024 13:08:57 +0100 Subject: [PATCH 11/11] missing word Signed-off-by: Gilles Peskine --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7d3894e8b2..8bb0b60a40 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ You need the following tools to build the library with the provided makefiles: ### Git usage -The `development` branch and the `mbedtls-3.6` long-term support of Mbed TLS use a [Git submodule](https://git-scm.com/book/en/v2/Git-Tools-Submodules#_cloning_submodules) ([framework](https://github.com/Mbed-TLS/mbedtls-framework)). This is not needed to merely compile the library at a release tag. This is not needed to consume a release archive (zip or tar). +The `development` branch and the `mbedtls-3.6` long-term support branch of Mbed TLS use a [Git submodule](https://git-scm.com/book/en/v2/Git-Tools-Submodules#_cloning_submodules) ([framework](https://github.com/Mbed-TLS/mbedtls-framework)). This is not needed to merely compile the library at a release tag. This is not needed to consume a release archive (zip or tar). ### Generated source files in the development branch