From c360d8be98729963169152fd512156638456264b Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Fri, 29 Aug 2025 11:51:27 +0200 Subject: [PATCH] feat: support moving idf components to component registry and mark it as root dependency --- .gitlab/ci/rules.yml | 1 + docs/en/api-guides/tools/idf-tools.rst | 2 + tools/ci/check_public_headers.py | 13 ++- tools/cmake/build.cmake | 15 ++++ tools/cmake/idf.cmake | 7 ++ .../scripts/component_get_requirements.cmake | 3 + tools/idf_extra_components.yml | 11 +++ tools/test_build_system/README.md | 14 ++++ tools/test_build_system/conftest.py | 21 +++++ tools/test_build_system/pytest.ini | 4 + .../test_component_manager.py | 84 +++++++++++++++++++ 11 files changed, 174 insertions(+), 1 deletion(-) create mode 100644 tools/idf_extra_components.yml diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index 335efc0c92..659eedea2b 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -86,6 +86,7 @@ - "tools/idf_tools.py" - "tools/test_idf_tools/**/*" - "tools/install_util.py" + - "tools/idf_extra_components.yml" - "tools/export_utils/utils.py" - "tools/export_utils/shell_types.py" diff --git a/docs/en/api-guides/tools/idf-tools.rst b/docs/en/api-guides/tools/idf-tools.rst index 6c167a1fd3..992f359f34 100644 --- a/docs/en/api-guides/tools/idf-tools.rst +++ b/docs/en/api-guides/tools/idf-tools.rst @@ -45,6 +45,8 @@ Inside the ``IDF_TOOLS_PATH`` directory, the tools installation scripts create t - ``python_env`` — not related to the tools; virtual Python environments are installed in the sub-directories. Note that the Python environment directory can be placed elsewhere by setting the ``IDF_PYTHON_ENV_PATH`` environment variable. - ``idf_version.txt`` — located within each specific Python environment sub-directory under ``python_env``, this file records the ESP-IDF version corresponding to that environment. The version is stored in a format like ``5.3`` to represent ESP-IDF version ``v5.3``. + +- ``root_managed_components`` — directory managed by ``idf-component-manager`` for components installed globally. - ``espidf.constraints.*.txt`` — one constraint file for each ESP-IDF release containing Python package version requirements. GitHub Assets Mirror diff --git a/tools/ci/check_public_headers.py b/tools/ci/check_public_headers.py index 4931b68fc1..24c0f702df 100644 --- a/tools/ci/check_public_headers.py +++ b/tools/ci/check_public_headers.py @@ -336,6 +336,13 @@ class PublicHeaderChecker: # Get compilation data from an example to list all public header files def list_public_headers(self, ignore_dirs: list, ignore_files: list | set, only_dir: str | None = None) -> None: idf_path = self.idf_path + # idf_path = os.getenv('IDF_PATH') + if idf_path is None: + raise RuntimeError("Environment variable 'IDF_PATH' wasn't set.") + + idf_tools_path = os.getenv('IDF_TOOLS_PATH') or os.path.expanduser(os.path.join('~', '.espressif')) + idf_root_dep_path = os.path.join(idf_tools_path, 'root_managed_components') + project_dir = os.path.join(idf_path, 'examples', 'get-started', 'blink') sdkconfig = os.path.join(self.build_dir, 'sdkconfig') if self.libc_type == 'newlib': @@ -398,7 +405,11 @@ class PublicHeaderChecker: if os.path.relpath(d, idf_path).startswith(tuple(ignore_dirs)): self.log(f'{d} - directory ignored') continue - for root, dirnames, filenames in os.walk(d): + for root, _, filenames in os.walk(d): + if root.startswith(idf_root_dep_path): + self.log(f'{root} - directory ignored (inside IDF_TOOLS_PATH/root_managed_components)') + continue + for filename in fnmatch.filter(filenames, '*.h'): all_include_files.append(os.path.join(root, filename)) self.main_c = main_c diff --git a/tools/cmake/build.cmake b/tools/cmake/build.cmake index 191e622833..aaac726b19 100644 --- a/tools/cmake/build.cmake +++ b/tools/cmake/build.cmake @@ -725,6 +725,21 @@ macro(idf_build_process target) endif() endif() + idf_build_get_property(prefix __PREFIX) + + file(GLOB root_dep_component_dirs + ${IDF_TOOLS_PATH}/root_managed_components/idf${IDF_VERSION_MAJOR}.${IDF_VERSION_MINOR}.${IDF_VERSION_PATCH}/*) + list(SORT root_dep_component_dirs) + foreach(component_dir ${root_dep_component_dirs}) + # A potential component must be a directory + if(IS_DIRECTORY ${component_dir}) + __component_dir_quick_check(is_component ${component_dir}) + if(is_component) + __component_add(${component_dir} ${prefix} "idf_managed_components") + endif() + endif() + endforeach() + # Perform early expansion of component CMakeLists.txt in CMake scripting mode. # It is here we retrieve the public and private requirements of each component. # It is also here we add the common component requirements to each component's diff --git a/tools/cmake/idf.cmake b/tools/cmake/idf.cmake index 384401cd76..0660049079 100644 --- a/tools/cmake/idf.cmake +++ b/tools/cmake/idf.cmake @@ -52,6 +52,13 @@ if(NOT __idf_env_set) include(prefix_map) include(openocd) + # ESP-IDF extra dependencies defined in tools/idf_extra_components.yml + if(WIN32) + set_default(IDF_TOOLS_PATH "$ENV{USERPROFILE}/.espressif") + else() + set_default(IDF_TOOLS_PATH "$ENV{HOME}/.espressif") + endif() + __build_init("${idf_path}") # Check if IDF_ENV_FPGA environment is set diff --git a/tools/cmake/scripts/component_get_requirements.cmake b/tools/cmake/scripts/component_get_requirements.cmake index bc94112e52..f4f4079c13 100644 --- a/tools/cmake/scripts/component_get_requirements.cmake +++ b/tools/cmake/scripts/component_get_requirements.cmake @@ -131,6 +131,8 @@ foreach(__component_target ${__component_targets}) if("${__component_source}" STREQUAL "idf_components") list(APPEND __TARGETS_IDF_COMPONENTS ${__component_target}) + elseif("${__component_source}" STREQUAL "idf_managed_components") + list(APPEND __TARGETS_IDF_MANAGED_COMPONENTS ${__component_target}) elseif("${__component_source}" STREQUAL "project_managed_components") list(APPEND __TARGETS_PROJECT_MANAGED_COMPONENTS ${__component_target}) elseif("${__component_source}" STREQUAL "project_extra_components") @@ -147,6 +149,7 @@ set(__sorted_component_targets "") foreach(__target IN LISTS __TARGETS_PROJECT_COMPONENTS __TARGETS_PROJECT_EXTRA_COMPONENTS __TARGETS_PROJECT_MANAGED_COMPONENTS + __TARGETS_IDF_MANAGED_COMPONENTS __TARGETS_IDF_COMPONENTS) __component_get_property(__component_name ${__target} COMPONENT_NAME) list(APPEND __sorted_component_targets ${__target}) diff --git a/tools/idf_extra_components.yml b/tools/idf_extra_components.yml new file mode 100644 index 0000000000..c851252cb4 --- /dev/null +++ b/tools/idf_extra_components.yml @@ -0,0 +1,11 @@ +# This file defines extra dependencies for ESP-IDF +# the dependencies defined here will be downloaded to +# $IDF_TOOLS_PATH/root_managed_components +# Each major.minor version of ESP-IDF can have its own subdirectory +# For example, for ESP-IDF v6.0, the dependencies will be installed to +# $IDF_TOOLS_PATH/root_managed_components/idf6.0 + +# The syntax is defined in: +# https://docs.espressif.com/projects/idf-component-manager/en/latest/reference/manifest_file.html#dependencies + +#dependencies: diff --git a/tools/test_build_system/README.md b/tools/test_build_system/README.md index 0bdf25cc35..c41d92613a 100644 --- a/tools/test_build_system/README.md +++ b/tools/test_build_system/README.md @@ -149,6 +149,20 @@ def test_target_guessing() def test_target_guessing() ``` +### `pytest.mark.revert_later` Marker + +This marker reverts all files to their original state after the test is finished. should pass a list of file paths (absolute or relative to `IDF_PATH`) to the marker. The files will be reverted even if the test fails. + +```python +@pytest.mark.revert_later(['tools/idf_extra_components.yml']) +def test_modify_file(idf_copy): + path = os.path.join(os.getenv('IDF_PATH'), 'tools', 'idf_extra_components.yml') + with open(path, 'a') as f: + f.write('# some changes\n') + # The changes to idf_extra_components.yml will be reverted after the test +``` + + ### Build snapshots `get_snapshot(list_of_globs)` function takes a list of glob expressions, finds the files matching these expressions, and returns a `Snapshot` instance. `Snapshot` instances record file names and their modification timestamps. Two `Snapshot` instances can be compared using `assert_same` and `assert_different` methods: diff --git a/tools/test_build_system/conftest.py b/tools/test_build_system/conftest.py index 1640e84774..fa9e9cb815 100644 --- a/tools/test_build_system/conftest.py +++ b/tools/test_build_system/conftest.py @@ -347,3 +347,24 @@ def pytest_report_header(config: Config) -> str: return 'Testing ESP-IDF CMake-based build system v2' else: return 'Testing ESP-IDF CMake-based build system v1' + + +@pytest.fixture(autouse=True) +def revert_later(request: FixtureRequest) -> typing.Generator[None, None, None]: + origin_content_d: dict[str, str] = {} + + _marker = request.node.get_closest_marker('revert_later') + if _marker: + for filename in _marker.args[0]: + if not os.path.isabs(filename): + filename = os.path.join(EXT_IDF_PATH, filename) + + with open(filename, encoding='utf-8') as fr: + origin_content_d[filename] = fr.read() + + yield + + if origin_content_d: + for filename, content in origin_content_d.items(): + with open(filename, 'w', encoding='utf-8') as fw: + fw.write(content) diff --git a/tools/test_build_system/pytest.ini b/tools/test_build_system/pytest.ini index 0e6a45443b..580d271fa3 100644 --- a/tools/test_build_system/pytest.ini +++ b/tools/test_build_system/pytest.ini @@ -14,6 +14,9 @@ junit_family = xunit1 junit_logging = stdout junit_log_passing_tests = False +filterwarnings = + ignore::pytest.PytestExperimentalApiWarning + ## !! When adding new markers, don't forget to update also the tools\test_build_system\README.md !! markers = test_app_copy: specify relative path of the app to copy, and the prefix of the destination directory name @@ -22,3 +25,4 @@ markers = force_temp_work_dir: force temporary folder as the working directory with_idf_components: automatically create/delete components under IDF_PATH buildv2_skip: mark the test to run only when the --buildv2 command line option is not used + revert_later: revert the files to the original state after the test. (list of files to revert should be provided as a parameter) diff --git a/tools/test_build_system/test_component_manager.py b/tools/test_build_system/test_component_manager.py index be9d27f9af..9848a733d6 100644 --- a/tools/test_build_system/test_component_manager.py +++ b/tools/test_build_system/test_component_manager.py @@ -2,8 +2,11 @@ # SPDX-License-Identifier: Apache-2.0 import json import os.path +import textwrap from pathlib import Path +import pytest +from test_build_system_helpers import EXT_IDF_PATH from test_build_system_helpers import IdfPyFunc from test_build_system_helpers import replace_in_file @@ -199,3 +202,84 @@ class TestOptionalDependencyWithKconfig: data = json.load(open(test_app_copy / 'build' / 'project_description.json')) assert ['example__cmp'] == data['build_component_info']['foo']['priv_reqs'] assert ['espressif__mdns'] == data['build_component_info']['foo']['reqs'] + + +@pytest.mark.revert_later(['tools/idf_extra_components.yml']) +class TestIdfRootDependency: + def test_basic_build(self, idf_py: IdfPyFunc, test_app_copy: Path) -> None: + with open(os.path.join(EXT_IDF_PATH, 'tools', 'idf_extra_components.yml'), 'w') as fw: + fw.write( + textwrap.dedent(""" + dependencies: + espressif/mdns: "*" + """) + ) + + replace_in_file( + (test_app_copy / 'main' / 'build_test_app.c'), + '// placeholder_before_main', + '#include "mdns.h"', + ) + + replace_in_file( + (test_app_copy / 'main' / 'CMakeLists.txt'), + '# placeholder_inside_idf_component_register', + 'REQUIRES mdns', + ) + + idf_py('build') + + def test_build_only_when_required(self, idf_py: IdfPyFunc, test_app_copy: Path) -> None: + with open(os.path.join(EXT_IDF_PATH, 'tools', 'idf_extra_components.yml'), 'w') as fw: + fw.write( + textwrap.dedent(""" + dependencies: + espressif/mdns: "*" + example/cmp: "*" + """) + ) + + idf_py('reconfigure') + + data = json.load(open(test_app_copy / 'build' / 'project_description.json')) + assert 'espressif__mdns' not in data['build_components'] + assert 'example__cmp' not in data['build_components'] + + replace_in_file( + (test_app_copy / 'main' / 'CMakeLists.txt'), + '# placeholder_inside_idf_component_register', + 'REQUIRES mdns', + ) + + idf_py('reconfigure') + data = json.load(open(test_app_copy / 'build' / 'project_description.json')) + assert 'espressif__mdns' in data['build_components'] + assert 'example__cmp' not in data['build_components'] + + def test_cleanup_unused(self, idf_py: IdfPyFunc, test_app_copy: Path) -> None: + with open(os.path.join(EXT_IDF_PATH, 'tools', 'idf_extra_components.yml'), 'w') as fw: + fw.write( + textwrap.dedent(""" + dependencies: + espressif/mdns: "*" + """) + ) + + idf_py('reconfigure') + data = json.load(open(test_app_copy / 'build' / 'project_description.json')) + assert 'espressif__mdns' in data['all_component_info'] + + with open(os.path.join(EXT_IDF_PATH, 'tools', 'idf_extra_components.yml'), 'w') as fw: + fw.write( + textwrap.dedent(""" + dependencies: + espressif/led_strip: "*" + example/cmp: "*" + """) + ) + + idf_py('reconfigure') + data = json.load(open(test_app_copy / 'build' / 'project_description.json')) + assert 'espressif__led_strip' in data['all_component_info'] + assert 'example__cmp' in data['all_component_info'] + assert 'espressif__mdns' not in data['all_component_info']