Merge branch 'feat/move-cjson-to-root-dep' into 'master'

feat: support moving idf components to component registry

Closes PACMAN-1102

See merge request espressif/esp-idf!39704

.gitlab/ci/rules.yml

@@ -86,6 +86,7 @@
   - "tools/idf_tools.py"
   - "tools/test_idf_tools/**/*"
   - "tools/install_util.py"
+  - "tools/idf_extra_components.yml"
 
   - "tools/export_utils/utils.py"
   - "tools/export_utils/shell_types.py"

docs/en/api-guides/tools/idf-tools.rst

@@ -45,6 +45,8 @@ Inside the ``IDF_TOOLS_PATH`` directory, the tools installation scripts create t
 - ``python_env`` —  not related to the tools; virtual Python environments are installed in the sub-directories. Note that the Python environment directory can be placed elsewhere by setting the ``IDF_PYTHON_ENV_PATH`` environment variable.
 
   - ``idf_version.txt`` — located within each specific Python environment sub-directory under ``python_env``, this file records the ESP-IDF version corresponding to that environment. The version is stored in a format like ``5.3`` to represent ESP-IDF version ``v5.3``.
+
+- ``root_managed_components`` — directory managed by ``idf-component-manager`` for components installed globally.
 - ``espidf.constraints.*.txt`` — one constraint file for each ESP-IDF release containing Python package version requirements.
 
 GitHub Assets Mirror

tools/ci/check_public_headers.py

@@ -336,6 +336,13 @@ class PublicHeaderChecker:
     # Get compilation data from an example to list all public header files
     def list_public_headers(self, ignore_dirs: list, ignore_files: list | set, only_dir: str | None = None) -> None:
         idf_path = self.idf_path
+        # idf_path = os.getenv('IDF_PATH')
+        if idf_path is None:
+            raise RuntimeError("Environment variable 'IDF_PATH' wasn't set.")
+
+        idf_tools_path = os.getenv('IDF_TOOLS_PATH') or os.path.expanduser(os.path.join('~', '.espressif'))
+        idf_root_dep_path = os.path.join(idf_tools_path, 'root_managed_components')
+
         project_dir = os.path.join(idf_path, 'examples', 'get-started', 'blink')
         sdkconfig = os.path.join(self.build_dir, 'sdkconfig')
         if self.libc_type == 'newlib':
@@ -398,7 +405,11 @@ class PublicHeaderChecker:
             if os.path.relpath(d, idf_path).startswith(tuple(ignore_dirs)):
                 self.log(f'{d} - directory ignored')
                 continue
-            for root, dirnames, filenames in os.walk(d):
+            for root, _, filenames in os.walk(d):
+                if root.startswith(idf_root_dep_path):
+                    self.log(f'{root} - directory ignored (inside IDF_TOOLS_PATH/root_managed_components)')
+                    continue
+
                 for filename in fnmatch.filter(filenames, '*.h'):
                     all_include_files.append(os.path.join(root, filename))
         self.main_c = main_c

tools/cmake/build.cmake

@@ -725,6 +725,21 @@ macro(idf_build_process target)
         endif()
     endif()
 
+    idf_build_get_property(prefix __PREFIX)
+
+    file(GLOB root_dep_component_dirs
+        ${IDF_TOOLS_PATH}/root_managed_components/idf${IDF_VERSION_MAJOR}.${IDF_VERSION_MINOR}.${IDF_VERSION_PATCH}/*)
+    list(SORT root_dep_component_dirs)
+    foreach(component_dir ${root_dep_component_dirs})
+        # A potential component must be a directory
+        if(IS_DIRECTORY ${component_dir})
+            __component_dir_quick_check(is_component ${component_dir})
+            if(is_component)
+                __component_add(${component_dir} ${prefix} "idf_managed_components")
+            endif()
+        endif()
+    endforeach()
+
     # Perform early expansion of component CMakeLists.txt in CMake scripting mode.
     # It is here we retrieve the public and private requirements of each component.
     # It is also here we add the common component requirements to each component's

tools/cmake/idf.cmake

@@ -52,6 +52,13 @@ if(NOT __idf_env_set)
     include(prefix_map)
     include(openocd)
 
+    # ESP-IDF extra dependencies defined in tools/idf_extra_components.yml
+    if(WIN32)
+        set_default(IDF_TOOLS_PATH "$ENV{USERPROFILE}/.espressif")
+    else()
+        set_default(IDF_TOOLS_PATH "$ENV{HOME}/.espressif")
+    endif()
+
     __build_init("${idf_path}")
 
     # Check if IDF_ENV_FPGA environment is set

tools/cmake/scripts/component_get_requirements.cmake

@@ -131,6 +131,8 @@ foreach(__component_target ${__component_targets})
 
     if("${__component_source}" STREQUAL "idf_components")
         list(APPEND __TARGETS_IDF_COMPONENTS ${__component_target})
+    elseif("${__component_source}" STREQUAL "idf_managed_components")
+        list(APPEND __TARGETS_IDF_MANAGED_COMPONENTS ${__component_target})
     elseif("${__component_source}" STREQUAL "project_managed_components")
         list(APPEND __TARGETS_PROJECT_MANAGED_COMPONENTS ${__component_target})
     elseif("${__component_source}" STREQUAL "project_extra_components")
@@ -147,6 +149,7 @@ set(__sorted_component_targets "")
 foreach(__target IN LISTS __TARGETS_PROJECT_COMPONENTS
                           __TARGETS_PROJECT_EXTRA_COMPONENTS
                           __TARGETS_PROJECT_MANAGED_COMPONENTS
+                          __TARGETS_IDF_MANAGED_COMPONENTS
                           __TARGETS_IDF_COMPONENTS)
     __component_get_property(__component_name ${__target} COMPONENT_NAME)
     list(APPEND __sorted_component_targets ${__target})

tools/idf_extra_components.yml

@@ -0,0 +1,11 @@
+# This file defines extra dependencies for ESP-IDF
+# the dependencies defined here will be downloaded to
+# $IDF_TOOLS_PATH/root_managed_components
+# Each major.minor version of ESP-IDF can have its own subdirectory
+# For example, for ESP-IDF v6.0, the dependencies will be installed to
+# $IDF_TOOLS_PATH/root_managed_components/idf6.0
+
+# The syntax is defined in:
+# https://docs.espressif.com/projects/idf-component-manager/en/latest/reference/manifest_file.html#dependencies
+
+#dependencies:

tools/test_build_system/README.md

@@ -149,6 +149,20 @@ def test_target_guessing()
 def test_target_guessing()
 ```
 
+### `pytest.mark.revert_later` Marker
+
+This marker reverts all files to their original state after the test is finished. should pass a list of file paths (absolute or relative to `IDF_PATH`) to the marker. The files will be reverted even if the test fails.
+
+```python
+@pytest.mark.revert_later(['tools/idf_extra_components.yml'])
+def test_modify_file(idf_copy):
+    path = os.path.join(os.getenv('IDF_PATH'), 'tools', 'idf_extra_components.yml')
+    with open(path, 'a') as f:
+        f.write('# some changes\n')
+    # The changes to idf_extra_components.yml will be reverted after the test
+```
+
+
 ### Build snapshots
 
 `get_snapshot(list_of_globs)` function takes a list of glob expressions, finds the files matching these expressions, and returns a `Snapshot` instance. `Snapshot` instances record file names and their modification timestamps. Two `Snapshot` instances can be compared using `assert_same` and `assert_different` methods:

tools/test_build_system/conftest.py

@@ -347,3 +347,24 @@ def pytest_report_header(config: Config) -> str:
         return 'Testing ESP-IDF CMake-based build system v2'
     else:
         return 'Testing ESP-IDF CMake-based build system v1'
+
+
+@pytest.fixture(autouse=True)
+def revert_later(request: FixtureRequest) -> typing.Generator[None, None, None]:
+    origin_content_d: dict[str, str] = {}
+
+    _marker = request.node.get_closest_marker('revert_later')
+    if _marker:
+        for filename in _marker.args[0]:
+            if not os.path.isabs(filename):
+                filename = os.path.join(EXT_IDF_PATH, filename)
+
+            with open(filename, encoding='utf-8') as fr:
+                origin_content_d[filename] = fr.read()
+
+    yield
+
+    if origin_content_d:
+        for filename, content in origin_content_d.items():
+            with open(filename, 'w', encoding='utf-8') as fw:
+                fw.write(content)

tools/test_build_system/pytest.ini

@@ -14,6 +14,9 @@ junit_family = xunit1
 junit_logging = stdout
 junit_log_passing_tests = False
 
+filterwarnings =
+    ignore::pytest.PytestExperimentalApiWarning
+
 ## !! When adding new markers, don't forget to update also the tools\test_build_system\README.md !!
 markers =
     test_app_copy: specify relative path of the app to copy, and the prefix of the destination directory name
@@ -22,3 +25,4 @@ markers =
     force_temp_work_dir: force temporary folder as the working directory
     with_idf_components: automatically create/delete components under IDF_PATH
     buildv2_skip: mark the test to run only when the --buildv2 command line option is not used
+    revert_later: revert the files to the original state after the test. (list of files to revert should be provided as a parameter)

tools/test_build_system/test_component_manager.py

@@ -2,8 +2,11 @@
 # SPDX-License-Identifier: Apache-2.0
 import json
 import os.path
+import textwrap
 from pathlib import Path
 
+import pytest
+from test_build_system_helpers import EXT_IDF_PATH
 from test_build_system_helpers import IdfPyFunc
 from test_build_system_helpers import replace_in_file
 
@@ -199,3 +202,84 @@ class TestOptionalDependencyWithKconfig:
         data = json.load(open(test_app_copy / 'build' / 'project_description.json'))
         assert ['example__cmp'] == data['build_component_info']['foo']['priv_reqs']
         assert ['espressif__mdns'] == data['build_component_info']['foo']['reqs']
+
+
+@pytest.mark.revert_later(['tools/idf_extra_components.yml'])
+class TestIdfRootDependency:
+    def test_basic_build(self, idf_py: IdfPyFunc, test_app_copy: Path) -> None:
+        with open(os.path.join(EXT_IDF_PATH, 'tools', 'idf_extra_components.yml'), 'w') as fw:
+            fw.write(
+                textwrap.dedent("""
+            dependencies:
+              espressif/mdns: "*"
+            """)
+            )
+
+        replace_in_file(
+            (test_app_copy / 'main' / 'build_test_app.c'),
+            '// placeholder_before_main',
+            '#include "mdns.h"',
+        )
+
+        replace_in_file(
+            (test_app_copy / 'main' / 'CMakeLists.txt'),
+            '# placeholder_inside_idf_component_register',
+            'REQUIRES mdns',
+        )
+
+        idf_py('build')
+
+    def test_build_only_when_required(self, idf_py: IdfPyFunc, test_app_copy: Path) -> None:
+        with open(os.path.join(EXT_IDF_PATH, 'tools', 'idf_extra_components.yml'), 'w') as fw:
+            fw.write(
+                textwrap.dedent("""
+            dependencies:
+              espressif/mdns: "*"
+              example/cmp: "*"
+            """)
+            )
+
+        idf_py('reconfigure')
+
+        data = json.load(open(test_app_copy / 'build' / 'project_description.json'))
+        assert 'espressif__mdns' not in data['build_components']
+        assert 'example__cmp' not in data['build_components']
+
+        replace_in_file(
+            (test_app_copy / 'main' / 'CMakeLists.txt'),
+            '# placeholder_inside_idf_component_register',
+            'REQUIRES mdns',
+        )
+
+        idf_py('reconfigure')
+        data = json.load(open(test_app_copy / 'build' / 'project_description.json'))
+        assert 'espressif__mdns' in data['build_components']
+        assert 'example__cmp' not in data['build_components']
+
+    def test_cleanup_unused(self, idf_py: IdfPyFunc, test_app_copy: Path) -> None:
+        with open(os.path.join(EXT_IDF_PATH, 'tools', 'idf_extra_components.yml'), 'w') as fw:
+            fw.write(
+                textwrap.dedent("""
+            dependencies:
+              espressif/mdns: "*"
+            """)
+            )
+
+        idf_py('reconfigure')
+        data = json.load(open(test_app_copy / 'build' / 'project_description.json'))
+        assert 'espressif__mdns' in data['all_component_info']
+
+        with open(os.path.join(EXT_IDF_PATH, 'tools', 'idf_extra_components.yml'), 'w') as fw:
+            fw.write(
+                textwrap.dedent("""
+            dependencies:
+              espressif/led_strip: "*"
+              example/cmp: "*"
+            """)
+            )
+
+        idf_py('reconfigure')
+        data = json.load(open(test_app_copy / 'build' / 'project_description.json'))
+        assert 'espressif__led_strip' in data['all_component_info']
+        assert 'example__cmp' in data['all_component_info']
+        assert 'espressif__mdns' not in data['all_component_info']

tools/test_idf_py/test_idf_py.py

@@ -48,6 +48,11 @@ class TestWithoutExtensions(TestCase):
 
         super().setUpClass()
 
+    @classmethod
+    def tearDownClass(cls):
+        cls.env_patcher.stop()
+        super().tearDownClass()
+
 
 class TestExtensions(TestWithoutExtensions):
     @classmethod