Merge branch 'ci/windows_profiling' into 'master'

Profiling for Windows runners and enhancing the tests in the scope of time complexity

Closes IDF-14137

See merge request espressif/esp-idf!44163
This commit is contained in:
Jakub Kocka
2026-03-17 19:45:56 +08:00
13 changed files with 235 additions and 105 deletions
+5 -4
View File
@@ -87,13 +87,13 @@ test_tools_win:
- python "${SUBMODULE_FETCH_TOOL}" -s "all"
- cd ${IDF_PATH}\tools\test_build_system
- idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
- pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml=${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
- pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml=${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} --durations=10
pytest_build_system_win:
extends:
- .test_build_system_template_win
- .rules:labels:windows_pytest_build_system
parallel: 6
parallel: 10
needs:
- job: manual_gate
optional: true
@@ -111,6 +111,7 @@ pytest_build_system_win_minimal_cmake:
extends:
- .test_build_system_template_win
- .rules:labels:windows_pytest_build_system
parallel: 2
needs:
- job: manual_gate
optional: true
@@ -139,13 +140,13 @@ pytest_build_system_win_minimal_cmake:
- python "${SUBMODULE_FETCH_TOOL}" -s "all"
- cd ${IDF_PATH}\tools\test_build_system
- idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
- pytest -k cmake --junitxml=${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
- pytest -k cmake --junitxml=${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} --durations=10
pytest_buildv2_system_win:
extends:
- .test_build_system_template_win
- .rules:labels:buildv2
parallel: 2
parallel: 10
needs:
- job: manual_gate
optional: true
-1
View File
@@ -55,4 +55,3 @@ tools/templates/sample_component/main.c
tools/templates/sample_project/CMakeLists.txt
tools/templates/sample_project/main/CMakeLists.txt
tools/templates/sample_project/main/main.c
tools/test_build_system/buildv2_test_app/main/KConfig.projbuild
@@ -1,4 +0,0 @@
# Misspelled Kconfig file checks whether it is picked up by the build system
config FROM_MISSPELLED_KCONFIG
bool "From misspelled Kconfig"
default y
+101 -15
View File
@@ -21,6 +21,90 @@ from test_build_system_helpers import get_idf_build_env
from test_build_system_helpers import run_idf_py
def _get_git_submodule_paths(repo_path: Path) -> list[str]:
"""Get list of submodule paths from .gitmodules file."""
gitmodules = repo_path / '.gitmodules'
if not gitmodules.exists():
return []
submodule_paths = []
with open(gitmodules, encoding='utf-8') as f:
for line in f:
line = line.strip()
if line.startswith('path = '):
submodule_paths.append(line[7:]) # Remove 'path = ' prefix
return submodule_paths
def _create_idf_copy_via_worktree(path_from: Path, path_to: Path) -> str:
"""
Create IDF copy using git worktree (fast) + copying submodule directories.
Git worktree creates a fast checkout of tracked files, but submodules
appear as empty directories. We copy submodule content from the source
repo (which has them already checked out) instead of running git submodule
update (which can fail due to auth issues on CI).
"""
import uuid
timestamp = datetime.datetime.now().strftime('%H%M%S')
branch_name = f'test-worktree-{timestamp}-{uuid.uuid4().hex[:8]}'
logging.debug(f'creating git worktree at {path_to} (branch: {branch_name})')
subprocess.run(
['git', 'worktree', 'add', '-b', branch_name, str(path_to)], cwd=path_from, capture_output=True, check=True
)
# Copy submodule directories from source (they're already checked out there)
submodule_paths = _get_git_submodule_paths(path_from)
for submodule_rel_path in submodule_paths:
src_submodule = path_from / submodule_rel_path
dst_submodule = path_to / submodule_rel_path
# Only copy if source submodule exists and has content
if src_submodule.exists() and any(src_submodule.iterdir()):
logging.debug(f'copying submodule {submodule_rel_path}')
# Remove the empty directory created by worktree
if dst_submodule.exists():
shutil.rmtree(dst_submodule, ignore_errors=True)
# Copy the submodule content
shutil.copytree(src_submodule, dst_submodule, symlinks=True, ignore=shutil.ignore_patterns('.git'))
return branch_name
def _cleanup_worktree(path_from: Path, path_to: Path, branch_name: str) -> None:
"""Remove git worktree and its temporary branch."""
logging.debug(f'removing git worktree at {path_to}')
# Remove the worktree
subprocess.run(
['git', 'worktree', 'remove', '--force', str(path_to)],
cwd=path_from,
check=False, # Don't fail if already removed
)
# Delete the temporary branch
subprocess.run(
['git', 'branch', '-D', branch_name],
cwd=path_from,
check=False, # Don't fail if branch doesn't exist
)
def _create_idf_copy_via_shutil(path_from: Path, path_to: Path) -> None:
"""Create IDF copy using shutil.copytree (slower but always works)."""
# if the new directory inside the original directory,
# make sure not to go into recursion.
ignore = shutil.ignore_patterns(
path_to.name,
# also ignore the build directories which may be quite large
# plus ignore .git since it is causing trouble when removing on Windows
'**/build',
'.git',
)
logging.debug(f'copying {path_from} to {path_to} (shutil.copytree)')
shutil.copytree(path_from, path_to, ignore=ignore, symlinks=True)
# Pytest hook used to check if the test has passed or failed, from a fixture.
# Based on https://docs.pytest.org/en/latest/example/simple.html#making-test-result-information-available-in-fixtures
@pytest.hookimpl(tryfirst=True, hookwrapper=True)
@@ -199,23 +283,22 @@ def idf_copy(func_work_dir: Path, request: FixtureRequest) -> typing.Generator[P
if mark:
copy_to = mark.args[0]
path_from = EXT_IDF_PATH
path_from = Path(EXT_IDF_PATH)
path_to = func_work_dir / copy_to
# if the new directory inside the original directory,
# make sure not to go into recursion.
ignore = shutil.ignore_patterns(
path_to.name,
# also ignore the build directories which may be quite large
# plus ignore .git since it is causing trouble when removing on Windows
'**/build',
'.git',
)
logging.debug(f'copying {path_from} to {path_to}')
shutil.copytree(path_from, path_to, ignore=ignore, symlinks=True)
orig_idf_path = os.environ['IDF_PATH']
branch_name: str | None = None
# Try git worktree first (much faster), fall back to shutil.copytree
try:
branch_name = _create_idf_copy_via_worktree(path_from, path_to)
except (subprocess.CalledProcessError, FileNotFoundError, OSError) as e:
logging.debug(f'git worktree failed ({e}), falling back to shutil.copytree')
# Clean up any partial worktree before fallback
if path_to.exists():
shutil.rmtree(path_to, ignore_errors=True)
_create_idf_copy_via_shutil(path_from, path_to)
os.environ['IDF_PATH'] = str(path_to)
yield Path(path_to)
@@ -224,7 +307,10 @@ def idf_copy(func_work_dir: Path, request: FixtureRequest) -> typing.Generator[P
if should_clean_test_dir(request):
logging.debug(f'cleaning up work directory after a successful test: {path_to}')
shutil.rmtree(path_to, ignore_errors=True)
if branch_name:
_cleanup_worktree(path_from, path_to, branch_name)
else:
shutil.rmtree(path_to, ignore_errors=True)
@pytest.fixture(name='default_idf_env')
@@ -0,0 +1,9 @@
cmake_minimum_required(VERSION 3.22)
# placeholder_before_include_project_cmake
include($ENV{IDF_PATH}/tools/cmake/project.cmake)
# placeholder_after_include_project_cmake
project(kconfig_test_app)
@@ -0,0 +1,3 @@
Information about this test app can be found [here](../README.md#application-under-test).
This is a 1:1 copy of build_test_app, but with a malformed KConfig.projbuild file.
@@ -0,0 +1,5 @@
# placeholder_before_idf_component_register
idf_component_register(SRCS "kconfig_test_app.c"
# placeholder_inside_idf_component_register
)
@@ -0,0 +1,12 @@
/*
* SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
*
* SPDX-License-Identifier: Apache-2.0
*/
#include <stdio.h>
// placeholder_before_main
void app_main(void)
{
// placeholder_inside_main
}
+32 -29
View File
@@ -1,4 +1,4 @@
# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD
# SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
import logging
import os
@@ -17,6 +17,7 @@ from test_build_system_helpers import append_to_file
from test_build_system_helpers import file_contains
from test_build_system_helpers import get_idf_build_env
from test_build_system_helpers import replace_in_file
from test_build_system_helpers import run_cmake
from test_build_system_helpers import run_cmake_and_build
@@ -28,23 +29,22 @@ def assert_built(paths: list[str] | list[Path]) -> None:
def test_build_alternative_directories(idf_py: IdfPyFunc, func_work_dir: Path, test_app_copy: Path) -> None:
logging.info('Moving BUILD_DIR_BASE out of tree')
alt_build_dir = func_work_dir / 'alt_build'
try:
idf_py('-B', str(alt_build_dir), 'build')
assert os.listdir(alt_build_dir) != [], 'No files found in new build directory!'
default_build_dir = test_app_copy / 'build'
if default_build_dir.exists():
assert os.listdir(default_build_dir) == [], (
f'Some files were incorrectly put into the default build directory: {default_build_dir}'
)
except Exception:
raise
else:
shutil.rmtree(alt_build_dir)
# Use reconfigure instead of full build - we're testing directory placement, not compilation
idf_py('-B', str(alt_build_dir), 'reconfigure')
assert os.listdir(alt_build_dir) != [], 'No files found in new build directory!'
assert (alt_build_dir / 'CMakeCache.txt').exists(), 'CMakeCache.txt not found in alt build directory!'
default_build_dir = test_app_copy / 'build'
if default_build_dir.exists():
assert os.listdir(default_build_dir) == [], (
f'Some files were incorrectly put into the default build directory: {default_build_dir}'
)
shutil.rmtree(alt_build_dir)
logging.info('BUILD_DIR_BASE inside default build directory')
build_subdir_inside_build_dir = default_build_dir / 'subdirectory'
idf_py('-B', str(build_subdir_inside_build_dir), 'build')
idf_py('-B', str(build_subdir_inside_build_dir), 'reconfigure')
assert os.listdir(build_subdir_inside_build_dir) != [], 'No files found in new build directory!'
assert (build_subdir_inside_build_dir / 'CMakeCache.txt').exists(), 'CMakeCache.txt not found in subdirectory!'
@pytest.mark.usefixtures('test_app_copy')
@@ -84,30 +84,31 @@ def test_build_with_generator_makefile(idf_py: IdfPyFunc) -> None:
def test_build_with_cmake_and_idf_path_unset(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
# This test verifies CMake configuration works with various IDF_PATH setups.
# We use run_cmake (configure only) instead of full builds for speed.
idf_path = Path(os.environ['IDF_PATH'])
env = get_idf_build_env(idf_path)
env.pop('IDF_PATH')
build_dir = test_app_copy / 'build'
logging.info('Can build with IDF_PATH set via cmake cache not environment')
logging.info('Can configure with IDF_PATH set via cmake cache not environment')
replace_in_file('CMakeLists.txt', 'ENV{IDF_PATH}', '{IDF_PATH}')
run_cmake_and_build('-G', 'Ninja', '..', f'-DIDF_PATH={idf_path}', env=env)
assert_built(BOOTLOADER_BINS + APP_BINS + PARTITION_BIN)
idf_py('fullclean')
run_cmake('-G', 'Ninja', '..', f'-DIDF_PATH={idf_path}', env=env)
assert (build_dir / 'CMakeCache.txt').exists(), 'CMake configuration failed'
shutil.rmtree(build_dir)
logging.info('Can build with IDF_PATH unset and inferred by cmake when Kconfig needs it to be set')
# working with already changed CMakeLists.txt
logging.info('Can configure with IDF_PATH unset and inferred by cmake when Kconfig needs it to be set')
kconfig_file = test_app_copy / 'main' / 'Kconfig.projbuild'
kconfig_file.write_text('source "$IDF_PATH/examples/wifi/getting_started/station/main/Kconfig.projbuild"')
run_cmake_and_build('-G', 'Ninja', '..', f'-DIDF_PATH={idf_path}', env=env)
assert_built(BOOTLOADER_BINS + APP_BINS + PARTITION_BIN)
kconfig_file.unlink() # remove file to not affect following sub-test
idf_py('fullclean')
run_cmake('-G', 'Ninja', '..', f'-DIDF_PATH={idf_path}', env=env)
assert (build_dir / 'CMakeCache.txt').exists(), 'CMake configuration failed'
kconfig_file.unlink()
shutil.rmtree(build_dir)
logging.info('Can build with IDF_PATH unset and inferred by build system')
# replacing {IDF_PATH} not ENV{IDF_PATH} since CMakeLists.txt was already changed in this test
logging.info('Can configure with IDF_PATH unset and inferred by build system')
replace_in_file('CMakeLists.txt', '{IDF_PATH}', '{ci_idf_path}')
run_cmake_and_build('-G', 'Ninja', '-D', f'ci_idf_path={idf_path}', '..', env=env)
assert_built(BOOTLOADER_BINS + APP_BINS + PARTITION_BIN)
run_cmake('-G', 'Ninja', '-D', f'ci_idf_path={idf_path}', '..', env=env)
assert (build_dir / 'CMakeCache.txt').exists(), 'CMake configuration failed'
def test_build_skdconfig_phy_init_data(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
@@ -257,11 +258,13 @@ def test_build_cmake_executable_suffix(idf_py: IdfPyFunc, test_app_copy: Path) -
assert 'Project build complete' in ret.stdout, 'Build with CMAKE_EXECUTABLE_SUFFIX set failed'
@pytest.mark.test_app_copy('tools/test_build_system/kconfig_test_app')
def test_build_with_misspelled_kconfig(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
logging.info('idf.py can build with misspelled Kconfig file')
ret = idf_py('build')
assert " file should be named 'Kconfig.projbuild'" in ret.stderr, 'Misspelled Kconfig file should be detected'
assert_built(BOOTLOADER_BINS + APP_BINS + PARTITION_BIN)
kconfig_app_bins = ['build/kconfig_test_app.elf', 'build/kconfig_test_app.bin']
assert_built(BOOTLOADER_BINS + kconfig_app_bins + PARTITION_BIN)
with open(test_app_copy / 'sdkconfig') as f:
sdkconfig = f.read()
assert 'CONFIG_FROM_MISSPELLED_KCONFIG=y' in sdkconfig, (
+10 -9
View File
@@ -19,11 +19,10 @@ from test_build_system_helpers import run_cmake_and_build
from test_build_system_helpers import run_idf_py
# This test checks multiple targets in one test function. It would be better to have each target
# tested in a isolated test case, but that would mean doing idf_copy each time, and copying takes most of the time
# This test verifies ESP-IDF can be used as a library in custom CMake projects.
# We use cmake configure only (not full build) and test representative targets from each arch.
@pytest.mark.usefixtures('idf_copy')
def test_build_custom_cmake_project(test_app_copy: Path, request: pytest.FixtureRequest) -> None:
# Test is compatible with any target. Random targets in the list are selected for performance reasons
idf_path = Path(os.environ['IDF_PATH'])
is_buildv2 = request.config.getoption('buildv2', False)
if is_buildv2:
@@ -35,9 +34,11 @@ def test_build_custom_cmake_project(test_app_copy: Path, request: pytest.Fixture
base_cmake_args = ['-G', 'Ninja']
target_var = 'TARGET'
for target in ['esp32', 'esp32c2', 'esp32c3', 'esp32c6', 'esp32h2', 'esp32p4', 'esp32s2', 'esp32s3']:
logging.info(f'Test build ESP-IDF as a library to a custom CMake projects for {target}')
run_cmake_and_build(
# Test representative targets: Xtensa (esp32), RISC-V (esp32c3), and newest (esp32p4)
for target in ['esp32', 'esp32c3', 'esp32p4']:
logging.info(f'Test CMake configuration of ESP-IDF as a library for {target}')
# Use run_cmake (configure only) - compile_commands.json is generated during configure
run_cmake(
str(idf_as_lib_path),
*base_cmake_args,
'-DCMAKE_TOOLCHAIN_FILE={}'.format(idf_path / 'tools' / 'cmake' / f'toolchain-{target}.cmake'),
@@ -45,9 +46,9 @@ def test_build_custom_cmake_project(test_app_copy: Path, request: pytest.Fixture
)
assert file_contains((test_app_copy / 'build' / 'compile_commands.json'), '"command"')
shutil.rmtree(test_app_copy / 'build')
sdkconfig_path = idf_as_lib_path / 'sdkconfig'
if sdkconfig_path.exists():
os.remove(sdkconfig_path)
sdkconfig = idf_as_lib_path / 'sdkconfig'
if sdkconfig.exists():
sdkconfig.unlink()
@pytest.mark.skipif(
+17 -8
View File
@@ -1,4 +1,4 @@
# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
# SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
import logging
@@ -8,7 +8,10 @@ import shutil
from pathlib import Path
import pytest
from test_build_system_helpers import EnvDict, IdfPyFunc, append_to_file, replace_in_file
from test_build_system_helpers import EnvDict
from test_build_system_helpers import IdfPyFunc
from test_build_system_helpers import append_to_file
from test_build_system_helpers import replace_in_file
@pytest.mark.usefixtures('test_app_copy')
@@ -30,15 +33,21 @@ def test_partition_nearly_full_warning(idf_py: IdfPyFunc, test_app_copy: Path, d
logging.info('Warning is given if smallest partition is nearly full')
ret = idf_py('build')
# Build a first time to get the binary size and to check that no warning is issued.
assert 'partition is nearly full' not in ret.stdout, 'Warning for nearly full smallest partition was given when the condition is not fulfilled'
assert 'partition is nearly full' not in ret.stdout, (
'Warning for nearly full smallest partition was given when the condition is not fulfilled'
)
# Get the size of the binary, in KB. Convert it to next multiple of 4.
# The goal is to create an app partition which is slightly bigger than the binary itself
updated_file_size = int((os.stat(test_app_copy / 'build' / 'build_test_app.bin').st_size + 4095) / 4096) * 4
idf_path = Path(default_idf_env['IDF_PATH'])
shutil.copy2(idf_path / 'components' / 'partition_table' / 'partitions_singleapp.csv', test_app_copy / 'partitions.csv')
replace_in_file(test_app_copy / 'partitions.csv',
'factory, app, factory, , 1M',
f'factory, app, factory, , {updated_file_size}K')
(test_app_copy / 'sdkconfig').write_text('\n'.join(['CONFIG_PARTITION_TABLE_CUSTOM=y', 'CONFIG_FREERTOS_SMP=n']))
shutil.copy2(
idf_path / 'components' / 'partition_table' / 'partitions_singleapp.csv', test_app_copy / 'partitions.csv'
)
replace_in_file(
test_app_copy / 'partitions.csv',
'factory, app, factory, , 1M',
f'factory, app, factory, , {updated_file_size}K',
)
append_to_file(test_app_copy / 'sdkconfig', '\n'.join(['CONFIG_PARTITION_TABLE_CUSTOM=y', 'CONFIG_FREERTOS_SMP=n']))
ret = idf_py('build', check=False)
assert 'partition is nearly full' in ret.stdout
+41 -35
View File
@@ -1,6 +1,6 @@
#!/usr/bin/env python
#
# SPDX-FileCopyrightText: 2019-2025 Espressif Systems (Shanghai) CO LTD
# SPDX-FileCopyrightText: 2019-2026 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
import json
import os
@@ -50,48 +50,54 @@ class TestWithoutExtensions(TestCase):
class TestExtensions(TestWithoutExtensions):
def test_extension_loading(self):
@classmethod
def setUpClass(cls):
super().setUpClass()
# Create symlink once for all tests in this class
# Handle race conditions with parallel test execution (pytest-xdist)
try:
os.symlink(extension_path, link_path)
os.environ['IDF_EXTRA_ACTIONS_PATH'] = os.path.join(current_dir, 'extra_path')
output = subprocess.check_output([sys.executable, idf_py_path, '--help'], env=os.environ).decode(
'utf-8', 'ignore'
)
except FileExistsError:
# Another worker already created it - that's fine
pass
os.environ['IDF_EXTRA_ACTIONS_PATH'] = os.path.join(current_dir, 'extra_path')
self.assertIn('--test-extension-option', output)
self.assertIn('test_subcommand', output)
self.assertIn('--some-extension-option', output)
self.assertIn('extra_subcommand', output)
finally:
@classmethod
def tearDownClass(cls):
# Clean up symlink after all tests complete
# Use try/except to handle race conditions with parallel execution
try:
os.remove(link_path)
except FileNotFoundError:
# Another worker already removed it - that's fine
pass
super().tearDownClass()
def test_extension_loading(self):
output = subprocess.check_output([sys.executable, idf_py_path, '--help'], env=os.environ).decode(
'utf-8', 'ignore'
)
self.assertIn('--test-extension-option', output)
self.assertIn('test_subcommand', output)
self.assertIn('--some-extension-option', output)
self.assertIn('extra_subcommand', output)
def test_extension_execution(self):
try:
os.symlink(extension_path, link_path)
os.environ['IDF_EXTRA_ACTIONS_PATH'] = ';'.join([os.path.join(current_dir, 'extra_path')])
output = subprocess.check_output(
[sys.executable, idf_py_path, '--some-extension-option=awesome', 'test_subcommand', 'extra_subcommand'],
env=os.environ,
).decode('utf-8', 'ignore')
self.assertIn('!!! From some global callback: awesome', output)
self.assertIn('!!! From some subcommand', output)
self.assertIn('!!! From test global callback: test', output)
self.assertIn('!!! From some subcommand', output)
finally:
os.remove(link_path)
output = subprocess.check_output(
[sys.executable, idf_py_path, '--some-extension-option=awesome', 'test_subcommand', 'extra_subcommand'],
env=os.environ,
).decode('utf-8', 'ignore')
self.assertIn('!!! From some global callback: awesome', output)
self.assertIn('!!! From some subcommand', output)
self.assertIn('!!! From test global callback: test', output)
self.assertIn('!!! From some subcommand', output)
def test_hidden_commands(self):
try:
os.symlink(extension_path, link_path)
os.environ['IDF_EXTRA_ACTIONS_PATH'] = ';'.join([os.path.join(current_dir, 'extra_path')])
output = subprocess.check_output([sys.executable, idf_py_path, '--help'], env=os.environ).decode(
'utf-8', 'ignore'
)
self.assertIn('test_subcommand', output)
self.assertNotIn('hidden_one', output)
finally:
os.remove(link_path)
output = subprocess.check_output([sys.executable, idf_py_path, '--help'], env=os.environ).decode(
'utf-8', 'ignore'
)
self.assertIn('test_subcommand', output)
self.assertNotIn('hidden_one', output)
class TestDependencyManagement(TestWithoutExtensions):