From f00fbc88a42249dc5393c1662af87ba80ef3ce7c Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Mon, 2 Feb 2026 12:28:40 +0100 Subject: [PATCH 1/2] ci: set build status as fail when upload failed --- tools/ci/idf_ci_local/app.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tools/ci/idf_ci_local/app.py b/tools/ci/idf_ci_local/app.py index 0a0f3c1f62..a0eaf2b7a2 100644 --- a/tools/ci/idf_ci_local/app.py +++ b/tools/ci/idf_ci_local/app.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2024-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import os import subprocess @@ -8,6 +8,7 @@ import typing as t from dynamic_pipelines.constants import BINARY_SIZE_METRIC_NAME from idf_build_apps import App from idf_build_apps import CMakeApp +from idf_build_apps.constants import BuildStatus from idf_build_apps.utils import rmdir from idf_ci_utils import idf_relpath @@ -29,7 +30,7 @@ class IdfCMakeApp(CMakeApp): # only upload in CI if os.getenv('CI_JOB_ID'): - subprocess.run( + result = subprocess.run( [ 'idf-ci', 'gitlab', @@ -39,6 +40,10 @@ class IdfCMakeApp(CMakeApp): stdout=sys.stdout, stderr=sys.stderr, ) + if result.returncode != 0: + self.build_status = BuildStatus.FAILED + self.build_comment = 'Failed to upload artifacts' + rmdir( self.build_path, exclude_file_patterns=['build_log.txt', 'size*.json'], From 9ecd3ecad4beb062efbf3fc63c38641b27b78bc8 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Mon, 2 Feb 2026 10:56:54 +0100 Subject: [PATCH 2/2] ci: apply idf-ci 1.x --- .gitlab/ci/common.yml | 2 +- .idf_ci.toml | 51 +++++++++++-------- .pre-commit-config.yaml | 2 +- conftest.py | 36 ++++++++----- .../templates/.dynamic_jobs.yml | 1 - tools/ci/idf_ci_local/app.py | 2 + tools/requirements/requirements.ci.txt | 2 +- .../mmu_page_size/pytest_mmu_page_size.py | 21 +++----- .../pytest_unicore_bootloader.py | 21 +++----- 9 files changed, 71 insertions(+), 67 deletions(-) diff --git a/.gitlab/ci/common.yml b/.gitlab/ci/common.yml index 1bc00a65f8..cf30ab7651 100644 --- a/.gitlab/ci/common.yml +++ b/.gitlab/ci/common.yml @@ -62,7 +62,7 @@ variables: # Set this variable to the branch of idf-constraints repo in order to test a custom Python constraint file. The # branch name must be without the remote part ("origin/"). Keep the variable empty in order to use the constraint # file from https://dl.espressif.com/dl/esp-idf. - CI_PYTHON_CONSTRAINT_BRANCH: "" + CI_PYTHON_CONSTRAINT_BRANCH: "feat/update-idf-ci-version" # Update the filename for a specific ESP-IDF release. It is used only with CI_PYTHON_CONSTRAINT_BRANCH. CI_PYTHON_CONSTRAINT_FILE: "espidf.constraints.v6.1.txt" diff --git a/.idf_ci.toml b/.idf_ci.toml index e6e5f000c3..e09e72ff73 100644 --- a/.idf_ci.toml +++ b/.idf_ci.toml @@ -50,57 +50,64 @@ include: - tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml """ -[gitlab.artifacts.s3.debug] +[gitlab.artifacts.s3] +enable = true + +[gitlab.artifacts.s3.configs.debug] bucket = "idf-artifacts" +zip_first = true +build_dir_pattern = "**/build*/" patterns = [ - '**/build*/bootloader/*.map', - '**/build*/bootloader/*.elf', - '**/build*/*.map', - '**/build*/*.elf', + 'bootloader/*.map', + 'bootloader/*.elf', + '*.map', + '*.elf', # customized - '**/build*/esp_tee/*.map', - '**/build*/esp_tee/*.elf', - '**/build*/gdbinit/*', + 'esp_tee/*.map', + 'esp_tee/*.elf', + 'gdbinit/*', ] if_clause = 'CI_JOB_GROUP_NAME != "build_non_test_related_apps"' -[gitlab.artifacts.s3.flash] +[gitlab.artifacts.s3.configs.flash] bucket = "idf-artifacts" +zip_first = true +build_dir_pattern = "**/build*/" patterns = [ - '**/build*/bootloader/*.bin', - '**/build*/*.bin', - '**/build*/partition_table/*.bin', - '**/build*/flasher_args.json', - '**/build*/flash_project_args', - '**/build*/config/sdkconfig.json', - '**/build*/sdkconfig', - '**/build*/project_description.json', + 'bootloader/*.bin', + '*.bin', + 'partition_table/*.bin', + 'flasher_args.json', + 'flash_project_args', + 'config/sdkconfig.json', + 'sdkconfig', + 'project_description.json', # customized - '**/build*/esp_tee/*.bin', + 'esp_tee/*.bin', ] if_clause = 'CI_JOB_GROUP_NAME != "build_non_test_related_apps"' -[gitlab.artifacts.s3.log] +[gitlab.artifacts.s3.configs.log] bucket = "idf-artifacts" patterns = [ '**/build*/build_log.txt', '**/build*/size*.json', ] -[gitlab.artifacts.s3.junit] +[gitlab.artifacts.s3.configs.junit] bucket = "idf-artifacts" patterns = [ '**/XUNIT_RESULT_*.xml', '**/build_summary_*.xml', ] -[gitlab.artifacts.s3.env] +[gitlab.artifacts.s3.configs.env] bucket = "idf-artifacts" patterns = [ '**/pipeline.env', ] -[gitlab.artifacts.s3.longterm] +[gitlab.artifacts.s3.configs.longterm] bucket = "longterm" if_clause = 'CI_COMMIT_REF_NAME == "master"' patterns = [ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7cbc826751..6bb22f4da7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -233,7 +233,7 @@ repos: - id: check-kconfig-files - id: check-deprecated-kconfig-options - repo: https://github.com/espressif/idf-ci - rev: v1.0.0b4 + rev: v1.0.0 hooks: - id: check-tests-missing-config files: 'pytest_.*\.py$|sdkconfig(\..*)?$' diff --git a/conftest.py b/conftest.py index 91cd649514..b4229026ea 100644 --- a/conftest.py +++ b/conftest.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2021-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 # pylint: disable=W0621 # redefined-outer-name # @@ -36,6 +36,7 @@ from _pytest.config import Config from _pytest.fixtures import FixtureRequest from idf_ci import PytestCase from idf_ci.idf_pytest import IDF_CI_PYTEST_CASE_KEY +from idf_ci_utils import IDF_PATH from idf_ci_utils import idf_relpath from idf_pytest.constants import DEFAULT_LOGDIR from idf_pytest.plugin import IDF_LOCAL_PLUGIN_KEY @@ -125,7 +126,7 @@ class AppDownloader: self.commit_sha = commit_sha self.pipeline_id = pipeline_id - def download_app(self, app_build_path: str, artifact_type: str | None = None) -> None: + def download_app(self, app_dir: str, build_dir: str, artifact_type: str | None = None) -> None: args = [ 'idf-ci', 'gitlab', @@ -135,18 +136,26 @@ class AppDownloader: ] if artifact_type: args.extend(['--type', artifact_type]) + if self.pipeline_id: args.extend(['--pipeline-id', self.pipeline_id]) - args.append(app_build_path) - subprocess.run( - args, - stdout=sys.stdout, - stderr=sys.stderr, + args.extend( + [ + app_dir, + '--build-dir', + build_dir, + ] ) - - -PRESIGNED_JSON = 'presigned.json' + result = subprocess.run( + args, + capture_output=True, + text=True, + cwd=IDF_PATH, + ) + logging.info(result.stdout) + if result.stderr: + logging.info(result.stderr) class OpenOCD: @@ -322,12 +331,13 @@ def build_dir( downloader = app_downloader if downloader: + app_dir = idf_relpath(app_path) + build_dir = f'build_{target}_{config}' # somehow hardcoded... - app_build_path = os.path.join(idf_relpath(app_path), f'build_{target}_{config}') if requires_elf_or_map(case): - downloader.download_app(app_build_path) + downloader.download_app(app_dir, build_dir) else: - downloader.download_app(app_build_path, 'flash') + downloader.download_app(app_dir, build_dir, 'flash') check_dirs = [f'build_{target}_{config}'] else: check_dirs = [] diff --git a/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml index 7db95ac674..564f682418 100644 --- a/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml +++ b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml @@ -86,7 +86,6 @@ - run_cmd python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs # CI specific options start from "--known-failure-cases-file xxx". could ignore when running locally - run_cmd pytest $nodes - --pipeline-id $PARENT_PIPELINE_ID --junitxml=XUNIT_RESULT_${CI_JOB_ID}.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} --parallel-count ${CI_NODE_TOTAL:-1} diff --git a/tools/ci/idf_ci_local/app.py b/tools/ci/idf_ci_local/app.py index a0eaf2b7a2..3cd4ccc21c 100644 --- a/tools/ci/idf_ci_local/app.py +++ b/tools/ci/idf_ci_local/app.py @@ -36,6 +36,8 @@ class IdfCMakeApp(CMakeApp): 'gitlab', 'upload-artifacts', self.app_dir, + '--build-dir', + self.build_dir, ], stdout=sys.stdout, stderr=sys.stderr, diff --git a/tools/requirements/requirements.ci.txt b/tools/requirements/requirements.ci.txt index 2865254683..7874fbabdd 100644 --- a/tools/requirements/requirements.ci.txt +++ b/tools/requirements/requirements.ci.txt @@ -6,7 +6,7 @@ # https://docs.espressif.com/projects/esp-idf/en/latest/api-guides/tools/idf-tools.html # ci -idf-ci>=0.3,<1 +idf-ci coverage jsonschema diff --git a/tools/test_apps/system/mmu_page_size/pytest_mmu_page_size.py b/tools/test_apps/system/mmu_page_size/pytest_mmu_page_size.py index 09d1b94ab5..88492e046f 100644 --- a/tools/test_apps/system/mmu_page_size/pytest_mmu_page_size.py +++ b/tools/test_apps/system/mmu_page_size/pytest_mmu_page_size.py @@ -1,9 +1,8 @@ -# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2024-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: CC0-1.0 import os import pytest -from idf_ci_utils import IDF_PATH from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize @@ -16,15 +15,12 @@ def test_app_mmu_page_size_32k_and_bootloader_mmu_page_size_64k(dut: Dut, app_do assert '32K' in config app_config = config.replace('32K', '64K') + build_dir = f'build_{dut.target}_{app_config}' - path_to_mmu_page_size_64k_build = os.path.join(dut.app.app_path, f'build_{dut.target}_{app_config}') if app_downloader: - app_downloader.download_app( - os.path.relpath(path_to_mmu_page_size_64k_build, IDF_PATH), - 'flash', - ) + app_downloader.download_app(dut.app.app_path, build_dir, 'flash') - dut.serial.bootloader_flash(path_to_mmu_page_size_64k_build) + dut.serial.bootloader_flash(os.path.join(dut.app.app_path, build_dir)) dut.expect('MMU page size mismatch') dut.expect('App is running') dut.expect('Partition test done') @@ -39,15 +35,12 @@ def test_app_mmu_page_size_64k_and_bootloader_mmu_page_size_32k(dut: Dut, app_do assert '64K' in config app_config = config.replace('64K', '32K') + build_dir = f'build_{dut.target}_{app_config}' - path_to_mmu_page_size_32k_build = os.path.join(dut.app.app_path, f'build_{dut.target}_{app_config}') if app_downloader: - app_downloader.download_app( - os.path.relpath(path_to_mmu_page_size_32k_build, IDF_PATH), - 'flash', - ) + app_downloader.download_app(dut.app.app_path, build_dir, 'flash') - dut.serial.bootloader_flash(path_to_mmu_page_size_32k_build) + dut.serial.bootloader_flash(os.path.join(dut.app.app_path, build_dir)) dut.expect('MMU page size mismatch') dut.expect('App is running') dut.expect('Partition test done') diff --git a/tools/test_apps/system/unicore_bootloader/pytest_unicore_bootloader.py b/tools/test_apps/system/unicore_bootloader/pytest_unicore_bootloader.py index 0fde23758a..de5222c79c 100644 --- a/tools/test_apps/system/unicore_bootloader/pytest_unicore_bootloader.py +++ b/tools/test_apps/system/unicore_bootloader/pytest_unicore_bootloader.py @@ -1,10 +1,9 @@ -# SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2023-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: CC0-1.0 import os import re import pytest -from idf_ci_utils import IDF_PATH from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize from pytest_embedded_idf.utils import soc_filtered_targets @@ -20,15 +19,12 @@ def test_multicore_app_and_unicore_bootloader(dut: Dut, app_downloader, config) assert 'multicore' in config app_config = config.replace('multicore', 'unicore') + build_dir = f'build_{dut.target}_{app_config}' - path_to_unicore_build = os.path.join(dut.app.app_path, f'build_{dut.target}_{app_config}') if app_downloader: - app_downloader.download_app( - os.path.relpath(path_to_unicore_build, IDF_PATH), - 'flash', - ) + app_downloader.download_app(dut.app.app_path, build_dir, 'flash') - dut.serial.bootloader_flash(path_to_unicore_build) + dut.serial.bootloader_flash(os.path.join(dut.app.app_path, build_dir)) dut.expect('Unicore bootloader') dut.expect('Multicore app') if 'psram' in config: @@ -47,15 +43,12 @@ def test_unicore_app_and_multicore_bootloader(dut: Dut, app_downloader, config) assert 'unicore' in config app_config = config.replace('unicore', 'multicore') + build_dir = f'build_{dut.target}_{app_config}' - path_to_multicore_build = os.path.join(dut.app.app_path, f'build_{dut.target}_{app_config}') if app_downloader: - app_downloader.download_app( - os.path.relpath(path_to_multicore_build, IDF_PATH), - 'flash', - ) + app_downloader.download_app(dut.app.app_path, build_dir, 'flash') - dut.serial.bootloader_flash(path_to_multicore_build) + dut.serial.bootloader_flash(os.path.join(dut.app.app_path, build_dir)) dut.expect('Multicore bootloader') dut.expect('Unicore app') if 'psram' in config: