Merge branch 'ci/apply-idf-ci-1.x' into 'master'

ci: upgrade idf-ci to 1.x

Closes CII-108

See merge request espressif/esp-idf!46623

.gitlab/ci/common.yml

@@ -62,7 +62,7 @@ variables:
   # Set this variable to the branch of idf-constraints repo in order to test a custom Python constraint file. The
   # branch name must be without the remote part ("origin/"). Keep the variable empty in order to use the constraint
   # file from https://dl.espressif.com/dl/esp-idf.
-  CI_PYTHON_CONSTRAINT_BRANCH: ""
+  CI_PYTHON_CONSTRAINT_BRANCH: "feat/update-idf-ci-version"
 
   # Update the filename for a specific ESP-IDF release. It is used only with CI_PYTHON_CONSTRAINT_BRANCH.
   CI_PYTHON_CONSTRAINT_FILE: "espidf.constraints.v6.1.txt"

.idf_ci.toml

@@ -50,57 +50,64 @@ include:
   - tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml
 """
 
-[gitlab.artifacts.s3.debug]
+[gitlab.artifacts.s3]
+enable = true
+
+[gitlab.artifacts.s3.configs.debug]
 bucket = "idf-artifacts"
+zip_first = true
+build_dir_pattern = "**/build*/"
 patterns = [
-    '**/build*/bootloader/*.map',
+    'bootloader/*.map',
-    '**/build*/bootloader/*.elf',
+    'bootloader/*.elf',
-    '**/build*/*.map',
+    '*.map',
-    '**/build*/*.elf',
+    '*.elf',
     # customized
-    '**/build*/esp_tee/*.map',
+    'esp_tee/*.map',
-    '**/build*/esp_tee/*.elf',
+    'esp_tee/*.elf',
-    '**/build*/gdbinit/*',
+    'gdbinit/*',
 ]
 if_clause = 'CI_JOB_GROUP_NAME != "build_non_test_related_apps"'
 
-[gitlab.artifacts.s3.flash]
+[gitlab.artifacts.s3.configs.flash]
 bucket = "idf-artifacts"
+zip_first = true
+build_dir_pattern = "**/build*/"
 patterns = [
-    '**/build*/bootloader/*.bin',
+    'bootloader/*.bin',
-    '**/build*/*.bin',
+    '*.bin',
-    '**/build*/partition_table/*.bin',
+    'partition_table/*.bin',
-    '**/build*/flasher_args.json',
+    'flasher_args.json',
-    '**/build*/flash_project_args',
+    'flash_project_args',
-    '**/build*/config/sdkconfig.json',
+    'config/sdkconfig.json',
-    '**/build*/sdkconfig',
+    'sdkconfig',
-    '**/build*/project_description.json',
+    'project_description.json',
     # customized
-    '**/build*/esp_tee/*.bin',
+    'esp_tee/*.bin',
 ]
 if_clause = 'CI_JOB_GROUP_NAME != "build_non_test_related_apps"'
 
-[gitlab.artifacts.s3.log]
+[gitlab.artifacts.s3.configs.log]
 bucket = "idf-artifacts"
 patterns = [
     '**/build*/build_log.txt',
     '**/build*/size*.json',
 ]
 
-[gitlab.artifacts.s3.junit]
+[gitlab.artifacts.s3.configs.junit]
 bucket = "idf-artifacts"
 patterns = [
     '**/XUNIT_RESULT_*.xml',
     '**/build_summary_*.xml',
 ]
 
-[gitlab.artifacts.s3.env]
+[gitlab.artifacts.s3.configs.env]
 bucket = "idf-artifacts"
 patterns = [
     '**/pipeline.env',
 ]
 
-[gitlab.artifacts.s3.longterm]
+[gitlab.artifacts.s3.configs.longterm]
 bucket = "longterm"
 if_clause = 'CI_COMMIT_REF_NAME == "master"'
 patterns = [

.pre-commit-config.yaml

@@ -233,7 +233,7 @@ repos:
       - id: check-kconfig-files
       - id: check-deprecated-kconfig-options
   - repo: https://github.com/espressif/idf-ci
-    rev: v1.0.0b4
+    rev: v1.0.0
     hooks:
       - id: check-tests-missing-config
         files: 'pytest_.*\.py$|sdkconfig(\..*)?$'

conftest.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2021-2025 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2021-2026 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 # pylint: disable=W0621  # redefined-outer-name
 #
@@ -36,6 +36,7 @@ from _pytest.config import Config
 from _pytest.fixtures import FixtureRequest
 from idf_ci import PytestCase
 from idf_ci.idf_pytest import IDF_CI_PYTEST_CASE_KEY
+from idf_ci_utils import IDF_PATH
 from idf_ci_utils import idf_relpath
 from idf_pytest.constants import DEFAULT_LOGDIR
 from idf_pytest.plugin import IDF_LOCAL_PLUGIN_KEY
@@ -126,7 +127,7 @@ class AppDownloader:
         self.commit_sha = commit_sha
         self.pipeline_id = pipeline_id
 
-    def download_app(self, app_build_path: str, artifact_type: str | None = None) -> None:
+    def download_app(self, app_dir: str, build_dir: str, artifact_type: str | None = None) -> None:
         args = [
             'idf-ci',
             'gitlab',
@@ -136,18 +137,26 @@ class AppDownloader:
         ]
         if artifact_type:
             args.extend(['--type', artifact_type])
+
         if self.pipeline_id:
             args.extend(['--pipeline-id', self.pipeline_id])
-        args.append(app_build_path)
 
-        subprocess.run(
+        args.extend(
-            args,
+            [
-            stdout=sys.stdout,
+                app_dir,
-            stderr=sys.stderr,
+                '--build-dir',
+                build_dir,
+            ]
         )
-
+        result = subprocess.run(
-
+            args,
-PRESIGNED_JSON = 'presigned.json'
+            capture_output=True,
+            text=True,
+            cwd=IDF_PATH,
+        )
+        logging.info(result.stdout)
+        if result.stderr:
+            logging.info(result.stderr)
 
 
 class OpenOCD:
@@ -323,12 +332,13 @@ def build_dir(
         downloader = app_downloader
 
     if downloader:
+        app_dir = idf_relpath(app_path)
+        build_dir = f'build_{target}_{config}'
         # somehow hardcoded...
-        app_build_path = os.path.join(idf_relpath(app_path), f'build_{target}_{config}')
         if requires_elf_or_map(case):
-            downloader.download_app(app_build_path)
+            downloader.download_app(app_dir, build_dir)
         else:
-            downloader.download_app(app_build_path, 'flash')
+            downloader.download_app(app_dir, build_dir, 'flash')
         check_dirs = [f'build_{target}_{config}']
     else:
         check_dirs = []

tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml

@@ -86,7 +86,6 @@
     - run_cmd python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
     # CI specific options start from "--known-failure-cases-file xxx". could ignore when running locally
     - run_cmd pytest $nodes
-      --pipeline-id $PARENT_PIPELINE_ID
       --junitxml=XUNIT_RESULT_${CI_JOB_ID}.xml
       --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
       --parallel-count ${CI_NODE_TOTAL:-1}

tools/ci/idf_ci_local/app.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2024-2026 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 import os
 import subprocess
@@ -8,6 +8,7 @@ import typing as t
 from dynamic_pipelines.constants import BINARY_SIZE_METRIC_NAME
 from idf_build_apps import App
 from idf_build_apps import CMakeApp
+from idf_build_apps.constants import BuildStatus
 from idf_build_apps.utils import rmdir
 from idf_ci_utils import idf_relpath
 
@@ -29,16 +30,22 @@ class IdfCMakeApp(CMakeApp):
 
         # only upload in CI
         if os.getenv('CI_JOB_ID'):
-            subprocess.run(
+            result = subprocess.run(
                 [
                     'idf-ci',
                     'gitlab',
                     'upload-artifacts',
                     self.app_dir,
+                    '--build-dir',
+                    self.build_dir,
                 ],
                 stdout=sys.stdout,
                 stderr=sys.stderr,
             )
+            if result.returncode != 0:
+                self.build_status = BuildStatus.FAILED
+                self.build_comment = 'Failed to upload artifacts'
+
             rmdir(
                 self.build_path,
                 exclude_file_patterns=['build_log.txt', 'size*.json'],

tools/requirements/requirements.ci.txt

@@ -6,7 +6,7 @@
 # https://docs.espressif.com/projects/esp-idf/en/latest/api-guides/tools/idf-tools.html
 
 # ci
-idf-ci>=0.3,<1
+idf-ci
 
 coverage
 jsonschema

tools/test_apps/system/mmu_page_size/pytest_mmu_page_size.py

@@ -1,9 +1,8 @@
-# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2024-2026 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: CC0-1.0
 import os
 
 import pytest
-from idf_ci_utils import IDF_PATH
 from pytest_embedded import Dut
 from pytest_embedded_idf.utils import idf_parametrize
 
@@ -16,15 +15,12 @@ def test_app_mmu_page_size_32k_and_bootloader_mmu_page_size_64k(dut: Dut, app_do
 
     assert '32K' in config
     app_config = config.replace('32K', '64K')
+    build_dir = f'build_{dut.target}_{app_config}'
 
-    path_to_mmu_page_size_64k_build = os.path.join(dut.app.app_path, f'build_{dut.target}_{app_config}')
     if app_downloader:
-        app_downloader.download_app(
+        app_downloader.download_app(dut.app.app_path, build_dir, 'flash')
-            os.path.relpath(path_to_mmu_page_size_64k_build, IDF_PATH),
-            'flash',
-        )
 
-    dut.serial.bootloader_flash(path_to_mmu_page_size_64k_build)
+    dut.serial.bootloader_flash(os.path.join(dut.app.app_path, build_dir))
     dut.expect('MMU page size mismatch')
     dut.expect('App is running')
     dut.expect('Partition test done')
@@ -39,15 +35,12 @@ def test_app_mmu_page_size_64k_and_bootloader_mmu_page_size_32k(dut: Dut, app_do
 
     assert '64K' in config
     app_config = config.replace('64K', '32K')
+    build_dir = f'build_{dut.target}_{app_config}'
 
-    path_to_mmu_page_size_32k_build = os.path.join(dut.app.app_path, f'build_{dut.target}_{app_config}')
     if app_downloader:
-        app_downloader.download_app(
+        app_downloader.download_app(dut.app.app_path, build_dir, 'flash')
-            os.path.relpath(path_to_mmu_page_size_32k_build, IDF_PATH),
-            'flash',
-        )
 
-    dut.serial.bootloader_flash(path_to_mmu_page_size_32k_build)
+    dut.serial.bootloader_flash(os.path.join(dut.app.app_path, build_dir))
     dut.expect('MMU page size mismatch')
     dut.expect('App is running')
     dut.expect('Partition test done')

tools/test_apps/system/unicore_bootloader/pytest_unicore_bootloader.py

@@ -1,10 +1,9 @@
-# SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2023-2026 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: CC0-1.0
 import os
 import re
 
 import pytest
-from idf_ci_utils import IDF_PATH
 from pytest_embedded import Dut
 from pytest_embedded_idf.utils import idf_parametrize
 from pytest_embedded_idf.utils import soc_filtered_targets
@@ -20,15 +19,12 @@ def test_multicore_app_and_unicore_bootloader(dut: Dut, app_downloader, config)
 
     assert 'multicore' in config
     app_config = config.replace('multicore', 'unicore')
+    build_dir = f'build_{dut.target}_{app_config}'
 
-    path_to_unicore_build = os.path.join(dut.app.app_path, f'build_{dut.target}_{app_config}')
     if app_downloader:
-        app_downloader.download_app(
+        app_downloader.download_app(dut.app.app_path, build_dir, 'flash')
-            os.path.relpath(path_to_unicore_build, IDF_PATH),
-            'flash',
-        )
 
-    dut.serial.bootloader_flash(path_to_unicore_build)
+    dut.serial.bootloader_flash(os.path.join(dut.app.app_path, build_dir))
     dut.expect('Unicore bootloader')
     dut.expect('Multicore app')
     if 'psram' in config:
@@ -47,15 +43,12 @@ def test_unicore_app_and_multicore_bootloader(dut: Dut, app_downloader, config)
 
     assert 'unicore' in config
     app_config = config.replace('unicore', 'multicore')
+    build_dir = f'build_{dut.target}_{app_config}'
 
-    path_to_multicore_build = os.path.join(dut.app.app_path, f'build_{dut.target}_{app_config}')
     if app_downloader:
-        app_downloader.download_app(
+        app_downloader.download_app(dut.app.app_path, build_dir, 'flash')
-            os.path.relpath(path_to_multicore_build, IDF_PATH),
-            'flash',
-        )
 
-    dut.serial.bootloader_flash(path_to_multicore_build)
+    dut.serial.bootloader_flash(os.path.join(dut.app.app_path, build_dir))
     dut.expect('Multicore bootloader')
     dut.expect('Unicore app')
     if 'psram' in config: