espressif/esp-idf
1
0
Fork 0
mirror of https://github.com/espressif/esp-idf.git synced 2026-04-27 19:13:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'ci/simplify-ci' into 'master'

Browse Source 
Ci/simplify ci

Closes IDFCI-6717

See merge request espressif/esp-idf!43157
This commit is contained in:
Fu Hanxi
2025-11-21 07:34:24 +01:00
parent 80454bbc59 accfcc96f4
commit 6c585a0e18
18 changed files with 315 additions and 527 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitlab-ci.yml
-1
View File
@@ -29,5 +29,4 @@ include:
- ".gitlab/ci/host-test.yml"
- ".gitlab/ci/deploy.yml"
- ".gitlab/ci/post_deploy.yml"
- ".gitlab/ci/retry_failed_jobs.yml"
- ".gitlab/ci/test-win.yml"
.gitlab/ci/build.yml
+66 -276
View File
@@ -1,20 +1,66 @@
.build_template:
stage: build
extends:
- .after_script:build:ccache-show-stats:upload-failed-job-logs
- .before_script:build
- .after_script:build
image: $ESP_ENV_IMAGE
tags: [build, shiny]
variables:
# Enable ccache for all build jobs. See configure_ci_environment.sh for more ccache related settings.
IDF_CCACHE_ENABLE: "1"
dependencies: []
dependencies: # set dependencies to null to avoid missing artifacts issue
######################
# build_template_app #
######################
.build_template_app_template:
extends:
- .build_template
artifacts:
paths:
- log_template_app/*
- size_info.txt
- build_template_app/**/size*.json
expire_in: 1 week
when: always
script:
# Set the variable for 'esp-idf-template' testing
- ESP_IDF_TEMPLATE_GIT=${ESP_IDF_TEMPLATE_GIT:-"https://github.com/espressif/esp-idf-template.git"}
- retry_failed git clone ${ESP_IDF_TEMPLATE_GIT}
# Try to use the same branch name for esp-idf-template that we're
# using on esp-idf. If it doesn't exist then just stick to the default branch
- python $CHECKOUT_REF_SCRIPT esp-idf-template esp-idf-template
- export PATH="$IDF_PATH/tools:$PATH"
# Only do the default cmake build for each target, remaining part are done in the build_template_app job
- tools/ci/build_template_app.sh ${BUILD_COMMAND_ARGS}
# Build at least one project for each target at earliest stage to reduce build cost for obvious failing commits
fast_template_app:
extends:
- .build_template_app_template
- .rules:build
stage: pre_check
tags: [fast_run, shiny]
variables:
BUILD_COMMAND_ARGS: "-p"
# This job builds template app with permutations of targets and optimization levels
build_template_app:
extends:
- .build_template_app_template
- .rules:build
stage: host_test
needs:
- job: fast_template_app
artifacts: false
########################################
# Clang Build Apps Without Tests Cases #
########################################
.build_cmake_clang_template:
extends:
- .build_template
- .before_script:build
- .after_script:build:ccache-show-stats
dependencies: # set dependencies to null to avoid missing artifacts issue
- .rules:build
needs:
- job: fast_template_app
artifacts: false
@@ -45,92 +91,14 @@
--modified-files ${MR_MODIFIED_FILES}
$TEST_BUILD_OPTS_EXTRA
######################
# build_template_app #
######################
.build_template_app_template:
extends:
- .build_template
- .before_script:build
variables:
LOG_PATH: "${CI_PROJECT_DIR}/log_template_app"
BUILD_PATH: "${CI_PROJECT_DIR}/build_template_app"
BUILD_DIR: "${BUILD_PATH}/@t/@w"
BUILD_LOG_CMAKE: "${LOG_PATH}/cmake_@t_@w.txt"
BUILD_COMMAND_ARGS: ""
artifacts:
paths:
- log_template_app/*
- size_info.txt
- build_template_app/**/size*.json
expire_in: 1 week
when: always
script:
# Set the variable for 'esp-idf-template' testing
- ESP_IDF_TEMPLATE_GIT=${ESP_IDF_TEMPLATE_GIT:-"https://github.com/espressif/esp-idf-template.git"}
- retry_failed git clone ${ESP_IDF_TEMPLATE_GIT}
# Try to use the same branch name for esp-idf-template that we're
# using on esp-idf. If it doesn't exist then just stick to the default branch
- python $CHECKOUT_REF_SCRIPT esp-idf-template esp-idf-template
- export PATH="$IDF_PATH/tools:$PATH"
# Only do the default cmake build for each target, remaining part are done in the build_template_app job
- tools/ci/build_template_app.sh ${BUILD_COMMAND_ARGS}
build_clang_test_apps_xtensa:
extends: .build_cmake_clang_template
parallel:
matrix:
- IDF_TARGET: [esp32, esp32s2, esp32s3]
# build-related-pre-check-jobs ------------------------------------------------
# Build at least one project for each target at earliest stage to reduce build cost for obvious failing commits
fast_template_app:
extends:
- .build_template_app_template
- .rules:build:target_test
stage: pre_check
tags: [fast_run, shiny]
variables:
BUILD_COMMAND_ARGS: "-p"
#------------------------------------------------------------------------------
#######################
# gnu_static_analyzer #
#######################
gcc_static_analyzer:
extends:
- .build_template_app_template
- .rules:build:target_test
stage: pre_check
tags: [build, shiny]
variables:
CI_CCACHE_DISABLE: 1
ANALYZING_APP: "examples/get-started/hello_world"
script:
- echo "CONFIG_COMPILER_STATIC_ANALYZER=y" >> ${ANALYZING_APP}/sdkconfig.defaults
- idf-build-apps build -p ${ANALYZING_APP}
########################################
# Clang Build Apps Without Tests Cases #
########################################
build_clang_test_apps_esp32:
extends:
- .build_cmake_clang_template
- .rules:build
variables:
IDF_TARGET: esp32
build_clang_test_apps_esp32s2:
extends:
- .build_cmake_clang_template
- .rules:build
variables:
IDF_TARGET: esp32s2
build_clang_test_apps_esp32s3:
extends:
- .build_cmake_clang_template
- .rules:build
variables:
IDF_TARGET: esp32s3
.build_clang_test_apps_riscv:
extends:
- .build_cmake_clang_template
build_clang_test_apps_riscv:
extends: .build_cmake_clang_template
variables:
# https://reviews.llvm.org/D90108.
# GNU 'as' lets .weak override .globl since binutils-gdb
@@ -138,191 +106,9 @@ build_clang_test_apps_esp32s3:
# while MC lets the last directive win (PR38921).
# For RISCV chips we use integrated assembler by default, so suppress this warning to pass CI pipeline.
TEST_BUILD_OPTS_EXTRA: "--ignore-warning-str 'changed binding to STB_WEAK'"
build_clang_test_apps_esp32c3:
extends:
- .build_clang_test_apps_riscv
- .rules:build
variables:
IDF_TARGET: esp32c3
build_clang_test_apps_esp32c2:
extends:
- .build_clang_test_apps_riscv
- .rules:build
variables:
IDF_TARGET: esp32c2
build_clang_test_apps_esp32c6:
extends:
- .build_clang_test_apps_riscv
- .rules:build
variables:
IDF_TARGET: esp32c6
build_clang_test_apps_esp32c5:
extends:
- .build_clang_test_apps_riscv
- .rules:build
variables:
IDF_TARGET: esp32c5
build_clang_test_apps_esp32h2:
extends:
- .build_clang_test_apps_riscv
- .rules:build
variables:
IDF_TARGET: esp32h2
build_clang_test_apps_esp32p4:
extends:
- .build_clang_test_apps_riscv
- .rules:build
variables:
IDF_TARGET: esp32p4
######################
# Build System Tests #
######################
.test_build_system_template:
stage: host_test
extends:
- .build_template
- .rules:build:check
dependencies: # set dependencies to null to avoid missing artifacts issue
needs:
- job: fast_template_app
artifacts: false
optional: true
artifacts:
reports:
junit: XUNIT_RESULT.xml
paths:
- XUNIT_RESULT.xml
- test_build_system
expire_in: 1 week
when: always
script:
- ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
- cd ${IDF_PATH}/tools/test_build_system
- run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
- pytest
--cleanup-idf-copy
--parallel-count ${CI_NODE_TOTAL:-1}
--parallel-index ${CI_NODE_INDEX:-1}
--work-dir ${CI_PROJECT_DIR}/test_build_system
--junitxml ${CI_PROJECT_DIR}/XUNIT_RESULT.xml
--ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
.test_build_system_minimal_cmake_template:
extends: .test_build_system_template
variables:
INSTALL_EXTRA_TOOLS: cmake@3.22.1
script:
- MINIMAL_SUPPORTED_CMAKE_VERSION=$(echo "${INSTALL_EXTRA_TOOLS}" | sed -n 's/.*cmake@\([0-9.]*\).*/\1/p')
- export PATH=$(echo "$PATH" | sed -E "s|/tools/cmake/[0-9.]+|/tools/cmake/${MINIMAL_SUPPORTED_CMAKE_VERSION}|")
- ACTUAL_CMAKE_VERSION=$(cmake --version | head -n1 | awk '{print $3}')
- |
if [ "${ACTUAL_CMAKE_VERSION}" != "${MINIMAL_SUPPORTED_CMAKE_VERSION}" ]; then
echo "ERROR: Wrong minimal CMake version! Detected: ${ACTUAL_CMAKE_VERSION}, but should be: ${MINIMAL_SUPPORTED_CMAKE_VERSION}"
exit 1
fi
- ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
- cd ${IDF_PATH}/tools/test_build_system
- run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
- pytest
-k cmake
--cleanup-idf-copy
--parallel-count ${CI_NODE_TOTAL:-1}
--parallel-index ${CI_NODE_INDEX:-1}
--work-dir ${CI_PROJECT_DIR}/test_build_system
--junitxml ${CI_PROJECT_DIR}/XUNIT_RESULT.xml
--ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
pytest_build_system:
extends: .test_build_system_template
parallel: 3
pytest_buildv2_system:
extends:
- .test_build_system_template
- .rules:labels:buildv2
parallel: 3
script:
- ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
- cd ${IDF_PATH}/tools/test_build_system
- run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
- pytest
--buildv2
--cleanup-idf-copy
--parallel-count ${CI_NODE_TOTAL:-1}
--parallel-index ${CI_NODE_INDEX:-1}
--work-dir ${CI_PROJECT_DIR}/test_build_system
--junitxml ${CI_PROJECT_DIR}/XUNIT_RESULT.xml
--ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
--
test_non_default_target.py
test_component_manager.py
test_build.py
test_bootloader.py
test_git.py
test_kconfig.py
test_partition.py
test_reproducible_build.py
test_sdkconfig.py
test_versions.py
test_common.py
test_components.py
test_cmake.py
test_idf_extension.py
test_rebuild.py
pytest_build_system_macos:
extends:
- .test_build_system_template
- .brew-macos-settings
- .rules:build:macos
parallel: 3
pytest_build_system_minimal_cmake:
extends: .test_build_system_minimal_cmake_template
pytest_build_system_macos_minimal_cmake:
extends:
- .test_build_system_minimal_cmake_template
- .brew-macos-settings
- .rules:build:macos
variables:
INSTALL_EXTRA_TOOLS: ninja cmake@3.22.1
build_docker:
extends:
- .before_script:minimal
- .rules:build:docker
stage: host_test
needs: []
image: espressif/docker-builder:1
tags: [shiny, dind]
variables:
DOCKER_TMP_IMAGE_NAME: "idf_tmp_image"
script:
- export DOCKER_BUILD_ARGS="--build-arg IDF_CLONE_URL=${CI_REPOSITORY_URL} --build-arg IDF_CLONE_BRANCH_OR_TAG=${CI_COMMIT_REF_NAME} --build-arg IDF_CHECKOUT_REF=${CI_COMMIT_TAG:-$CI_COMMIT_SHA} --build-arg IDF_CLONE_SHALLOW=1 --build-arg IDF_GITHUB_ASSETS=${INTERNAL_GITHUB_ASSETS}"
- docker build --tag ${DOCKER_TMP_IMAGE_NAME} ${DOCKER_BUILD_ARGS} tools/docker/
# We can't mount $PWD/examples/get-started/blink into the container, see https://gitlab.com/gitlab-org/gitlab-ce/issues/41227.
# The workaround mentioned there works, but leaves around directories which need to be cleaned up manually.
# Therefore, build a copy of the example located inside the container.
- docker run --rm --workdir /opt/esp/idf/examples/get-started/blink ${DOCKER_TMP_IMAGE_NAME} idf.py build
# This job builds template app with permutations of targets and optimization levels
build_template_app:
extends:
- .build_template_app_template
- .rules:build
stage: host_test
dependencies: # set dependencies to null to avoid missing artifacts issue
needs:
- job: fast_template_app
artifacts: false
parallel:
matrix:
- IDF_TARGET: [esp32c3, esp32c2, esp32c6, esp32c5, esp32h2, esp32p4]
####################
# Dynamic Pipeline #
@@ -344,7 +130,11 @@ generate_build_child_pipeline:
expire_in: 1 week
when: always
script:
- run_cmd python tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py
- run_cmd idf-ci --debug gitlab build-child-pipeline
-p components
-p examples
-p tools/test_apps
--modified-files $MR_MODIFIED_FILES
build_child_pipeline:
stage: build
.gitlab/ci/common.yml
+3 -16
View File
@@ -12,7 +12,6 @@ stages:
- test_deploy
- deploy
- post_deploy
- retry_failed_jobs
variables:
# System environment
@@ -122,11 +121,6 @@ variables:
# add extra python packages
export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci:$IDF_PATH/tools/esp_app_trace:$IDF_PATH/components/partition_table:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH"
# minio configuration
# added here since the precedence of variables in gitlab-ci.yml is lower than project settings
export IDF_S3_SERVER="$IDF_S3_NEW_SERVER"
export IDF_S3_ACCESS_KEY="$IDF_S3_NEW_ACCESS_KEY"
.setup_tools_and_idf_python_venv: &setup_tools_and_idf_python_venv |
# must use after setup_tools_except_target_test
# otherwise the export.sh won't work properly
@@ -229,7 +223,9 @@ variables:
.show_ccache_statistics: &show_ccache_statistics |
# Show ccache statistics if enabled globally
section_start "ccache_show_stats" "Show ccache statistics"
test "$CI_CCACHE_STATS" == 1 && test -n "$(which ccache)" && ccache --show-stats -vv || true
section_end "ccache_show_stats"
.upload_failed_job_log_artifacts: &upload_failed_job_log_artifacts |
if [ $CI_JOB_STATUS = "failed" ]; then
@@ -249,19 +245,10 @@ variables:
- export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
- export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
.after_script:build:ccache-show-stats:
.after_script:build:
after_script:
- source tools/ci/utils.sh
- section_start "ccache_show_stats" "Show ccache statistics"
- *show_ccache_statistics
- section_end "ccache_show_stats"
.after_script:build:ccache-show-stats:upload-failed-job-logs:
after_script:
- source tools/ci/utils.sh
- section_start "ccache_show_stats" "Show ccache statistics"
- *show_ccache_statistics
- section_end "ccache_show_stats"
- *upload_failed_job_log_artifacts
##############################
.gitlab/ci/dependencies/dependencies.yml
-3
View File
@@ -9,7 +9,6 @@
- build_system
- downloadable-tools
included_in:
- build:target_test
- build:check
# -------------------
@@ -41,8 +40,6 @@
"patterns:template-app":
patterns:
- build_template-app
included_in:
- build:target_test
"patterns:build-check":
patterns:
.gitlab/ci/host-test.yml
+192 -81
View File
@@ -6,13 +6,16 @@
dependencies: # set dependencies to null to avoid missing artifacts issue
# run host_test jobs immediately, only after upload cache
needs:
- pipeline_variables
- job: upload-pip-cache
optional: true
artifacts: false
- job: upload-submodules-cache
optional: true
artifacts: false
- pipeline_variables
- job: fast_template_app
optional: true
artifacts: false
artifacts:
expire_in: 1 week
when: always
@@ -52,14 +55,14 @@ test_partition_table_on_host:
extends: .host_test_template
script:
- cd components/partition_table/test_gen_esp32part_host
- ./gen_esp32part_tests.py
- pytest_for_ut ./gen_esp32part_tests.py
test_ldgen_on_host:
extends: .host_test_template
script:
- cd tools/ldgen/test
- export PYTHONPATH=$PYTHONPATH:..
- python -m unittest
- pytest_for_ut .
variables:
LC_ALL: C.UTF-8
@@ -67,21 +70,19 @@ test_spiffs_on_host:
extends: .host_test_template
script:
- cd components/spiffs/test_spiffsgen/
- ./test_spiffsgen.py
- pytest_for_ut ./test_spiffsgen.py
test_fatfsgen_on_host:
extends: .host_test_template
script:
- cd components/fatfs/test_fatfsgen/
- ./test_fatfsgen.py
- ./test_wl_fatfsgen.py
- ./test_fatfsparse.py
- pytest_for_ut ./test_fatfsgen.py ./test_wl_fatfsgen.py ./test_fatfsparse.py
test_certificate_bundle_on_host:
extends: .host_test_template
script:
- cd components/mbedtls/esp_crt_bundle/test_gen_crt_bundle/
- ./test_gen_crt_bundle.py
- pytest_for_ut ./test_gen_crt_bundle.py
# Test for create virtualenv. It must be invoked from Python, not from virtualenv.
# Use docker image system python without any extra dependencies
@@ -104,15 +105,17 @@ test_cli_installer:
- python3 ${IDF_PATH}/tools/idf_tools.py download required qemu-riscv32 qemu-xtensa cmake cmake@3.22.1
- cd ${IDF_PATH}/tools/test_idf_tools
- python3 -m pip install jsonschema
# Testing with system python3, so don't use any third-party packages
- python3 ./test_idf_tools.py -v
- python3 ./test_idf_tools_python_env.py
# It runs at the end because it modifies dependencies
- IDF_TEST_MAY_BREAK_DEPENDENCIES=1 python3 ./test_idf_tools.py -v TestSystemDependencies.test_commands_when_nodeps
.test_efuse_table_on_host_template:
test_efuse_table_on_host:
extends: .host_test_template
variables:
IDF_TARGET: "esp32"
parallel:
matrix:
- IDF_TARGET: [esp32, esp32s2, esp32c3, esp32s3, esp32c2, esp32c6, esp32h2, esp32p4, esp32c5, esp32c61]
artifacts:
when: on_failure
paths:
@@ -122,35 +125,7 @@ test_cli_installer:
- ./efuse_table_gen.py -t "${IDF_TARGET}" ${IDF_PATH}/components/efuse/${IDF_TARGET}/esp_efuse_table.csv
- git diff --exit-code -- ${IDF_TARGET}/esp_efuse_table.c || { echo 'Differences found for ${IDF_TARGET} target. Please run idf.py efuse-common-table and commit the changes.'; exit 1; }
- cd ${IDF_PATH}/components/efuse/test_efuse_host
- ./efuse_tests.py
test_efuse_table_on_host_esp32:
extends: .test_efuse_table_on_host_template
test_efuse_table_on_host_esp32s2:
extends: .test_efuse_table_on_host_template
variables:
IDF_TARGET: esp32s2
test_efuse_table_on_host_esp32s3:
extends: .test_efuse_table_on_host_template
variables:
IDF_TARGET: esp32s3
test_efuse_table_on_host_esp32c3:
extends: .test_efuse_table_on_host_template
variables:
IDF_TARGET: esp32c3
test_efuse_table_on_host_esp32h2:
extends: .test_efuse_table_on_host_template
variables:
IDF_TARGET: esp32h2
test_efuse_table_on_host_esp32c6:
extends: .test_efuse_table_on_host_template
variables:
IDF_TARGET: esp32c6
- pytest_for_ut ./efuse_tests.py
test_logtrace_proc:
extends: .host_test_template
@@ -217,7 +192,7 @@ test_split_path_by_spaces:
extends: .host_test_template
script:
- cd ${IDF_PATH}/tools
- python -m unittest split_paths_by_spaces.py
- pytest_for_ut ./split_paths_by_spaces.py
test_transport_on_host:
extends: .host_test_template
@@ -256,8 +231,78 @@ test_gen_soc_caps_kconfig:
extends: .host_test_template
script:
- cd ${IDF_PATH}/tools/gen_soc_caps_kconfig/
- ./test/test_gen_soc_caps_kconfig.py
- pytest_for_ut ./test/test_gen_soc_caps_kconfig.py
test_idf_build_apps_load_soc_caps:
extends: .host_test_template
script:
- cd tools/ci
- pytest_for_ut ./test_soc_headers_load_in_idf_build_apps.py
test_nvs_gen_check:
extends: .host_test_template
artifacts:
paths:
- XUNIT_RESULT.xml
- components/nvs_flash/nvs_partition_tool
reports:
junit: XUNIT_RESULT.xml
variables:
LC_ALL: C.UTF-8
script:
- cd ${IDF_PATH}/components/nvs_flash/nvs_partition_tool
- pytest_for_ut test_nvs_gen_check.py
test_esp_rom:
extends: .host_test_template
artifacts:
paths:
- XUNIT_RESULT.xml
reports:
junit: XUNIT_RESULT.xml
script:
- cd ${IDF_PATH}/components/esp_rom/
- pytest_for_ut test_esp_rom.py
make_sure_soc_caps_compatible_in_idf_build_apps:
extends:
- .host_test_template
- .rules:dev-push
artifacts:
paths:
- new.json
- base.json
when: always
when: manual
script:
- python tools/ci/idf_build_apps_dump_soc_caps.py new.json
- git fetch --depth=1 origin $CI_MERGE_REQUEST_DIFF_BASE_SHA
- git checkout -f $CI_MERGE_REQUEST_DIFF_BASE_SHA
- git checkout $CI_COMMIT_SHA -- tools/ci/idf_build_apps_dump_soc_caps.py
- python tools/ci/idf_build_apps_dump_soc_caps.py base.json
- diff new.json base.json
build_docker:
extends:
- .before_script:minimal
- .rules:build:docker
stage: host_test
needs: []
image: espressif/docker-builder:1
tags: [shiny, dind]
variables:
DOCKER_TMP_IMAGE_NAME: "idf_tmp_image"
script:
- export DOCKER_BUILD_ARGS="--build-arg IDF_CLONE_URL=${CI_REPOSITORY_URL} --build-arg IDF_CLONE_BRANCH_OR_TAG=${CI_COMMIT_REF_NAME} --build-arg IDF_CHECKOUT_REF=${CI_COMMIT_TAG:-$CI_COMMIT_SHA} --build-arg IDF_CLONE_SHALLOW=1 --build-arg IDF_GITHUB_ASSETS=${INTERNAL_GITHUB_ASSETS}"
- docker build --tag ${DOCKER_TMP_IMAGE_NAME} ${DOCKER_BUILD_ARGS} tools/docker/
# We can't mount $PWD/examples/get-started/blink into the container, see https://gitlab.com/gitlab-org/gitlab-ce/issues/41227.
# The workaround mentioned there works, but leaves around directories which need to be cleaned up manually.
# Therefore, build a copy of the example located inside the container.
- docker run --rm --workdir /opt/esp/idf/examples/get-started/blink ${DOCKER_TMP_IMAGE_NAME} idf.py build
############################
# Host test with test apps #
############################
test_pytest_qemu:
extends:
- .host_test_template
@@ -344,50 +389,116 @@ test_pytest_macos:
--junitxml=XUNIT_RESULT.xml
--ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
test_idf_build_apps_load_soc_caps:
extends: .host_test_template
script:
- python tools/ci/check_soc_headers_load_in_idf_build_apps.py
test_nvs_gen_check:
extends: .host_test_template
######################
# Build System Tests #
######################
.test_build_system_template:
stage: host_test
extends:
- .build_template
- .rules:build:check
dependencies: # set dependencies to null to avoid missing artifacts issue
needs:
- job: fast_template_app
artifacts: false
optional: true
artifacts:
paths:
- XUNIT_RESULT.xml
- components/nvs_flash/nvs_partition_tool
reports:
junit: XUNIT_RESULT.xml
paths:
- XUNIT_RESULT.xml
- test_build_system
expire_in: 1 week
when: always
script:
- ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
- cd ${IDF_PATH}/tools/test_build_system
- run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
- pytest
--cleanup-idf-copy
--parallel-count ${CI_NODE_TOTAL:-1}
--parallel-index ${CI_NODE_INDEX:-1}
--work-dir ${CI_PROJECT_DIR}/test_build_system
--junitxml ${CI_PROJECT_DIR}/XUNIT_RESULT.xml
--ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
.test_build_system_minimal_cmake_template:
extends: .test_build_system_template
variables:
LC_ALL: C.UTF-8
INSTALL_EXTRA_TOOLS: cmake@3.22.1
script:
- cd ${IDF_PATH}/components/nvs_flash/nvs_partition_tool
- pytest --noconftest test_nvs_gen_check.py --junitxml=XUNIT_RESULT.xml
- MINIMAL_SUPPORTED_CMAKE_VERSION=$(echo "${INSTALL_EXTRA_TOOLS}" | sed -n 's/.*cmake@\([0-9.]*\).*/\1/p')
- export PATH=$(echo "$PATH" | sed -E "s|/tools/cmake/[0-9.]+|/tools/cmake/${MINIMAL_SUPPORTED_CMAKE_VERSION}|")
- ACTUAL_CMAKE_VERSION=$(cmake --version | head -n1 | awk '{print $3}')
- |
if [ "${ACTUAL_CMAKE_VERSION}" != "${MINIMAL_SUPPORTED_CMAKE_VERSION}" ]; then
echo "ERROR: Wrong minimal CMake version! Detected: ${ACTUAL_CMAKE_VERSION}, but should be: ${MINIMAL_SUPPORTED_CMAKE_VERSION}"
exit 1
fi
- ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
- cd ${IDF_PATH}/tools/test_build_system
- run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
- pytest
-k cmake
--cleanup-idf-copy
--parallel-count ${CI_NODE_TOTAL:-1}
--parallel-index ${CI_NODE_INDEX:-1}
--work-dir ${CI_PROJECT_DIR}/test_build_system
--junitxml ${CI_PROJECT_DIR}/XUNIT_RESULT.xml
--ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
test_esp_rom:
extends: .host_test_template
artifacts:
paths:
- XUNIT_RESULT.xml
reports:
junit: XUNIT_RESULT.xml
pytest_build_system:
extends: .test_build_system_template
parallel: 3
pytest_buildv2_system:
extends:
- .test_build_system_template
- .rules:labels:buildv2
parallel: 3
script:
- cd ${IDF_PATH}/components/esp_rom/
- pytest --noconftest test_esp_rom.py --junitxml=XUNIT_RESULT.xml
- ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
- cd ${IDF_PATH}/tools/test_build_system
- run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
- pytest
--buildv2
--cleanup-idf-copy
--parallel-count ${CI_NODE_TOTAL:-1}
--parallel-index ${CI_NODE_INDEX:-1}
--work-dir ${CI_PROJECT_DIR}/test_build_system
--junitxml ${CI_PROJECT_DIR}/XUNIT_RESULT.xml
--ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
--
test_non_default_target.py
test_component_manager.py
test_build.py
test_bootloader.py
test_git.py
test_kconfig.py
test_partition.py
test_reproducible_build.py
test_sdkconfig.py
test_versions.py
test_common.py
test_components.py
test_cmake.py
test_idf_extension.py
test_rebuild.py
make_sure_soc_caps_compatible_in_idf_build_apps:
extends:
- .host_test_template
- .rules:dev-push
artifacts:
paths:
- new.json
- base.json
when: always
when: manual
script:
- python tools/ci/idf_build_apps_dump_soc_caps.py new.json
- git fetch --depth=1 origin $CI_MERGE_REQUEST_DIFF_BASE_SHA
- git checkout -f $CI_MERGE_REQUEST_DIFF_BASE_SHA
- git checkout $CI_COMMIT_SHA -- tools/ci/idf_build_apps_dump_soc_caps.py
- python tools/ci/idf_build_apps_dump_soc_caps.py base.json
- diff new.json base.json
pytest_build_system_macos:
extends:
- .test_build_system_template
- .brew-macos-settings
- .rules:build:macos
parallel: 3
pytest_build_system_minimal_cmake:
extends: .test_build_system_minimal_cmake_template
pytest_build_system_macos_minimal_cmake:
extends:
- .test_build_system_minimal_cmake_template
- .brew-macos-settings
- .rules:build:macos
variables:
INSTALL_EXTRA_TOOLS: ninja cmake@3.22.1
.gitlab/ci/pre_check.yml
+17 -6
View File
@@ -191,13 +191,24 @@ baseline_manifest_sha:
expire_in: 1 week
when: always
redundant_pass_job:
gcc_static_analyzer:
extends:
- .pre_check_template
tags: [shiny, fast_run]
cache: []
- .rules:build
variables:
GIT_STRATEGY: none
before_script: []
CI_CCACHE_DISABLE: 1
ANALYZING_APP: "examples/get-started/hello_world"
script:
- echo "This job is redundant to ensure the 'retry_failed_jobs' job can exist and not be skipped"
- echo "CONFIG_COMPILER_STATIC_ANALYZER=y" >> ${ANALYZING_APP}/sdkconfig.defaults
- idf-build-apps build -p ${ANALYZING_APP}
retry_failed_jobs:
extends:
- .pre_check_template
- .rules:dev-push
tags: [shiny, fast_run]
allow_failure: true
script:
- echo "Retrieving and retrying all failed jobs for the pipeline..."
- python tools/ci/python_packages/gitlab_api.py retry_failed_jobs $CI_MERGE_REQUEST_PROJECT_ID --pipeline_id $CI_PIPELINE_ID
when: manual
.gitlab/ci/retry_failed_jobs.yml
-15
View File
@@ -1,15 +0,0 @@
retry_failed_jobs:
stage: retry_failed_jobs
tags: [shiny, fast_run]
allow_failure: true
image: $ESP_ENV_IMAGE
dependencies: null
before_script: []
cache: []
extends: []
script:
- echo "Retrieving and retrying all failed jobs for the pipeline..."
- python tools/ci/python_packages/gitlab_api.py retry_failed_jobs $CI_MERGE_REQUEST_PROJECT_ID --pipeline_id $CI_PIPELINE_ID
when: manual
needs:
- redundant_pass_job
.gitlab/ci/rules.yml
+3 -21
View File
@@ -41,9 +41,6 @@
- "tools/idf_tools.py"
- "tools/tools.json"
.patterns-build_template-app: &patterns-build_template-app
- "tools/ci/build_template_app.sh"
.patterns-build_system: &patterns-build_system
- "tools/cmake/**/*"
- "tools/cmakev2/**/*"
@@ -54,14 +51,12 @@
- "tools/ci/ignore_build_warnings.txt"
- "tools/ci/test_build_system*.sh"
- "tools/ci/test_build_system*.py"
- "tools/ci/build_template_app.sh"
- "tools/test_build_system/**/*"
.patterns-build_system_win: &patterns-build_system_win
- "tools/test_build_system/**/*"
.patterns-build_macos: &patterns-build_macos
- "tools/ci/test_configure_ci_environment.sh"
.patterns-build_check: &patterns-build_check
- "tools/test_build_system/**/*"
- "tools/ci/test_configure_ci_environment.sh"
@@ -134,6 +129,8 @@
- "tools/bsasm.py"
- "tools/test_bsasm/**/*"
- "tools/ci/test_soc_headers_load_in_idf_build_apps.py"
.patterns-docker: &patterns-docker
- "tools/docker/**/*"
@@ -351,21 +348,6 @@
- <<: *if-label-macos
- <<: *if-label-macos_test
.rules:build:target_test:
rules:
- <<: *if-revert-branch
when: never
- <<: *if-protected-check
- <<: *if-label-build
- <<: *if-dev-push
changes: *patterns-build_components
- <<: *if-dev-push
changes: *patterns-build_system
- <<: *if-dev-push
changes: *patterns-build_template-app
- <<: *if-dev-push
changes: *patterns-downloadable-tools
.rules:labels:buildv2:
rules:
- <<: *if-revert-branch
.idf_ci.toml
+1
View File
@@ -89,6 +89,7 @@ patterns = [
bucket = "idf-artifacts"
patterns = [
'**/XUNIT_RESULT_*.xml',
'**/build_summary_*.xml',
]
[gitlab.artifacts.s3.env]
tools/ci/build_template_app.sh
+7
View File
@@ -11,6 +11,13 @@ set -euo pipefail
TEMPLATE_APP_PATH="esp-idf-template"
# Set default paths for template app builds
BUILD_PATH="${IDF_PATH}/build_template_app"
BUILD_DIR="${BUILD_PATH}/@t/@w"
LOG_PATH="${IDF_PATH}/log_template_app"
BUILD_LOG_CMAKE="${LOG_PATH}/cmake_@t_@w.txt"
gen_configs() {
# CONFIG_COMPILER_OPTIMIZATION_NONE with flag -O0
echo "CONFIG_COMPILER_OPTIMIZATION_NONE=y" > ${TEMPLATE_APP_PATH}/sdkconfig.ci.O0
tools/ci/dynamic_pipelines/constants.py
-4
View File
@@ -15,10 +15,6 @@ TOP_N_APPS_BY_SIZE_DIFF = 10
SIZE_DIFFERENCE_BYTES_THRESHOLD = 500
BINARY_SIZE_METRIC_NAME = 'binary_size'
RETRY_JOB_PICTURE_PATH = 'tools/ci/dynamic_pipelines/templates/retry-jobs.png'
RETRY_JOB_TITLE = '\n\nRetry failed jobs with with help of "retry_failed_jobs" stage of the pipeline:'
RETRY_JOB_PICTURE_LINK = '![Retry Jobs Image]({pic_url})'
KNOWN_GENERATE_TEST_CHILD_PIPELINE_WARNINGS_FILEPATH = os.path.join(
IDF_PATH, 'tools', 'ci', 'dynamic_pipelines', 'templates', 'known_generate_test_child_pipeline_warnings.yml'
)
tools/ci/dynamic_pipelines/report.py
+3 -19
View File
@@ -19,16 +19,12 @@ from .constants import COMMENT_START_MARKER
from .constants import CSS_STYLES_FILEPATH
from .constants import JS_SCRIPTS_FILEPATH
from .constants import REPORT_TEMPLATE_FILEPATH
from .constants import RETRY_JOB_PICTURE_LINK
from .constants import RETRY_JOB_PICTURE_PATH
from .constants import RETRY_JOB_TITLE
from .constants import SIZE_DIFFERENCE_BYTES_THRESHOLD
from .constants import TOP_N_APPS_BY_SIZE_DIFF
from .models import GitlabJob
from .models import TestCase
from .utils import format_permalink
from .utils import get_artifacts_url
from .utils import get_repository_file_url
from .utils import is_url
@@ -286,25 +282,13 @@ class ReportGenerator:
return comment
def _update_mr_comment(self, comment: str, print_retry_jobs_message: bool) -> None:
retry_job_picture_comment = (f'{RETRY_JOB_TITLE}\n\n{RETRY_JOB_PICTURE_LINK}').format(
pic_url=get_repository_file_url(RETRY_JOB_PICTURE_PATH)
)
del_retry_job_pic_pattern = re.escape(RETRY_JOB_TITLE) + r'.*?' + re.escape(f'{RETRY_JOB_PICTURE_PATH})')
def _update_mr_comment(self, comment: str) -> None:
new_comment = f'{COMMENT_START_MARKER}\n\n{comment}'
if print_retry_jobs_message:
new_comment += retry_job_picture_comment
for note in self.mr.notes.list(iterator=True):
if note.body.startswith(COMMENT_START_MARKER):
updated_str = self._get_updated_comment(note.body, comment)
# Add retry job message only if any job has failed
if print_retry_jobs_message:
updated_str = re.sub(del_retry_job_pic_pattern, '', updated_str, flags=re.DOTALL)
updated_str += retry_job_picture_comment
note.body = updated_str
try:
note.save()
@@ -321,7 +305,7 @@ class ReportGenerator:
updated_str = f'{existing_comment.strip()}\n\n{new_comment}'
return updated_str
def post_report(self, print_retry_jobs_message: bool = False) -> None:
def post_report(self) -> None:
comment = self._generate_comment()
print(comment)
@@ -330,7 +314,7 @@ class ReportGenerator:
print('No MR found, skip posting comment')
return
self._update_mr_comment(comment, print_retry_jobs_message=print_retry_jobs_message)
self._update_mr_comment(comment)
class BuildReportGenerator(ReportGenerator):
tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py
-82
View File
@@ -1,82 +0,0 @@
# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
"""This file is used for generating the child pipeline for build jobs."""
import argparse
import logging
import os
import __init__ # noqa: F401 # inject the system path
from idf_build_apps.utils import semicolon_separated_str_to_list
from idf_ci.idf_gitlab import build_child_pipeline
from idf_ci.utils import setup_logging
from idf_ci_utils import IDF_PATH
BUILD_CHILD_PIPELINE_FILEPATH = os.path.join(IDF_PATH, 'build_child_pipeline.yml')
TEST_PATHS = ['examples', os.path.join('tools', 'test_apps'), 'components']
def _separate_str_to_list(s: str) -> list[str]:
"""
Gitlab env file will escape the doublequotes in the env file, so we need to remove them
For example,
in pipeline.env file we have
MR_MODIFIED_COMPONENTS="app1;app2"
MR_MODIFIED_FILES="main/app1.c;main/app2.c"
gitlab will load the doublequotes as well, so we need to remove the doublequotes
"""
return semicolon_separated_str_to_list(s.strip('"')) # type: ignore
def main(arguments: argparse.Namespace) -> None:
setup_logging(logging.DEBUG)
build_child_pipeline(
paths=args.paths,
modified_files=args.modified_files,
compare_manifest_sha_filepath=args.compare_manifest_sha_filepath,
yaml_output=args.yaml_output,
)
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description='Generate build child pipeline',
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument(
'-o',
'--yaml-output',
default=BUILD_CHILD_PIPELINE_FILEPATH,
help='Output YAML path',
)
# use relative path to avoid absolute path in pipeline
parser.add_argument(
'-p',
'--paths',
nargs='+',
default=TEST_PATHS,
help='Paths to the apps to build.',
)
parser.add_argument(
'--compare-manifest-sha-filepath',
default=os.path.join(IDF_PATH, '.manifest_sha'),
help='Path to the recorded manifest sha file generated by `idf-build-apps dump-manifest-sha`',
)
parser.add_argument(
'--modified-files',
type=_separate_str_to_list,
default=os.getenv('MR_MODIFIED_FILES'),
help='semicolon-separated string which specifies the modified files. '
'app with `depends_filepatterns` set in the corresponding manifest files would only be built '
'if any of the specified file pattern matches any of the specified modified files. '
'If set to "", the value would be considered as None. '
'If set to ";", the value would be considered as an empty list',
)
args = parser.parse_args()
main(args)
tools/ci/dynamic_pipelines/scripts/generate_report.py
+1 -1
View File
@@ -110,7 +110,7 @@ def generate_jobs_report(args: argparse.Namespace) -> None:
report_generator = JobReportGenerator(
args.project_id, args.mr_iid, args.pipeline_id, args.job_id, args.commit_id, args.local_commit_id, jobs=jobs
)
report_generator.post_report(print_retry_jobs_message=any(job.is_failed for job in jobs))
report_generator.post_report()
if GitlabEnvVars().IDF_CI_IS_DEBUG_PIPELINE:
print('Debug pipeline detected, exit non-zero to fail the pipeline in order to block merge')
tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml
+1 -1
View File
@@ -7,7 +7,7 @@
.dynamic_build_template:
extends:
- .before_script:build
- .after_script:build:ccache-show-stats:upload-failed-job-logs
- .after_script:build
image: $ESP_ENV_IMAGE
tags: [build, shiny]
stage: build
tools/ci/dynamic_pipelines/templates/retry-jobs.png
BIN
View File
Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 17 KiB

tools/ci/check_soc_headers_load_in_idf_build_apps.py → tools/ci/test_soc_headers_load_in_idf_build_apps.py
+2 -1
View File
@@ -13,7 +13,8 @@ class TestLoadSocHeaders(unittest.TestCase):
def test_targets(self) -> None:
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
from idf_py_actions.constants import PREVIEW_TARGETS, SUPPORTED_TARGETS
from idf_py_actions.constants import PREVIEW_TARGETS
from idf_py_actions.constants import SUPPORTED_TARGETS
self.assertEqual(IBA_PREVIEW_TARGETS, PREVIEW_TARGETS)
self.assertEqual(IBA_SUPPORTED_TARGETS, SUPPORTED_TARGETS)
tools/ci/utils.sh
+19
View File
@@ -96,6 +96,25 @@ function run_cmd() {
fi
}
function pytest_for_ut() {
pytest_args="-c NUL -p no:idf-ci -p no:pytest_embedded"
if [ -n "${CI_JOB_ID-}" ]; then
if [ -z "${KNOWN_FAILURE_CASES_FILE_NAME-}" ]; then
echo "Error: KNOWN_FAILURE_CASES_FILE_NAME is not set."
return 1
fi
pytest_args="${pytest_args} --junitxml XUNIT_RESULT_${CI_JOB_ID}.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}"
if [ ! -e "${KNOWN_FAILURE_CASES_FILE_NAME}" ]; then
run_cmd idf-ci gitlab download-known-failure-cases-file "${KNOWN_FAILURE_CASES_FILE_NAME}"
fi
fi
run_cmd pytest "$@" "${pytest_args}"
}
# Retries a command RETRY_ATTEMPTS times in case of failure
# Inspired by https://stackoverflow.com/a/8351489
function retry_failed() {