From 6ff6d7ad6c33c7314ea527a565961bced3e4d5a0 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Wed, 11 Feb 2026 14:48:58 +0100 Subject: [PATCH] ci: backport master changes to 6.0 --- .gitlab-ci.yml | 20 +- .gitlab/ci/build.yml | 13 +- .gitlab/ci/common.yml | 158 +-- .gitlab/ci/deploy.yml | 63 -- .gitlab/ci/docs.yml | 85 +- .gitlab/ci/host-test.yml | 2 - .gitlab/ci/pre_check.yml | 65 +- .gitlab/ci/static-code-analysis.yml | 11 + .../templates/.dynamic_jobs.yml | 10 +- .../templates/test_child_pipeline.yml | 1 + tools/ci/exclude_check_tools_files.txt | 1 - tools/ci/gen_disabled_report.py | 980 ------------------ tools/ci/idf_ci_utils.py | 50 +- tools/ci/utils.sh | 9 +- 14 files changed, 126 insertions(+), 1342 deletions(-) delete mode 100644 tools/ci/gen_disabled_report.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7c02159d1a..e55ad1825d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,21 +1,11 @@ -workflow: - rules: - # Disable those non-protected push triggered pipelines - - if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && $CI_COMMIT_TAG !~ /^qa-test/ && $CI_PIPELINE_SOURCE == "push"' - when: never - # merged result pipelines - - if: $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA - variables: - PIPELINE_COMMIT_SHA: $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA - # else - - if: $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA == null || $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA == "" - variables: - PIPELINE_COMMIT_SHA: $CI_COMMIT_SHA - - when: always - # Place the default settings in `.gitlab/ci/common.yml` instead include: + - project: "ci/actions/common" + file: + - "templates/stable/default-workflow.yml" + - "templates/idf/deploy-github.yml" + - "templates/idf/deploy-docs.yml" - ".gitlab/ci/danger.yml" - ".gitlab/ci/common.yml" - ".gitlab/ci/rules.yml" diff --git a/.gitlab/ci/build.yml b/.gitlab/ci/build.yml index c44fcb70d7..b67f345368 100644 --- a/.gitlab/ci/build.yml +++ b/.gitlab/ci/build.yml @@ -1,7 +1,6 @@ .build_template: stage: build extends: - - .before_script:build - .after_script:build image: $ESP_ENV_IMAGE tags: [build, shiny] @@ -155,21 +154,17 @@ build_child_pipeline: job: generate_build_child_pipeline strategy: depend -generate_disabled_apps_report: +generate_prebuild_report: extends: - .build_template tags: [fast_run, shiny] - dependencies: # set dependencies to null to avoid missing artifacts issue needs: - pipeline_variables - - job: baseline_manifest_sha - optional: true artifacts: paths: - - disabled_report.html + - prebuild_report.html expire_in: 1 week when: always script: - - pip install dominate idf-build-apps - - run_cmd python tools/ci/gen_disabled_report.py --output disabled_report.html --verbose --enable-preview-targets - - echo "Report generated at https://${CI_PAGES_HOSTNAME}:${CI_SERVER_PORT}/-/esp-idf/-/jobs/${CI_JOB_ID}/artifacts/disabled_report.html" + - run_cmd idf-ci build collect --format html -o prebuild_report.html + - echo "Report generated at https://${CI_PAGES_HOSTNAME}:${CI_SERVER_PORT}/-/esp-idf/-/jobs/${CI_JOB_ID}/artifacts/prebuild_report.html" diff --git a/.gitlab/ci/common.yml b/.gitlab/ci/common.yml index 60533ffa2a..0cd6b5b320 100644 --- a/.gitlab/ci/common.yml +++ b/.gitlab/ci/common.yml @@ -34,8 +34,7 @@ variables: # since we're using merged-result pipelines, the last commit should work for most cases # --prune --prune-tags: in case remote branch or tag is force pushed GIT_FETCH_EXTRA_FLAGS: "--no-recurse-submodules --prune --prune-tags" - # we're using .cache folder for caches - GIT_CLEAN_FLAGS: -ffdx -e .cache/ + LATEST_GIT_TAG: v6.0-rc1 SUBMODULE_FETCH_TOOL: "tools/ci/ci_fetch_submodule.py" @@ -168,9 +167,6 @@ variables: # Done after sourcing export.sh so that we could easily invoke the right pip section_start "upgrade_ci_dependencies" "Upgrading CI dependencies" pip install --upgrade --upgrade-strategy=eager -r $IDF_PATH/tools/requirements/requirements.ci.txt -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE} - if [[ "${CI_JOB_STAGE}" == "target_test" ]]; then - pip install --upgrade --upgrade-strategy=eager -r $IDF_PATH/tools/requirements/requirements.test-specific.txt -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE} - fi section_end "upgrade_ci_dependencies" REEXPORT_NEEDED=0 @@ -245,155 +241,20 @@ variables: before_script: - *common-before_scripts -.before_script:build: - before_script: - - *common-before_scripts - - *setup_tools_and_idf_python_venv - - add_gitlab_ssh_keys - - fetch_submodules - - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS} - - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS} - .after_script:build: after_script: - source tools/ci/utils.sh - *show_ccache_statistics - *upload_failed_job_log_artifacts -############################## -# Git Strategy Job Templates # -############################## -.git_init: &git_init | - mkdir -p "${CI_PROJECT_DIR}" - cd "${CI_PROJECT_DIR}" - git init - -.git_fetch_from_mirror_url_if_exists: &git_fetch_from_mirror_url_if_exists | - # check if set mirror - if [ -n "${LOCAL_GITLAB_HTTPS_HOST:-}" ] && [ -n "${ESPCI_TOKEN:-}" ]; then - MIRROR_REPO_URL="https://bot:${ESPCI_TOKEN}@${LOCAL_GITLAB_HTTPS_HOST}/${CI_PROJECT_PATH}" - elif [ -n "${LOCAL_GIT_MIRROR:-}" ]; then - MIRROR_REPO_URL="${LOCAL_GIT_MIRROR}/${CI_PROJECT_PATH}" - fi - - # fetch from mirror first if set - if [ -n "${MIRROR_REPO_URL:-}" ]; then - if git remote -v | grep origin; then - git remote set-url origin "${MIRROR_REPO_URL}" - else - git remote add origin "${MIRROR_REPO_URL}" - fi - # mirror url may fail with authentication issue - git fetch origin --no-recurse-submodules || true - fi - - # set remote url to CI_REPOSITORY_URL - if git remote -v | grep origin; then - git remote set-url origin "${CI_REPOSITORY_URL}" - else - git remote add origin "${CI_REPOSITORY_URL}" - fi - -.git_checkout_ci_commit_sha: &git_checkout_ci_commit_sha | - git checkout $CI_COMMIT_SHA - eval "git clean ${GIT_CLEAN_FLAGS}" - -# git diff requires two commits, with different CI env var -# -# By default, we use git strategy "clone" with depth 1 to speed up the clone process. -# But for jobs requires running `git diff`, we need to fetch more commits to get the correct diffs. -# -# Since there's no way to get the correct git_depth before the job starts, -# we can't set `GIT_DEPTH` in the job definition. -# -# Set git strategy to "none" and fetch manually instead. -.before_script:fetch:git_diff: - variables: - GIT_STRATEGY: none - before_script: - - *git_init - - *git_fetch_from_mirror_url_if_exists - - | - # Store the diff output in a temporary file - TEMP_FILE=$(mktemp) - # merged results pipelines, by default - if [[ -n $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA ]]; then - git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_SHA - git fetch origin $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA - - git diff --name-only $CI_MERGE_REQUEST_TARGET_BRANCH_SHA...$CI_MERGE_REQUEST_SOURCE_BRANCH_SHA > "$TEMP_FILE" - GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE") - git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} - # merge request pipelines, when the mr got conflicts - elif [[ -n $CI_MERGE_REQUEST_DIFF_BASE_SHA ]]; then - git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} - git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} - git diff --name-only $CI_MERGE_REQUEST_DIFF_BASE_SHA $CI_COMMIT_SHA > "$TEMP_FILE" - GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE") - # other pipelines, like the protected branches pipelines - elif [[ "$CI_COMMIT_BEFORE_SHA" != "0000000000000000000000000000000000000000" ]]; then - git fetch origin $CI_COMMIT_BEFORE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} - git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} - git diff --name-only $CI_COMMIT_BEFORE_SHA $CI_COMMIT_SHA > "$TEMP_FILE" - GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE") - else - # pipeline source could be web, scheduler, etc. - git fetch origin $CI_COMMIT_SHA --depth=2 ${GIT_FETCH_EXTRA_FLAGS} - git diff --name-only $CI_COMMIT_SHA~1 $CI_COMMIT_SHA > "$TEMP_FILE" - GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE") - fi - - *git_checkout_ci_commit_sha - - *common-before_scripts - - *setup_tools_and_idf_python_venv - - add_gitlab_ssh_keys - -# target test runners may locate in different places -# for runners set git mirror, we fetch from the mirror first, then fetch the HEAD commit -.before_script:fetch:target_test: - variables: - GIT_STRATEGY: none - before_script: - - *git_init - - *git_fetch_from_mirror_url_if_exists - - eval "git fetch --depth=1 ${GIT_FETCH_EXTRA_FLAGS} origin ${CI_COMMIT_SHA}" - - *git_checkout_ci_commit_sha - - *common-before_scripts - - *setup_tools_and_idf_python_venv - - add_gitlab_ssh_keys - # no submodules - .brew-macos-settings: variables: - GIT_STRATEGY: none # we do manual git clone to use local mirror + GIT_STRATEGY: fetch IDF_CCACHE_ENABLE: "0" CCACHE_DIR: "/var/tmp/cache/idf_ccache" tags: - macos-tart image: macos-sequoia-idf-v6.0 - cache: [] # pip cache is created under amd64, and submodules are downloaded with brew mirror, so disable cache here - before_script: - # assert LOCAL_GIT_MIRROR is set - - echo -e "section_start:`date +%s`:check_out\r\e[0Kchecking out from local git mirror, then reset to CI_COMMIT_SHA" - - | - if [ -z "${LOCAL_GIT_MIRROR:-}" ]; then - echo "Error: LOCAL_GIT_MIRROR not set, cannot clone from mirror." - exit 1 - fi - - MIRROR_REPO_URL="${LOCAL_GIT_MIRROR}/${CI_PROJECT_PATH}" - - cd "${CI_PROJECT_DIR}" - # since .cache exists in CI_PROJECT_DIR, so can't direct `git clone .` - - git clone -b ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:-${CI_COMMIT_BRANCH}} --depth=1 --recursive --shallow-submodules "${MIRROR_REPO_URL}" tmp - - mv tmp/.git ./ - - rm -rf tmp - - git reset --hard - # set remote url back - - git remote set-url origin "${CI_REPOSITORY_URL}" - - eval "git fetch --depth=1 ${GIT_FETCH_EXTRA_FLAGS} origin ${CI_COMMIT_SHA}" - - git checkout FETCH_HEAD - - git submodule update --init --recursive --depth=1 - - echo -e "section_end:`date +%s`:check_out\r\e[0K" - - *common-before_scripts - - *setup_tools_and_idf_python_venv after_script: [] # ccache now is disabled for macos brew runners timeout: 30m @@ -421,16 +282,5 @@ default: - *setup_tools_and_idf_python_venv - add_gitlab_ssh_keys - fetch_submodules -# gitlab bug, setting them here doesn't work -# - expire_in: https://gitlab.com/gitlab-org/gitlab/-/issues/404563 -# - when: https://gitlab.com/gitlab-org/gitlab/-/issues/440672 -# artifacts: -# expire_in: 1 week -# when: always - retry: - max: 2 - when: - # In case of a runner failure we could hop to another one, or a network error could go away. - - runner_system_failure - # Job execution timeout may be caused by a network issue. - - job_execution_timeout + - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS} + - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS} diff --git a/.gitlab/ci/deploy.yml b/.gitlab/ci/deploy.yml index 3406fe73bd..c25fa7a869 100644 --- a/.gitlab/ci/deploy.yml +++ b/.gitlab/ci/deploy.yml @@ -3,51 +3,6 @@ image: $ESP_ENV_IMAGE tags: [ deploy ] -check_submodule_sync: - extends: - - .deploy_job_template - - .rules:test:submodule - stage: test_deploy - tags: [ brew, github_sync ] - retry: 2 - variables: - GIT_STRATEGY: fetch # use brew local mirror first - SUBMODULES_TO_FETCH: "none" - PUBLIC_IDF_URL: "https://github.com/espressif/esp-idf.git" - dependencies: [] - script: - - git submodule deinit --force . - - rm -rf .git/modules # remove all the cached metadata - # setting the default remote URL to the public one, to resolve relative location URLs - - git config remote.origin.url ${PUBLIC_IDF_URL} - # check if all submodules are correctly synced to public repository - - git submodule init - - git config --get-regexp '^submodule\..*\.url$' || true - - git submodule update --recursive - - echo "IDF was cloned from ${PUBLIC_IDF_URL} completely" - -push_to_github: - extends: - - .deploy_job_template - - .before_script:minimal - - .rules:protected:deploy - needs: - # submodule must be synced before pushing to github - - check_submodule_sync - tags: [ brew, github_sync ] - variables: - GIT_STRATEGY: fetch # use brew local mirror first - GIT_DEPTH: 0 # github needs full record of commits - script: - - add_github_ssh_keys - - git remote remove github &>/dev/null || true - - git remote add github git@github.com:espressif/esp-idf.git - - tools/ci/push_to_github.sh - environment: - name: push_to_github_production - deployment_tier: production - url: "https://github.com/espressif/esp-idf" - deploy_update_SHA_in_esp-dockerfiles: extends: - .deploy_job_template @@ -64,21 +19,3 @@ deploy_update_SHA_in_esp-dockerfiles: environment: name: deploy_update_SHA_in_esp-dockerfiles_production deployment_tier: production - -upload_junit_report: - extends: - - .deploy_job_template - tags: [ fast_run, shiny ] - needs: - - pipeline_variables - - job: build_child_pipeline - artifacts: false - script: - - run_cmd idf-ci gitlab download-artifacts --type junit - rules: - - when: always - artifacts: - reports: - junit: XUNIT_RESULT_*.xml - expire_in: 1 week - when: always diff --git a/.gitlab/ci/docs.yml b/.gitlab/ci/docs.yml index 35121a2e5a..9e8762fc0c 100644 --- a/.gitlab/ci/docs.yml +++ b/.gitlab/ci/docs.yml @@ -16,9 +16,6 @@ .patterns-example-readme: &patterns-example-readme - "examples/**/*.md" -.patterns-docs-preview: &patterns-docs-preview - - "docs/**/*" - .if-protected-check: &if-protected-check if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/)' @@ -58,7 +55,9 @@ check_readme_links: extends: - .pre_check_template - tags: ["amd64", "brew"] + variables: + GIT_STRATEGY: "fetch" + tags: ["check_doc_links"] allow_failure: true rules: - <<: *if-protected-check @@ -100,6 +99,9 @@ check_docs_gh_links: extends: - .pre_check_template - .doc-rules:build:docs-full + variables: + GIT_STRATEGY: "fetch" + tags: ["check_doc_links"] script: - cd docs - build-docs gh-linkcheck @@ -138,77 +140,6 @@ build_docs_html_partial: - DOCLANG: "zh_CN" DOCTGT: "esp32p4" -.deploy_docs_template: - image: $ESP_IDF_DOC_ENV_IMAGE - variables: - DOCS_BUILD_DIR: "${IDF_PATH}/docs/_build/" - PYTHONUNBUFFERED: 1 - # ensure all tags are fetched, need to know the latest/stable tag for the docs - GIT_STRATEGY: clone - GIT_DEPTH: 0 - stage: test_deploy - tags: - - brew - - amd64 - script: - # ensure all tags are fetched, need to know the latest/stable tag for the docs - - git fetch --tags --prune - - add_doc_server_ssh_keys $DOCS_DEPLOY_PRIVATEKEY $DOCS_DEPLOY_SERVER $DOCS_DEPLOY_SERVER_USER - - export GIT_VER=$(git describe --always ${PIPELINE_COMMIT_SHA} --) - - deploy-docs - -# stage: test_deploy -deploy_docs_preview: - extends: - - .deploy_docs_template - rules: - - <<: *if-label-build_docs - - <<: *if-label-docs_full - - <<: *if-dev-push - changes: *patterns-docs-preview - needs: - - job: build_docs_html_partial - optional: true - - job: build_docs_html_full - optional: true - variables: - TYPE: "preview" - # older branches use DOCS_DEPLOY_KEY, DOCS_SERVER, DOCS_SERVER_USER, DOCS_PATH for preview server so we keep these names for 'preview' - DOCS_DEPLOY_PRIVATEKEY: "$DOCS_DEPLOY_KEY" - DOCS_DEPLOY_SERVER: "$DOCS_SERVER" - DOCS_DEPLOY_SERVER_USER: "$DOCS_SERVER_USER" - DOCS_DEPLOY_PATH: "$DOCS_PATH" - DOCS_DEPLOY_URL_BASE: "https://$DOCS_PREVIEW_SERVER_URL/docs/esp-idf" - environment: - name: deploy_docs_preview - deployment_tier: staging - url: "https://$DOCS_PREVIEW_SERVER_URL/docs/esp-idf" - -# stage: post_deploy -deploy_docs_production: - # The DOCS_PROD_* variables used by this job are "Protected" so these branches must all be marked "Protected" in Gitlab settings - extends: - - .deploy_docs_template - - .rules:protected:deploy - stage: post_deploy - dependencies: # set dependencies to null to avoid missing artifacts issue - needs: # ensure runs after push_to_github succeeded - - build_docs_html_full - - job: push_to_github - artifacts: false - variables: - TYPE: "preview" - DOCS_DEPLOY_PRIVATEKEY: "$DOCS_PROD_DEPLOY_KEY" - DOCS_DEPLOY_SERVER: "$DOCS_PROD_SERVER" - DOCS_DEPLOY_SERVER_USER: "$DOCS_PROD_SERVER_USER" - DOCS_DEPLOY_PATH: "$DOCS_PROD_PATH" - DOCS_DEPLOY_URL_BASE: "https://docs.espressif.com/projects/esp-idf" - DEPLOY_STABLE: 1 - environment: - name: deploy_docs_production - deployment_tier: production - url: "https://docs.espressif.com/projects/esp-idf" - check_doc_links: extends: - .build_docs_template @@ -217,7 +148,9 @@ check_doc_links: needs: - job: deploy_docs_production artifacts: false - tags: ["build", "amd64", "internet"] + variables: + GIT_STRATEGY: "fetch" + tags: ["check_doc_links"] artifacts: when: always paths: diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index acef80cfde..97a0ab0f61 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -320,7 +320,6 @@ build_docker: test_pytest_qemu: extends: - .host_test_template - - .before_script:build artifacts: paths: - XUNIT_RESULT.xml @@ -355,7 +354,6 @@ test_pytest_qemu: test_pytest_linux: extends: - .host_test_template - - .before_script:build artifacts: paths: - XUNIT_RESULT.xml diff --git a/.gitlab/ci/pre_check.yml b/.gitlab/ci/pre_check.yml index 49c3920ebd..5f87d341e5 100644 --- a/.gitlab/ci/pre_check.yml +++ b/.gitlab/ci/pre_check.yml @@ -115,17 +115,67 @@ check_version_tag: check_test_scripts_build_test_rules: extends: - .pre_check_template - - .before_script:build script: # requires basic pytest dependencies - python tools/ci/check_build_test_rules.py check-test-scripts examples/ tools/test_apps components +check_submodule_sync: + extends: + - .pre_check_template + - .rules:test:submodule + tags: [ brew, github_sync ] + retry: 2 + variables: + GIT_STRATEGY: fetch # use brew local mirror first + SUBMODULES_TO_FETCH: "none" + PUBLIC_IDF_URL: "https://github.com/espressif/esp-idf.git" + script: + - git submodule deinit --force . + - rm -rf .git/modules # remove all the cached metadata + # setting the default remote URL to the public one, to resolve relative location URLs + - git config remote.origin.url ${PUBLIC_IDF_URL} + # check if all submodules are correctly synced to public repository + - git submodule init + - git config --get-regexp '^submodule\..*\.url$' || true + - git submodule update --recursive + - echo "IDF was cloned from ${PUBLIC_IDF_URL} completely" + pipeline_variables: extends: - .pre_check_template - - .before_script:fetch:git_diff tags: [fast_run, shiny] + variables: + GIT_STRATEGY: "fetch" script: + - | + # Store the diff output in a temporary file + TEMP_FILE=$(mktemp) + # merged results pipelines, by default + if [[ -n $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA ]]; then + git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_SHA + git fetch origin $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA + + git diff --name-only $CI_MERGE_REQUEST_TARGET_BRANCH_SHA...$CI_MERGE_REQUEST_SOURCE_BRANCH_SHA > "$TEMP_FILE" + GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE") + git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} + # merge request pipelines, when the mr got conflicts + elif [[ -n $CI_MERGE_REQUEST_DIFF_BASE_SHA ]]; then + git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} + git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} + git diff --name-only $CI_MERGE_REQUEST_DIFF_BASE_SHA $CI_COMMIT_SHA > "$TEMP_FILE" + GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE") + # other pipelines, like the protected branches pipelines + elif [[ "$CI_COMMIT_BEFORE_SHA" != "0000000000000000000000000000000000000000" ]]; then + git fetch origin $CI_COMMIT_BEFORE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} + git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} + git diff --name-only $CI_COMMIT_BEFORE_SHA $CI_COMMIT_SHA > "$TEMP_FILE" + GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE") + else + # pipeline source could be web, scheduler, etc. + git fetch origin $CI_COMMIT_SHA --depth=2 ${GIT_FETCH_EXTRA_FLAGS} + git diff --name-only $CI_COMMIT_SHA~1 $CI_COMMIT_SHA > "$TEMP_FILE" + GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE") + fi # MODIFIED_FILES is a list of files that changed, could be used everywhere - MODIFIED_FILES=$(echo "$GIT_DIFF_OUTPUT" | xargs) - echo "MODIFIED_FILES=$MODIFIED_FILES" >> pipeline.env @@ -191,17 +241,6 @@ baseline_manifest_sha: expire_in: 1 week when: always -gcc_static_analyzer: - extends: - - .pre_check_template - - .rules:build - variables: - CI_CCACHE_DISABLE: 1 - ANALYZING_APP: "examples/get-started/hello_world" - script: - - echo "CONFIG_COMPILER_STATIC_ANALYZER=y" >> ${ANALYZING_APP}/sdkconfig.defaults - - idf-build-apps build -p ${ANALYZING_APP} - retry_failed_jobs: extends: - .pre_check_template diff --git a/.gitlab/ci/static-code-analysis.yml b/.gitlab/ci/static-code-analysis.yml index 15eabc1598..6666ff884d 100644 --- a/.gitlab/ci/static-code-analysis.yml +++ b/.gitlab/ci/static-code-analysis.yml @@ -16,6 +16,17 @@ clang_tidy_check: --limit-file tools/ci/static-analysis-rules.yml --xtensa-include-dir +gcc_static_analyzer: + extends: + - .pre_check_template + - .rules:patterns:clang_tidy + variables: + CI_CCACHE_DISABLE: 1 + ANALYZING_APP: "examples/get-started/hello_world" + script: + - echo "CONFIG_COMPILER_STATIC_ANALYZER=y" >> ${ANALYZING_APP}/sdkconfig.defaults + - idf-build-apps build -p ${ANALYZING_APP} + # ## build stage ## Sonarqube related jobs put here for this reason: diff --git a/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml index af3e16bd77..0e50de03a3 100644 --- a/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml +++ b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml @@ -6,7 +6,6 @@ ######################## .dynamic_build_template: extends: - - .before_script:build - .after_script:build image: $ESP_ENV_IMAGE tags: [build, shiny] @@ -44,12 +43,11 @@ --modified-files ${MR_MODIFIED_FILES} .dynamic_target_test_template: - extends: - - .before_script:fetch:target_test image: $TARGET_TEST_ENV_IMAGE stage: target_test timeout: 1 hour variables: + GIT_STRATEGY: "fetch" SUBMODULES_TO_FETCH: "none" # set while generating the pipeline nodes: "" @@ -68,10 +66,8 @@ paths: - XUNIT_RESULT*.xml - pytest-embedded/ - # Child pipeline reports won't be collected in the main one - # https://gitlab.com/groups/gitlab-org/-/epics/8205 - # reports: - # junit: XUNIT_RESULT.xml + reports: + junit: XUNIT_RESULT*.xml when: always expire_in: 1 week script: diff --git a/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml b/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml index 18f41ddbf0..f986108c54 100644 --- a/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml +++ b/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml @@ -12,6 +12,7 @@ all_build_finished: expire_in: 1 week when: always before_script: [] + dependencies: [] script: - echo "all test jobs finished" diff --git a/tools/ci/exclude_check_tools_files.txt b/tools/ci/exclude_check_tools_files.txt index 5ccf75a904..1eac90962a 100644 --- a/tools/ci/exclude_check_tools_files.txt +++ b/tools/ci/exclude_check_tools_files.txt @@ -17,7 +17,6 @@ tools/ci/dynamic_pipelines/**/* tools/ci/envsubst.py tools/ci/executable-list.txt tools/ci/fix_empty_prototypes.sh -tools/ci/gen_disabled_report.py tools/ci/generate_rules.py tools/ci/get-full-sources.sh tools/ci/get_all_test_results.py diff --git a/tools/ci/gen_disabled_report.py b/tools/ci/gen_disabled_report.py deleted file mode 100644 index 570eb90382..0000000000 --- a/tools/ci/gen_disabled_report.py +++ /dev/null @@ -1,980 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 - -import argparse -import fnmatch -import json -import os -import subprocess -import sys -from collections import Counter -from datetime import datetime - -from dominate import document -from dominate.tags import a -from dominate.tags import button -from dominate.tags import div -from dominate.tags import h1 -from dominate.tags import h3 -from dominate.tags import input_ -from dominate.tags import label -from dominate.tags import li -from dominate.tags import script -from dominate.tags import span -from dominate.tags import strong -from dominate.tags import style -from dominate.tags import table -from dominate.tags import tbody -from dominate.tags import td -from dominate.tags import th -from dominate.tags import thead -from dominate.tags import tr -from dominate.tags import ul -from dominate.util import raw -from idf_build_apps import App -from idf_build_apps.app import AppDeserializer - - -def run_idf_build_apps_find( - output_file: str = 'apps.json', - verbose: bool = False, - enable_preview_targets: bool = False, -) -> str: - """ - Run idf-build-apps find command to get application list - - :param output_file: Output file path - :param verbose: Whether to enable verbose output - :param enable_preview_targets: Whether to enable preview targets - :return: Output file path - """ - cmd = [ - 'idf-build-apps', - 'find', - '--output-format', - 'json', - '--include-all-apps', - '--output', - output_file, - ] - - # Add verbose parameter - if verbose: - cmd.append('--verbose') - - # Add enable_preview_targets parameter - if enable_preview_targets: - cmd.append('--enable-preview-targets') - - print(f'Running command: {" ".join(cmd)}') - - try: - result = subprocess.run(cmd, check=True, capture_output=True, text=True) - print('Command completed successfully') - print(f'stdout: {result.stdout}') - if result.stderr: - print(f'stderr: {result.stderr}') - return output_file - except subprocess.CalledProcessError as e: - print(f'Command failed with exit code {e.returncode}') - print(f'stdout: {e.stdout}') - print(f'stderr: {e.stderr}') - raise - - -def parse_codeowners(codeowners_path: str) -> list[tuple[str, list[str]]]: - """ - Parse CODEOWNERS file and return a list of (pattern, codeowners) tuples. - The list preserves the order of the rules in the file. - - :param codeowners_path: Path to CODEOWNERS file - :return: List of (pattern, codeowners) tuples - """ - codeowners_mapping: list[tuple[str, list[str]]] = [] - - if not os.path.exists(codeowners_path): - print(f'Warning: CODEOWNERS file not found at {codeowners_path}') - return codeowners_mapping - - try: - with open(codeowners_path, encoding='utf-8') as f: - for line_num, line in enumerate(f, 1): - line = line.strip() - - # Skip empty lines and comments - if not line or line.startswith('#'): - continue - - # Parse pattern and codeowners - parts = line.split() - if len(parts) < 2: - print(f'Warning: Invalid CODEOWNERS line {line_num}: {line}') - continue - - pattern = parts[0] - codeowners = parts[1:] - - # Remove @ symbol and the long prefix from codeowners if present - codeowners = [owner.lstrip('@').replace('esp-idf-codeowners/', '') for owner in codeowners] - - codeowners_mapping.append((pattern, codeowners)) - - except Exception as e: - print(f'Error reading CODEOWNERS file: {e}') - - return codeowners_mapping - - -def match_codeowners(app_dir: str, codeowners_mapping: list[tuple[str, list[str]]]) -> list[str]: - """ - Match app directory against CODEOWNERS patterns, supporting wildcards ('*' and '**'). - The matching traverses up from the app's directory and respects the 'last match wins' rule. - - :param app_dir: Application directory path (e.g., './components/esp_driver_touch_sens/test_apps/touch_sens') - :param codeowners_mapping: List of (pattern, codeowners) tuples, in file order - :return: List of matching codeowners from the last matching rule - """ - # 1. Normalize the app_dir path - # Remove leading './' and ensure it starts with a single '/' - if app_dir.startswith('./'): - app_dir = app_dir[1:] - if not app_dir.startswith('/'): - app_dir = '/' + app_dir - app_dir = os.path.normpath(app_dir) - - # 2. Generate all parent paths - parent_paths = [] - current_path = app_dir - while True: - parent_paths.append(current_path) - if current_path == '/': - break - parent, _ = os.path.split(current_path) - if parent == current_path: # Root reached - break - current_path = parent - - # 3. Iterate through rules in reverse to find the last match - for pattern, owners in reversed(codeowners_mapping): - # The pattern needs to be anchored to the root for matching - if not pattern.startswith('/'): - pattern = '/**/' + pattern # Handles patterns like '*.py' or 'docs' anywhere - - for path in parent_paths: - if fnmatch.fnmatch(path, pattern) or fnmatch.fnmatch(path + '/', pattern): - return owners - - # Fallback to the root-level wildcard rule if no other match is found - for pattern, owners in reversed(codeowners_mapping): - if pattern == '*': - return owners - - return [] - - -def add_codeowners_to_apps(apps: list[App], codeowners_mapping: list[tuple[str, list[str]]]) -> dict[str, list[str]]: - """ - Match apps to codeowners and return a dictionary mapping app paths to their owners. - If an app does not match any rule, it is assigned to the 'others' group. - - :param apps: List of App objects - :param codeowners_mapping: List of (pattern, codeowners) tuples, in file order - :return: A dictionary mapping app.app_dir to a list of codeowners - """ - app_to_owners_map = {} - for app in apps: - matched_owners = match_codeowners(app.app_dir, codeowners_mapping) - if not matched_owners: - app_to_owners_map[app.app_dir] = ['others'] - else: - app_to_owners_map[app.app_dir] = matched_owners - return app_to_owners_map - - -def load_apps_from_json(json_file: str) -> list[App]: - """ - Load application list from JSON file (clean version without codeowners) - """ - apps = [] - - with open(json_file, encoding='utf-8') as f: - if json_file.endswith('.json'): - # If it's JSON format, try to parse as array - try: - data = json.load(f) - if isinstance(data, list): - for app_data in data: - # Handle build_system compatibility: 'idf_cmake' -> 'cmake' - if app_data.get('build_system') == 'idf_cmake': - app_data['build_system'] = 'cmake' - app = AppDeserializer.from_json(json.dumps(app_data)) - apps.append(app) - else: - # If not an array, try line-by-line parsing - f.seek(0) - for line in f: - line = line.strip() - if line: - app = AppDeserializer.from_json(line) - apps.append(app) - except json.JSONDecodeError: - # If JSON parsing fails, try line-by-line parsing - f.seek(0) - for line in f: - line = line.strip() - if line: - try: - app = AppDeserializer.from_json(line) - apps.append(app) - except Exception as e: - print(f'Warning: Failed to parse line: {line[:100]}... Error: {e}') - else: - # If not JSON format, parse line by line - for line in f: - line = line.strip() - if line: - try: - app = AppDeserializer.from_json(line) - apps.append(app) - except Exception as e: - print(f'Warning: Failed to parse line: {line[:100]}... Error: {e}') - - print(f'Loaded {len(apps)} apps from {json_file}') - return apps - - -def generate_disabled_report(apps: list[App], app_to_owners_map: dict[str, list[str]], report_path: str) -> None: - """Generate disabled report""" - # Categorize applications - cant_build_temp = [] - can_build_cant_test_temp = [] - cant_build_not_temp = [] - can_build_cant_test_not_temp = [] - can_test = [] - all_codeowners: set[str] = set() - all_targets: set[str] = set() - owner_app_counts: Counter[str] = Counter() - target_app_counts: Counter[str] = Counter() - - for app in apps: - # Get owners from the map - app_owners = app_to_owners_map.get(app.app_dir, []) - - # Update owner app counts - for owner in app_owners: - owner_app_counts[owner] += 1 - - # Update target app counts - all_targets.add(app.target) - target_app_counts[app.target] += 1 - - # Check for unsupported build_status values and abort if found - if app.build_status not in ['should be built', 'disabled']: - print( - f'ERROR: Found unsupported build_status "{app.build_status}" for app {app.app_dir}, target {app.target}' - ) - print( - 'This task only supports "should be built" and "disabled" status. Please fix the build configuration.' - ) - sys.exit(1) - - # Map build_status to the expected status values - if app.build_status == 'should be built': - if app.test_comment is None: - status = 'can_build_and_test' - else: - status = 'can_build_no_test' - else: # if app.build_status == 'disabled': - status = 'cannot_build' - - # Categorize apps based on status and temporary flags - if status == 'can_build_and_test': - can_test.append(app) - elif status == 'cannot_build': - # Handle cases where build_comment might be None for a disabled app - build_comment = app.build_comment or 'Reason not specified' - if 'temporary' in build_comment.lower(): - cant_build_temp.append(app) - else: - cant_build_not_temp.append(app) - elif status == 'can_build_no_test': - # Handle cases where test_comment might be None - test_comment = app.test_comment or 'Reason not specified' - if 'temporary' in test_comment.lower(): - can_build_cant_test_temp.append(app) - else: - can_build_cant_test_not_temp.append(app) - - all_codeowners.update(app_owners) - - # Create HTML document - doc = document(title='Build and Test Status Report') - - def get_css_styles() -> str: - """Return CSS styles""" - return """ - body { - font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; - line-height: 1.6; - margin: 0; - padding: 20px; - background-color: #f5f5f5; - } - .container { - max-width: 1200px; - margin: 0 auto; - background-color: white; - padding: 30px; - border-radius: 8px; - box-shadow: 0 2px 10px rgba(0,0,0,0.1); - } - h1 { - color: #333; - text-align: center; - margin-bottom: 30px; - border-bottom: 3px solid #007acc; - padding-bottom: 10px; - } - h2 { - color: #007acc; - margin-top: 40px; - margin-bottom: 20px; - padding: 10px; - background-color: #f8f9fa; - border-left: 4px solid #007acc; - border-radius: 4px; - } - .navigation { - background-color: #f8f9fa; - padding: 20px; - border-radius: 8px; - margin-bottom: 30px; - border: 1px solid #dee2e6; - } - .navigation h3 { - margin-top: 0; - color: #495057; - margin-bottom: 15px; - } - .nav-links { - display: flex; - flex-wrap: wrap; - gap: 10px; - } - .nav-link { - background-color: #007acc; - color: white; - padding: 8px 16px; - border-radius: 20px; - text-decoration: none; - font-size: 0.9em; - transition: background-color 0.3s; - } - .nav-link:hover { - background-color: #005a9e; - } - .control-buttons { - text-align: center; - margin: 20px 0; - } - .control-btn { - background-color: #28a745; - color: white; - border: none; - padding: 10px 20px; - border-radius: 5px; - cursor: pointer; - margin: 0 10px; - font-size: 0.9em; - transition: background-color 0.3s; - } - .control-btn:hover { - background-color: #218838; - } - .control-btn.collapse-all { - background-color: #dc3545; - } - .control-btn.collapse-all:hover { - background-color: #c82333; - } - .filters-wrapper { - display: flex; - gap: 20px; - margin-bottom: 20px; - justify-content: center; - } - .filter-container { - flex: 1; - max-width: 500px; - margin-bottom: 20px; - text-align: center; - padding: 15px; - border: 1px solid #dee2e6; - border-radius: 8px; - background-color: #f8f9fa; - } - #codeownerFilter { - padding: 10px; - border-radius: 5px; - border: 1px solid #ccc; - font-size: 1em; - } - table { - width: 100%; - border-collapse: collapse; - margin-bottom: 30px; - box-shadow: 0 2px 8px rgba(0,0,0,0.1); - } - th, td { - padding: 12px; - text-align: left; - border-bottom: 1px solid #ddd; - } - th { - background-color: #007acc; - color: white; - font-weight: 600; - } - tr:nth-child(even) { - background-color: #f8f9fa; - } - tr:hover { - background-color: #e3f2fd; - } - .summary { - background-color: #e8f5e8; - padding: 20px; - border-radius: 8px; - margin-bottom: 30px; - border-left: 4px solid #4caf50; - } - .summary h3 { - margin-top: 0; - color: #2e7d32; - } - .summary ul { - margin: 10px 0; - padding-left: 20px; - } - .summary li { - margin: 5px 0; - } - .category-summary { - background-color: #f8f9fa; - padding: 15px; - border-radius: 6px; - margin-bottom: 20px; - border-left: 4px solid #007acc; - } - .category-summary h3 { - margin: 0; - color: #007acc; - display: flex; - justify-content: space-between; - align-items: center; - } - .category-summary .count { - background-color: #007acc; - color: white; - padding: 4px 8px; - border-radius: 12px; - font-size: 0.9em; - font-weight: bold; - } - .category-details { - margin-left: 20px; - margin-bottom: 30px; - } - .directory-info { - background-color: #fff3cd; - padding: 10px; - border-radius: 4px; - margin-bottom: 15px; - border-left: 3px solid #ffc107; - font-family: monospace; - font-size: 0.9em; - cursor: pointer; - transition: background-color 0.3s; - } - .directory-info:hover { - background-color: #ffeaa7; - } - .total-count { - color: #666; - font-size: 0.9em; - margin-left: 10px; - } - .empty-section { - text-align: center; - color: #666; - font-style: italic; - padding: 20px; - background-color: #f8f9fa; - border-radius: 4px; - } - .timestamp { - text-align: center; - color: #666; - font-size: 0.9em; - margin-bottom: 30px; - } - .table-container { - margin-bottom: 20px; - } - .table-header { - background-color: #f8f9fa; - padding: 10px 15px; - border-radius: 4px; - cursor: pointer; - border: 1px solid #dee2e6; - transition: background-color 0.2s; - } - .table-header:hover { - background-color: #e9ecef; - } - .table-header h4 { - margin: 0; - display: flex; - justify-content: space-between; - align-items: center; - color: #495057; - } - .toggle-icon { - font-size: 0.8em; - transition: transform 0.2s; - } - .table-content { - margin-top: 10px; - } - .app-row { - display: table-row; - } - .checkbox-group { - display: flex; - flex-wrap: wrap; - gap: 15px; - justify-content: center; - margin-top: 10px; - } - .checkbox-label { - display: flex; - align-items: center; - cursor: pointer; - font-size: 0.9em; - } - .checkbox-label input[type="checkbox"] { - margin-right: 5px; - } - .owner-label { - margin-left: 10px; - font-size: 0.9em; - color: #555; - } - """ - - def create_summary_section() -> None: - """Create summary section""" - with div(cls='summary'): - h3('Summary') - with ul(): - li(strong('Total apps analyzed: '), str(len(apps))) - li(strong('Build temporarily disabled: '), str(len(cant_build_temp))) - li(strong('Test temporarily disabled: '), str(len(can_build_cant_test_temp))) - li(strong('Build disabled permanently: '), str(len(cant_build_not_temp))) - li(strong('Test disabled permanently: '), str(len(can_build_cant_test_not_temp))) - li(strong('Normal: '), str(len(can_test))) - - def create_navigation_section() -> None: - """Create navigation section with links to each category""" - with div(cls='navigation'): - h3('Quick Navigation') - with div(cls='nav-links'): - for category_id, title, _, _ in categories: - a(title, href=f'#{category_id}', cls='nav-link') - - def create_filter_group( - title: str, - filter_type: str, - items: list[str], - counts: Counter, - total_count: int, - ) -> None: - """Create a generic filter checkbox group with app counts""" - with div(cls='filter-container'): - strong(title, style='display: block; margin-bottom: 10px;') - with div(id=f'{filter_type}Filter', cls='checkbox-group'): - # 'All' checkbox - with label(cls='checkbox-label'): - input_( - type='checkbox', - id=f'{filter_type}-all', - value='all', - onchange=f'toggleAll(this, "{filter_type}")', - checked=True, - ) - span( - f'All ({total_count})', - style='margin-left: 5px; font-weight: bold;', - ) - - # Individual item checkboxes - for item in items: - count = counts.get(item, 0) - with label(cls='checkbox-label'): - input_( - type='checkbox', - cls=f'{filter_type}-checkbox', - value=item, - onchange='applyFilters()', - checked=True, - ) - span(f'{item} ({count})', style='margin-left: 5px;') - - def create_control_buttons() -> None: - """Create control buttons for expand/collapse all""" - with div(cls='control-buttons'): - button('Expand All', onclick='expandAll()', cls='control-btn') - button('Collapse All', onclick='collapseAll()', cls='control-btn collapse-all') - - def create_category_section( - category_id: str, - title: str, - apps: list[App], - all_apps: list[App], - show_status: bool = False, - ) -> None: - """Create category section""" - with div(cls='category-summary'): - with h3(): - span(title) - span(str(len(apps)), cls='count') - - # Always show section content (no collapse) - with div(cls='category-details', id=category_id, style='display: block;'): - if apps: - # Group by full directory path (preserve last level directory) - dir_groups: dict[str, list[App]] = {} - for app in apps: - # Use full path as grouping key - dir_name = app.app_dir - if dir_name not in dir_groups: - dir_groups[dir_name] = [] - dir_groups[dir_name].append(app) - - # Calculate total apps for each directory across all categories - total_apps_by_dir = {} - for app in all_apps: - dir_name = app.app_dir - if dir_name not in total_apps_by_dir: - total_apps_by_dir[dir_name] = 0 - total_apps_by_dir[dir_name] += 1 - - for dir_name, apps_in_dir in dir_groups.items(): - total_apps = total_apps_by_dir.get(dir_name, len(apps_in_dir)) - - # Determine the owners for this directory group - dir_owners = set() - for app in apps_in_dir: - dir_owners.update(app_to_owners_map.get(app.app_dir, [])) - owners_str = ', '.join(sorted(list(dir_owners))) - - with div(cls='directory-info', onclick=f"toggleTable('{category_id}-{hash(dir_name)}')"): - span( - '+', - cls='toggle-icon', - style='float: left; color: #666; margin-right: 8px; font-weight: bold;', - ) - strong(f'{dir_name.replace("./", "")}') - span(f' ({len(apps_in_dir)}/{total_apps} apps)', style='color: #666;') - span(f'Owners: {owners_str}', cls='owner-label') - - # Table can be collapsed, controlled by table header row - with div(cls='table-container'): - with div(cls='table-content', id=f'{category_id}-{hash(dir_name)}', style='display: none;'): - with table(): - with thead(cls='table-header'): - with tr(): - th('Target') - th('Config') - if show_status: - th('Status') - else: - th('Reason') - with tbody(): - for app in apps_in_dir: - app_owners = app_to_owners_map.get(app.app_dir, []) - with tr( - cls='app-row', - data_codeowners=','.join(app_owners), - ): - td(app.target) - td(app.config_name or '-') - if show_status: - td('Success') - else: - reason = ( - app.build_comment - if app.build_status == 'disabled' - else app.test_comment - ) - td(reason or 'Reason not specified') - else: - div('No apps in this category', cls='empty-section') - - def get_javascript() -> str: - """Return JavaScript code""" - return """ - function toggleTable(id) { - const content = document.getElementById(id); - if (!content) { - console.error('Content element not found:', id); - return; - } - - const directoryInfo = content.parentElement.previousElementSibling; - if (!directoryInfo) { - console.error('Directory info element not found'); - return; - } - - const icon = directoryInfo.querySelector('.toggle-icon'); - - if (content.style.display === 'none' || content.style.display === '') { - content.style.display = 'block'; - if (icon) icon.textContent = '-'; - } else { - content.style.display = 'none'; - if (icon) icon.textContent = '+'; - } - } - - function expandAll() { - const allContents = document.querySelectorAll('.table-content'); - const allIcons = document.querySelectorAll('.toggle-icon'); - - allContents.forEach(content => { - content.style.display = 'block'; - }); - - allIcons.forEach(icon => { - icon.textContent = '-'; - }); - } - - function collapseAll() { - const allContents = document.querySelectorAll('.table-content'); - const allIcons = document.querySelectorAll('.toggle-icon'); - - allContents.forEach(content => { - content.style.display = 'none'; - }); - - allIcons.forEach(icon => { - icon.textContent = '+'; - }); - } - - function applyFilters() { - // Get selected codeowners - const selectedCodeowners = Array.from( - document.querySelectorAll('.codeowner-checkbox:checked') - ).map(cb => cb.value); - - // Get selected targets - const selectedTargets = Array.from( - document.querySelectorAll('.target-checkbox:checked') - ).map(cb => cb.value); - - // Update 'All' checkbox states - const allCodeownerCheckbox = document.getElementById('codeowner-all'); - const allCodeownerCheckboxes = document.querySelectorAll('.codeowner-checkbox'); - allCodeownerCheckbox.checked = selectedCodeowners.length === allCodeownerCheckboxes.length; - - const allTargetCheckbox = document.getElementById('target-all'); - const allTargetCheckboxes = document.querySelectorAll('.target-checkbox'); - allTargetCheckbox.checked = selectedTargets.length === allTargetCheckboxes.length; - - // Filter rows and hide empty directory sections - const directorySections = document.querySelectorAll('.directory-info'); - directorySections.forEach(section => { - const tableContainer = section.nextElementSibling; - if (!tableContainer) return; - - const rows = tableContainer.querySelectorAll('.app-row'); - let visibleRows = 0; - rows.forEach(row => { - const codeowners = row.getAttribute('data-codeowners').split(','); - // Assumes target is the first cell - const target = row.querySelector('td:first-child').textContent; - - const codeownerMatch = selectedCodeowners.some(owner => codeowners.includes(owner)); - const targetMatch = selectedTargets.includes(target); - - if (codeownerMatch && targetMatch) { - row.style.display = 'table-row'; - visibleRows++; - } else { - row.style.display = 'none'; - } - }); - - // Hide the entire directory section if no rows are visible - if (visibleRows === 0) { - section.style.display = 'none'; - tableContainer.style.display = 'none'; - } else { - section.style.display = 'block'; - tableContainer.style.display = 'block'; - } - }); - } - - function toggleAll(checkbox, filterType) { - const allCheckboxes = document.querySelectorAll(`.${filterType}-checkbox`); - allCheckboxes.forEach(cb => { - cb.checked = checkbox.checked; - }); - applyFilters(); // Apply filters immediately - } - - // Initial filter call to show all rows at the beginning - document.addEventListener('DOMContentLoaded', function() { - applyFilters(); - }); - """ - - # Define category configuration - categories = [ - ('cant-build-temp', '1. Build temporarily disabled', cant_build_temp, False), - ('can-build-cant-test-temp', '2. Test temporarily disabled', can_build_cant_test_temp, False), - ('cant-build-not-temp', '3. Build disabled permanently', cant_build_not_temp, False), - ('can-build-cant-test-not_temp', '4. Test disabled permanently', can_build_cant_test_not_temp, False), - ('can-test', '5. Normal', can_test, True), - ] - - with doc.head: - style(get_css_styles()) - script(raw(get_javascript())) - - with doc: - with div(cls='container'): - h1('Build and Test Status Report') - div(f'Generated at: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}', cls='timestamp') - - # Create summary section - create_summary_section() - - # Create navigation section - create_navigation_section() - - # Create filter section and pass owner counts - with div(cls='filters-wrapper'): - create_filter_group( - 'Filter by Codeowner:', - 'codeowner', - sorted(list(all_codeowners)), - owner_app_counts, - len(apps), - ) - create_filter_group( - 'Filter by Target:', - 'target', - sorted(list(all_targets)), - target_app_counts, - len(apps), - ) - - # Create control buttons - create_control_buttons() - - # Generate each category using configuration - for category_id, title, apps_list, show_status in categories: - create_category_section(category_id, title, apps_list, apps, show_status) - - # Write to file - with open(report_path, 'w', encoding='utf-8') as f: - f.write(doc.render()) - - -def main() -> int: - parser = argparse.ArgumentParser( - description='Generate a report of disabled and skipped builds/tests', - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=""" -Examples: - # Basic usage (uses default paths: examples, tools/test_apps, components) - python gen_disabled_report.py --output report.html - - # Use existing JSON file - python gen_disabled_report.py --input apps.json --output report.html - - # Enable verbose output - python gen_disabled_report.py --input apps.json --output report.html --verbose - """, - ) - - # Input options - parser.add_argument('--input', type=str, help='Input JSON file containing apps data') - - # Output options - parser.add_argument( - '--output', - type=str, - default='disabled_report.html', - help='Output report file path (default: disabled_report.html)', - ) - - # idf-build-apps find options - parser.add_argument( - '--temp-json', - type=str, - default='apps.json', - help='Temporary JSON file path for idf-build-apps find output (default: apps.json)', - ) - parser.add_argument('--verbose', action='store_true', help='Enable verbose output') - parser.add_argument('--enable-preview-targets', action='store_true', help='Enable preview targets') - - args = parser.parse_args() - - try: - # Always run idf-build-apps find to get the most up-to-date data - print('Running idf-build-apps find to generate app list...') - input_file = run_idf_build_apps_find( - output_file=args.temp_json, - verbose=args.verbose, - enable_preview_targets=args.enable_preview_targets, - ) - - # Load application data - print(f'Loading apps from {input_file}...') - apps = load_apps_from_json(input_file) - - # Add codeowners information - print('Adding codeowners information...') - idf_path = os.environ.get('IDF_PATH') - if not idf_path: - raise ValueError('IDF_PATH environment variable is not set') - codeowners_path = os.path.join(idf_path, '.gitlab', 'CODEOWNERS') - codeowners_mapping = parse_codeowners(codeowners_path) - app_to_owners_map = add_codeowners_to_apps(apps, codeowners_mapping) - - # Generate report - print(f'Generating report to {args.output}...') - generate_disabled_report(apps, app_to_owners_map, args.output) - - print(f'Report generated successfully: {args.output}') - - return 0 - - except Exception as e: - print(f'Error: {e}') - if args.verbose: - import traceback - - traceback.print_exc() - return 1 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/tools/ci/idf_ci_utils.py b/tools/ci/idf_ci_utils.py index b8db76beeb..73183e31e4 100644 --- a/tools/ci/idf_ci_utils.py +++ b/tools/ci/idf_ci_utils.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2020-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2020-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 # internal use only for CI # some CI related util functions @@ -13,7 +13,7 @@ from functools import cached_property IDF_PATH: str = os.path.abspath(os.getenv('IDF_PATH', os.path.join(os.path.dirname(__file__), '..', '..'))) -def get_submodule_dirs(full_path: bool = False) -> t.List[str]: +def get_submodule_dirs(full_path: bool = False) -> list[str]: """ To avoid issue could be introduced by multi-os or additional dependency, we use python and git to get this output @@ -71,7 +71,7 @@ def is_executable(full_path: str) -> bool: return os.access(full_path, os.X_OK) -def get_git_files(path: str = IDF_PATH, full_path: bool = False) -> t.List[str]: +def get_git_files(path: str = IDF_PATH, full_path: bool = False) -> list[str]: """ Get the result of git ls-files :param path: path to run git ls-files @@ -98,11 +98,11 @@ def get_git_files(path: str = IDF_PATH, full_path: bool = False) -> t.List[str]: return [os.path.join(path, f) for f in files] if full_path else files -def to_list(s: t.Any) -> t.List[t.Any]: +def to_list(s: t.Any) -> list[t.Any]: if not s: return [] - if isinstance(s, (set, tuple)): + if isinstance(s, set | tuple): return list(s) if isinstance(s, list): @@ -113,8 +113,8 @@ def to_list(s: t.Any) -> t.List[t.Any]: class GitlabYmlConfig: def __init__(self, root_yml_filepath: str = os.path.join(IDF_PATH, '.gitlab-ci.yml')) -> None: - self._config: t.Dict[str, t.Any] = {} - self._defaults: t.Dict[str, t.Any] = {} + self._config: dict[str, t.Any] = {} + self._defaults: dict[str, t.Any] = {} self._load(root_yml_filepath) @@ -127,6 +127,14 @@ class GitlabYmlConfig: # expanding "include" for item in root_yml.pop('include', []) or []: + if isinstance(item, dict): + if 'project' in item: + continue + elif 'local' in item: + item = item['local'] + else: + continue + all_config.update(yaml.load(open(os.path.join(IDF_PATH, item)), Loader=yaml.FullLoader)) if 'default' in all_config: @@ -135,41 +143,41 @@ class GitlabYmlConfig: self._config = all_config # anchor is the string that will be reused in templates - self._anchor_keys: t.Set[str] = set() + self._anchor_keys: set[str] = set() # template is a dict that will be extended - self._template_keys: t.Set[str] = set() - self._used_template_keys: t.Set[str] = set() # tracing the used templates + self._template_keys: set[str] = set() + self._used_template_keys: set[str] = set() # tracing the used templates # job is a dict that will be executed - self._job_keys: t.Set[str] = set() + self._job_keys: set[str] = set() self.expand_extends() @property - def default(self) -> t.Dict[str, t.Any]: + def default(self) -> dict[str, t.Any]: return self._defaults @property - def config(self) -> t.Dict[str, t.Any]: + def config(self) -> dict[str, t.Any]: return self._config @cached_property - def global_keys(self) -> t.List[str]: + def global_keys(self) -> list[str]: return ['default', 'include', 'workflow', 'variables', 'stages'] @cached_property - def anchors(self) -> t.Dict[str, t.Any]: + def anchors(self) -> dict[str, t.Any]: return {k: v for k, v in self.config.items() if k in self._anchor_keys} @cached_property - def jobs(self) -> t.Dict[str, t.Any]: + def jobs(self) -> dict[str, t.Any]: return {k: v for k, v in self.config.items() if k in self._job_keys} @cached_property - def templates(self) -> t.Dict[str, t.Any]: + def templates(self) -> dict[str, t.Any]: return {k: v for k, v in self.config.items() if k in self._template_keys} @cached_property - def used_templates(self) -> t.Set[str]: + def used_templates(self) -> set[str]: return self._used_template_keys def expand_extends(self) -> None: @@ -180,7 +188,7 @@ class GitlabYmlConfig: if k in self.global_keys: continue - if isinstance(v, (str, list)): + if isinstance(v, str | list): self._anchor_keys.add(k) elif k.startswith('.if-'): self._anchor_keys.add(k) @@ -201,7 +209,7 @@ class GitlabYmlConfig: for k in self._job_keys: self._expand_extends(k) - def _merge_dict(self, d1: t.Dict[str, t.Any], d2: t.Dict[str, t.Any]) -> t.Any: + def _merge_dict(self, d1: dict[str, t.Any], d2: dict[str, t.Any]) -> t.Any: for k, v in d2.items(): if k in d1: if isinstance(v, dict) and isinstance(d1[k], dict): @@ -213,7 +221,7 @@ class GitlabYmlConfig: return d1 - def _expand_extends(self, name: str) -> t.Dict[str, t.Any]: + def _expand_extends(self, name: str) -> dict[str, t.Any]: extends = to_list(self.config[name].pop('extends', None)) if not extends: return self.config[name] # type: ignore diff --git a/tools/ci/utils.sh b/tools/ci/utils.sh index 5d5fef4202..a4c3a4875b 100644 --- a/tools/ci/utils.sh +++ b/tools/ci/utils.sh @@ -5,7 +5,14 @@ function add_ssh_keys() { mkdir -p ~/.ssh chmod 700 ~/.ssh echo -n "${key_string}" >~/.ssh/id_rsa_base64 - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 >~/.ssh/id_rsa + # Detect base64 implementation via --help output + if base64 --help 2>&1 | grep -q -- '--ignore-garbage'; then + # GNU coreutils base64 + base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 >~/.ssh/id_rsa + else + # macOS/BSD base64 - requires stdin or -i flag + base64 --decode -i ~/.ssh/id_rsa_base64 -o ~/.ssh/id_rsa + fi chmod 600 ~/.ssh/id_rsa }