esp-idf/.gitlab/ci/pre_check.yml

.pre_check_template:
  stage: pre_check
  image: $ESP_ENV_IMAGE
  tags: [build, shiny]
  dependencies: # set dependencies to null to avoid missing artifacts issue

check_version:
  # Don't run this for feature/bugfix branches, so that it is possible to modify
  # esp_idf_version.h in a branch before tagging the next version.
  extends:
    - .pre_check_template
    - .rules:protected:check
  tags: [ brew, github_sync ]
  variables:
    # need a full clone to get the latest tag
    # the --shallow-since=$(git log -1 --format=%as $LATEST_GIT_TAG) option is not accurate
    GIT_STRATEGY: fetch
    SUBMODULES_TO_FETCH: "none"
    GIT_DEPTH: 0
  script:
    - export IDF_PATH=$PWD
    - tools/ci/check_idf_version.sh

check_api_usage:
  extends: .pre_check_template
  script:
    - python -m pip install ast-grep-cli # use ast-grep to describe customized lint rules
    - ast-grep scan
    - tools/ci/check_api_violation.sh
    - tools/ci/check_examples_extra_component_dirs.sh

check_wifi_remote_api:
  extends:
    - .pre_check_template
    - .rules:build:check
  script:
    - cd components/esp_wifi/remote/scripts/
    - python generate_and_check.py
    - git diff --exit-code || { echo 'Differences found. Please run generate_and_check.py and commit the changes.'; exit 1; }

check_blobs:
  extends:
    - .pre_check_template
    - .rules:build:check
  variables:
    SUBMODULES_TO_FETCH: "components/esp_wifi/lib;components/esp_phy/lib;components/esp_coex/lib"
  script:
    # Check if Wi-Fi library header files match between IDF and the version used when compiling the libraries
    - IDF_TARGET=esp32 $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh
    - IDF_TARGET=esp32s2 $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh
    - IDF_TARGET=esp32s3 $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh
    - IDF_TARGET=esp32c2 $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh
    - IDF_TARGET=esp32c3 $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh
    - IDF_TARGET=esp32c6 $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh
    - IDF_TARGET=esp32c5 $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh
    - IDF_TARGET=esp32c61 $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh
    - IDF_TARGET=esp32_host $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh
    # Check if Coexistence library header files match between IDF and the version used when compiling the libraries
    - IDF_TARGET=esp32 $IDF_PATH/components/esp_coex/test_md5/test_md5.sh
    - IDF_TARGET=esp32s2 $IDF_PATH/components/esp_coex/test_md5/test_md5.sh
    - IDF_TARGET=esp32s3 $IDF_PATH/components/esp_coex/test_md5/test_md5.sh
    - IDF_TARGET=esp32c2 $IDF_PATH/components/esp_coex/test_md5/test_md5.sh
    - IDF_TARGET=esp32c3 $IDF_PATH/components/esp_coex/test_md5/test_md5.sh
    - IDF_TARGET=esp32c6 $IDF_PATH/components/esp_coex/test_md5/test_md5.sh
    - IDF_TARGET=esp32h2 $IDF_PATH/components/esp_coex/test_md5/test_md5.sh
    - IDF_TARGET=esp32c5 $IDF_PATH/components/esp_coex/test_md5/test_md5.sh
    - IDF_TARGET=esp32c61 $IDF_PATH/components/esp_coex/test_md5/test_md5.sh
    # Check if Wi-Fi, PHY, BT blobs contain references to specific symbols
    - bash $IDF_PATH/tools/ci/check_blobs.sh

check_chip_support_components:
  extends:
    - .pre_check_template
    - .rules:build:check
  artifacts:
    when: on_failure
    paths:
      - esp_hw_support_part.h
      - bootloader_support_part.h
    expire_in: 1 week
  script:
    - python tools/ci/check_soc_headers_leak.py
    - find ${IDF_PATH}/components/soc \( -path "*/include/soc/*_struct.h" -o -path "*/register/soc/*_struct.h" -o -path "*/register/*/soc/*_struct.h" \) -print0 | xargs -0 -n1 ./tools/ci/check_soc_struct_headers.py
    - tools/ci/check_esp_memory_utils_headers.sh

check_esp_err_to_name:
  extends:
    - .pre_check_template
    - .rules:build:check
  artifacts:
    when: on_failure
    paths:
      - components/esp_common/esp_err_to_name.c
    expire_in: 1 week
  script:
    - cd ${IDF_PATH}/tools/
    - ./gen_esp_err_to_name.py
    - git diff --exit-code -- ../components/esp_common/src/esp_err_to_name.c || { echo 'Differences found. Please run gen_esp_err_to_name.py and commit the changes.'; exit 1; }

check_esp_system:
  extends:
    - .pre_check_template
    - .rules:build
  script:
    - python components/esp_system/check_system_init_priorities.py

# For release tag pipelines only, make sure the tag was created with 'git tag -a' so it will update
# the version returned by 'git describe'
# Don't forget to update the env var `LATEST_GIT_TAG` in .gitlab/ci/common.yml
check_version_tag:
  extends:
    - .pre_check_template
    - .rules:tag:release
  tags: [ brew, github_sync ]
  variables:
    # need a full clone to get the latest tag
    # the --shallow-since=$(git log -1 --format=%as $LATEST_GIT_TAG) option is not accurate
    GIT_STRATEGY: fetch
    SUBMODULES_TO_FETCH: "none"
    GIT_DEPTH: 0
  script:
    - (git cat-file -t $CI_COMMIT_REF_NAME | grep tag) || (echo "ESP-IDF versions must be annotated tags." && exit 1)

check_test_scripts_build_test_rules:
  extends:
    - .pre_check_template
  script:
    # requires basic pytest dependencies
    - python tools/ci/check_build_test_rules.py check-test-scripts examples/ tools/test_apps components

check_submodule_sync:
  extends:
    - .pre_check_template
    - .rules:test:submodule
  tags: [ brew, github_sync ]
  retry: 2
  variables:
    GIT_STRATEGY: fetch  # use brew local mirror first
    SUBMODULES_TO_FETCH: "none"
    PUBLIC_IDF_URL: "https://github.com/espressif/esp-idf.git"
  script:
    - git submodule deinit --force .
    - rm -rf .git/modules  # remove all the cached metadata
    # setting the default remote URL to the public one, to resolve relative location URLs
    - git config remote.origin.url ${PUBLIC_IDF_URL}
    # check if all submodules are correctly synced to public repository
    - git submodule init
    - git config --get-regexp '^submodule\..*\.url$' || true
    - git submodule update --recursive
    - echo "IDF was cloned from ${PUBLIC_IDF_URL} completely"

pipeline_variables:
  extends:
    - .pre_check_template
  tags: [fast_run, shiny]
  variables:
    GIT_STRATEGY: "fetch"
  script:
    - |
      # Store the diff output in a temporary file
      TEMP_FILE=$(mktemp)
      # merged results pipelines, by default
      if [[ -n $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA ]]; then
        git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_SHA
        git fetch origin $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA

        git diff --name-only $CI_MERGE_REQUEST_TARGET_BRANCH_SHA...$CI_MERGE_REQUEST_SOURCE_BRANCH_SHA > "$TEMP_FILE"
        GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
        git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
      # merge request pipelines, when the mr got conflicts
      elif [[ -n $CI_MERGE_REQUEST_DIFF_BASE_SHA ]]; then
        git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
        git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
        git diff --name-only $CI_MERGE_REQUEST_DIFF_BASE_SHA $CI_COMMIT_SHA > "$TEMP_FILE"
        GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
      # other pipelines, like the protected branches pipelines
      elif [[ "$CI_COMMIT_BEFORE_SHA" != "0000000000000000000000000000000000000000" ]]; then
        git fetch origin $CI_COMMIT_BEFORE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
        git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
        git diff --name-only $CI_COMMIT_BEFORE_SHA $CI_COMMIT_SHA > "$TEMP_FILE"
        GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
      else
        # pipeline source could be web, scheduler, etc.
        git fetch origin $CI_COMMIT_SHA --depth=2 ${GIT_FETCH_EXTRA_FLAGS}
        git diff --name-only $CI_COMMIT_SHA~1 $CI_COMMIT_SHA > "$TEMP_FILE"
        GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
      fi
    # MODIFIED_FILES is a list of files that changed, could be used everywhere
    - MODIFIED_FILES=$(echo "$GIT_DIFF_OUTPUT" | xargs)
    - echo "MODIFIED_FILES=$MODIFIED_FILES" >> pipeline.env
    # MR_MODIFIED_FILES and MR_MODIFIED_COMPONENTS are semicolon separated lists that is used in MR only
    # for non MR pipeline, these are empty lists
    - |
      if [ -z "$CI_MERGE_REQUEST_IID" ]; then
        echo "MR_MODIFIED_FILES=\"\"" >> pipeline.env
        echo "MR_MODIFIED_COMPONENTS=\"\"" >> pipeline.env
      else
        MR_MODIFIED_FILES=$(echo "$GIT_DIFF_OUTPUT" | tr '\n' ';')
        echo "MR_MODIFIED_FILES=\"$MR_MODIFIED_FILES\"" >> pipeline.env

        MR_MODIFIED_COMPONENTS=$(run_cmd python tools/ci/ci_get_mr_info.py components --modified-files $MODIFIED_FILES | tr '\n' ';')
        echo "MR_MODIFIED_COMPONENTS=\"$MR_MODIFIED_COMPONENTS\"" >> pipeline.env
      fi
    - |
      if [ -n "$PIPELINE_COMMIT_SHA" ]; then
        echo "PIPELINE_COMMIT_SHA=$PIPELINE_COMMIT_SHA" >> pipeline.env
      fi
    - echo "OOCD_DISTRO_URL_ARMHF=$OOCD_DISTRO_URL_ARMHF" >> pipeline.env
    - echo "OOCD_DISTRO_URL_ARM64=$OOCD_DISTRO_URL_ARM64" >> pipeline.env
    - run_cmd idf-ci gitlab pipeline-variables >> pipeline.env
    - cat pipeline.env
    - run_cmd idf-ci gitlab upload-artifacts --type env
  artifacts:
    reports:
      dotenv: pipeline.env
    paths:
      - pipeline.env
    expire_in: 1 week
    when: always

baseline_manifest_sha:
  extends:
    - .pre_check_template
    - .rules:dev-push
  tags: [fast_run, shiny]
  script:
    - |
      # merged results pipelines, by default
      # diff between target-branch-head and merged-result-head
      if [ -n "$CI_MERGE_REQUEST_TARGET_BRANCH_SHA" ]; then
        git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_SHA --depth=1
        git checkout FETCH_HEAD
        idf-build-apps dump-manifest-sha \
          --manifest-files $(find . -name ".build-test-rules.yml" | xargs) \
          --output .manifest_sha
      # merge request pipelines, when the mr got conflicts
      # diff between diff-base-sha and merge-request-head
      elif [ -n "$CI_MERGE_REQUEST_DIFF_BASE_SHA" ]; then
        git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1
        git checkout FETCH_HEAD
        idf-build-apps dump-manifest-sha \
          --manifest-files $(find . -name ".build-test-rules.yml" | xargs) \
          --output .manifest_sha
      # other pipelines, like the protected branches pipelines
      # not triggered in this job
      fi
  artifacts:
    paths:
      - .manifest_sha
    expire_in: 1 week
    when: always

retry_failed_jobs:
  extends:
    - .pre_check_template
    - .rules:dev-push
  tags: [shiny, fast_run]
  allow_failure: true
  script:
    - echo "Retrieving and retrying all failed jobs for the pipeline..."
    - python tools/ci/python_packages/gitlab_api.py retry_failed_jobs $CI_MERGE_REQUEST_PROJECT_ID --pipeline_id $CI_PIPELINE_ID
  when: manual