ci: streamline custom before_script checkout code to preclone script

.gitlab/ci/build.yml

@@ -1,7 +1,6 @@
 .build_template:
   stage: build
   extends:
-    - .before_script:build
     - .after_script:build
   image: $ESP_ENV_IMAGE
   tags: [build, shiny]

.gitlab/ci/common.yml

@@ -34,8 +34,7 @@ variables:
   # since we're using merged-result pipelines, the last commit should work for most cases
   # --prune --prune-tags: in case remote branch or tag is force pushed
   GIT_FETCH_EXTRA_FLAGS: "--no-recurse-submodules --prune --prune-tags"
-  # we're using .cache folder for caches
-  GIT_CLEAN_FLAGS: -ffdx -e .cache/
+
   LATEST_GIT_TAG: v6.1-dev
 
   SUBMODULE_FETCH_TOOL: "tools/ci/ci_fetch_submodule.py"
@@ -245,15 +244,6 @@ variables:
   before_script:
     - *common-before_scripts
 
-.before_script:build:
-  before_script:
-    - *common-before_scripts
-    - *setup_tools_and_idf_python_venv
-    - add_gitlab_ssh_keys
-    - fetch_submodules
-    - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
-    - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
-
 .after_script:build:
   after_script:
     - source tools/ci/utils.sh
@@ -263,105 +253,6 @@ variables:
 ##############################
 # Git Strategy Job Templates #
 ##############################
-.git_init: &git_init |
-  mkdir -p "${CI_PROJECT_DIR}"
-  cd "${CI_PROJECT_DIR}"
-  git init
-
-.git_fetch_from_mirror_url_if_exists: &git_fetch_from_mirror_url_if_exists |
-  # check if set mirror
-  if [ -n "${LOCAL_GITLAB_HTTPS_HOST:-}" ] && [ -n "${ESPCI_TOKEN:-}" ]; then
-    MIRROR_REPO_URL="https://bot:${ESPCI_TOKEN}@${LOCAL_GITLAB_HTTPS_HOST}/${CI_PROJECT_PATH}"
-  elif [ -n "${LOCAL_GIT_MIRROR:-}" ]; then
-    MIRROR_REPO_URL="${LOCAL_GIT_MIRROR}/${CI_PROJECT_PATH}"
-  fi
-
-  # fetch from mirror first if set
-  if [ -n "${MIRROR_REPO_URL:-}" ]; then
-    if git remote -v | grep origin; then
-      git remote set-url origin "${MIRROR_REPO_URL}"
-    else
-      git remote add origin "${MIRROR_REPO_URL}"
-    fi
-    # mirror url may fail with authentication issue
-    git fetch origin --no-recurse-submodules || true
-  fi
-
-  # set remote url to CI_REPOSITORY_URL
-  if git remote -v | grep origin; then
-    git remote set-url origin "${CI_REPOSITORY_URL}"
-  else
-    git remote add origin "${CI_REPOSITORY_URL}"
-  fi
-
-.git_checkout_ci_commit_sha: &git_checkout_ci_commit_sha |
-  git checkout $CI_COMMIT_SHA
-  eval "git clean ${GIT_CLEAN_FLAGS}"
-
-# git diff requires two commits, with different CI env var
-#
-# By default, we use git strategy "clone" with depth 1 to speed up the clone process.
-# But for jobs requires running `git diff`, we need to fetch more commits to get the correct diffs.
-#
-# Since there's no way to get the correct git_depth before the job starts,
-# we can't set `GIT_DEPTH` in the job definition.
-#
-# Set git strategy to "none" and fetch manually instead.
-.before_script:fetch:git_diff:
-  variables:
-    GIT_STRATEGY: none
-  before_script:
-    - *git_init
-    - *git_fetch_from_mirror_url_if_exists
-    - |
-      # Store the diff output in a temporary file
-      TEMP_FILE=$(mktemp)
-      # merged results pipelines, by default
-      if [[ -n $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA ]]; then
-        git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_SHA
-        git fetch origin $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA
-
-        git diff --name-only $CI_MERGE_REQUEST_TARGET_BRANCH_SHA...$CI_MERGE_REQUEST_SOURCE_BRANCH_SHA > "$TEMP_FILE"
-        GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
-        git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
-      # merge request pipelines, when the mr got conflicts
-      elif [[ -n $CI_MERGE_REQUEST_DIFF_BASE_SHA ]]; then
-        git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
-        git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
-        git diff --name-only $CI_MERGE_REQUEST_DIFF_BASE_SHA $CI_COMMIT_SHA > "$TEMP_FILE"
-        GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
-      # other pipelines, like the protected branches pipelines
-      elif [[ "$CI_COMMIT_BEFORE_SHA" != "0000000000000000000000000000000000000000" ]]; then
-        git fetch origin $CI_COMMIT_BEFORE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
-        git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
-        git diff --name-only $CI_COMMIT_BEFORE_SHA $CI_COMMIT_SHA > "$TEMP_FILE"
-        GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
-      else
-        # pipeline source could be web, scheduler, etc.
-        git fetch origin $CI_COMMIT_SHA --depth=2 ${GIT_FETCH_EXTRA_FLAGS}
-        git diff --name-only $CI_COMMIT_SHA~1 $CI_COMMIT_SHA > "$TEMP_FILE"
-        GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
-      fi
-    - *git_checkout_ci_commit_sha
-    - *common-before_scripts
-    - *setup_tools_and_idf_python_venv
-    - add_gitlab_ssh_keys
-
-# target test runners may locate in different places
-# for runners set git mirror, we fetch from the mirror first, then fetch the HEAD commit
-.before_script:fetch:target_test:
-  variables:
-    GIT_STRATEGY: none
-  before_script:
-    - *git_init
-    - *git_fetch_from_mirror_url_if_exists
-    - eval "git fetch --depth=1 ${GIT_FETCH_EXTRA_FLAGS} origin ${CI_COMMIT_SHA}"
-    - *git_checkout_ci_commit_sha
-    - *common-before_scripts
-    - *setup_tools_and_idf_python_venv
-    - add_gitlab_ssh_keys
-    # no submodules
-
 .brew-macos-settings:
   variables:
     GIT_STRATEGY: none  # we do manual git clone to use local mirror
@@ -421,12 +312,8 @@ default:
     - *setup_tools_and_idf_python_venv
     - add_gitlab_ssh_keys
     - fetch_submodules
-# gitlab bug, setting them here doesn't work
-# - expire_in: https://gitlab.com/gitlab-org/gitlab/-/issues/404563
-# - when: https://gitlab.com/gitlab-org/gitlab/-/issues/440672
-#  artifacts:
-#    expire_in: 1 week
-#    when: always
+    - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
+    - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
   retry:
     max: 2
     when:

.gitlab/ci/host-test.yml

@@ -325,7 +325,6 @@ build_docker:
 test_pytest_qemu:
   extends:
     - .host_test_template
-    - .before_script:build
   artifacts:
     paths:
       - XUNIT_RESULT.xml
@@ -360,7 +359,6 @@ test_pytest_qemu:
 test_pytest_linux:
   extends:
     - .host_test_template
-    - .before_script:build
   artifacts:
     paths:
       - XUNIT_RESULT.xml

.gitlab/ci/pre_check.yml

@@ -115,7 +115,6 @@ check_version_tag:
 check_test_scripts_build_test_rules:
   extends:
     - .pre_check_template
-    - .before_script:build
   script:
     # requires basic pytest dependencies
     - python tools/ci/check_build_test_rules.py check-test-scripts examples/ tools/test_apps components
@@ -123,9 +122,39 @@ check_test_scripts_build_test_rules:
 pipeline_variables:
   extends:
     - .pre_check_template
-    - .before_script:fetch:git_diff
   tags: [fast_run, shiny]
+  variables:
+    GIT_STRATEGY: "fetch"
   script:
+    - |
+      # Store the diff output in a temporary file
+      TEMP_FILE=$(mktemp)
+      # merged results pipelines, by default
+      if [[ -n $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA ]]; then
+        git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_SHA
+        git fetch origin $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA
+
+        git diff --name-only $CI_MERGE_REQUEST_TARGET_BRANCH_SHA...$CI_MERGE_REQUEST_SOURCE_BRANCH_SHA > "$TEMP_FILE"
+        GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
+        git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
+      # merge request pipelines, when the mr got conflicts
+      elif [[ -n $CI_MERGE_REQUEST_DIFF_BASE_SHA ]]; then
+        git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
+        git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
+        git diff --name-only $CI_MERGE_REQUEST_DIFF_BASE_SHA $CI_COMMIT_SHA > "$TEMP_FILE"
+        GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
+      # other pipelines, like the protected branches pipelines
+      elif [[ "$CI_COMMIT_BEFORE_SHA" != "0000000000000000000000000000000000000000" ]]; then
+        git fetch origin $CI_COMMIT_BEFORE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
+        git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
+        git diff --name-only $CI_COMMIT_BEFORE_SHA $CI_COMMIT_SHA > "$TEMP_FILE"
+        GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
+      else
+        # pipeline source could be web, scheduler, etc.
+        git fetch origin $CI_COMMIT_SHA --depth=2 ${GIT_FETCH_EXTRA_FLAGS}
+        git diff --name-only $CI_COMMIT_SHA~1 $CI_COMMIT_SHA > "$TEMP_FILE"
+        GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
+      fi
     # MODIFIED_FILES is a list of files that changed, could be used everywhere
     - MODIFIED_FILES=$(echo "$GIT_DIFF_OUTPUT" | xargs)
     - echo "MODIFIED_FILES=$MODIFIED_FILES" >> pipeline.env

tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml

@@ -6,7 +6,6 @@
 ########################
 .dynamic_build_template:
   extends:
-    - .before_script:build
     - .after_script:build
   image: $ESP_ENV_IMAGE
   tags: [build, shiny]
@@ -44,12 +43,11 @@
       --modified-files ${MR_MODIFIED_FILES}
 
 .dynamic_target_test_template:
-  extends:
-    - .before_script:fetch:target_test
   image: $TARGET_TEST_ENV_IMAGE
   stage: target_test
   timeout: 1 hour
   variables:
+    GIT_STRATEGY: "fetch"
     SUBMODULES_TO_FETCH: "none"
     # set while generating the pipeline
     nodes: ""