#####################
# Default Variables #
#####################
stages:
  - upload_cache
  - pre_check
  - build
  - assign_test
  - build_doc
  - target_test
  - host_test
  - test_deploy
  - deploy
  - post_deploy

variables:
# System environment

  # Common parameters for the 'make' during CI tests
  MAKEFLAGS: "-j5 --no-keep-going"

# GitLab-CI environment
  # Thanks to pack-objects cache, clone strategy should behave faster than fetch
  #   so we pick "clone" as default git strategy
  # Shiny runners by default remove the CI_PROJECT_DIR every time at the beginning of one job
  #   and clone with a --depth=1
  # Brew runners will fetch from locally mirror first, and cache the local CI_PROJECT_DIR
  # In conclusion
  # - set GIT_STRATEGY: "clone" to shiny runners
  # - set GIT_STRATEGY: "fetch" to brew runners
  GIT_STRATEGY: clone
  GIT_DEPTH: 1
  GIT_SUBMODULE_STRATEGY: none  # here we use cache for submodules, so we don't need to fetch them every time
  # since we're using merged-result pipelines, the last commit should work for most cases
  # --prune --prune-tags: in case remote branch or tag is force pushed
  GIT_FETCH_EXTRA_FLAGS: "--no-recurse-submodules --prune --prune-tags"
  # we're using .cache folder for caches
  GIT_CLEAN_FLAGS: -ffdx -e .cache/
  LATEST_GIT_TAG: v6.1-dev

  SUBMODULE_FETCH_TOOL: "tools/ci/ci_fetch_submodule.py"
  # by default we will fetch all submodules
  # jobs can overwrite this variable to only fetch submodules they required
  # set to "none" if don't need to fetch submodules
  SUBMODULES_TO_FETCH: "all"
  # tell build system do not check submodule update as we download archive instead of clone
  IDF_SKIP_CHECK_SUBMODULES: 1

  IDF_PATH: "$CI_PROJECT_DIR"
  V: "0"
  CHECKOUT_REF_SCRIPT: "$CI_PROJECT_DIR/tools/ci/checkout_project_ref.py"

  # Docker images
  ESP_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/esp-env-v6.1:1"
  ESP_IDF_DOC_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/esp-idf-doc-env-v6.1:1-1"
  TARGET_TEST_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/target-test-env-v6.1:1"
  SONARQUBE_SCANNER_IMAGE: "${CI_DOCKER_REGISTRY}/sonarqube-scanner:5"

  # cache python dependencies
  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"

  # Set this variable to the branch of idf-constraints repo in order to test a custom Python constraint file. The
  # branch name must be without the remote part ("origin/"). Keep the variable empty in order to use the constraint
  # file from https://dl.espressif.com/dl/esp-idf.
  CI_PYTHON_CONSTRAINT_BRANCH: ""

  # Update the filename for a specific ESP-IDF release. It is used only with CI_PYTHON_CONSTRAINT_BRANCH.
  CI_PYTHON_CONSTRAINT_FILE: "espidf.constraints.v6.1.txt"

  # Set this variable to repository name of a Python tool you wish to install and test in the context of ESP-IDF CI.
  # Keep the variable empty when not used.
  CI_PYTHON_TOOL_REPO: ""

  # Set this variable to the branch of a Python tool repo specified in CI_PYTHON_TOOL_REPO. The
  # branch name must be without the remote part ("origin/"). Keep the variable empty when not used.
  # This is used only if CI_PYTHON_TOOL_REPO is not empty.
  CI_PYTHON_TOOL_BRANCH: ""

  # Set this variable to Clang toolchain distro URL to be used.
  # NOTE: We have separate toolchains for Xtensa and RISCV, therefore jobs for one arch will fail.
  # This is OK as far as we use CI_CLANG_DISTRO_URL for pre-release tests purposes only.
  # Keep the variable empty when not used.
  CI_CLANG_DISTRO_URL: ""

  # Set this variable to specify the file name for the known failure cases.
  KNOWN_FAILURE_CASES_FILE_NAME: "master.txt"

  IDF_CI_BUILD: 1

  # ccache settings
  # some settings need to set in .gitlab-ci.yml as it takes effect while start-up the job
  # https://ccache.dev/manual/latest.html#_configuring_ccache

  # host mapping volume to share ccache between runner concurrent jobs
  CCACHE_DIR: "/cache/idf_ccache"
  CCACHE_MAXSIZE: "50G"

  FF_USE_NEW_BASH_EVAL_STRATEGY: "true"
  FORCE_COLOR: "1"  # rich print with color

################################################
# `before_script` and `after_script` Templates #
################################################
.common_before_scripts: &common-before_scripts |
  source tools/ci/utils.sh

  is_based_on_commits $REQUIRED_ANCESTOR_COMMITS

  if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then
    export IDF_MIRROR_PREFIX_MAP=
  fi

  if echo "$CI_MERGE_REQUEST_LABELS" | egrep "(^|,)include_nightly_run(,|$)"; then
    export INCLUDE_NIGHTLY_RUN="1"
    export NIGHTLY_RUN="1"
  fi

  # configure cmake related flags
  source tools/ci/configure_ci_environment.sh

  # add extra python packages
  export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci:$IDF_PATH/tools/esp_app_trace:$IDF_PATH/components/partition_table:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH"

.setup_tools_and_idf_python_venv: &setup_tools_and_idf_python_venv |
  # must use after setup_tools_except_target_test
  # otherwise the export.sh won't work properly

  # download constraint file for dev
  if [[ -n "$CI_PYTHON_CONSTRAINT_BRANCH" ]]; then
        wget -O /tmp/constraint.txt --header="Authorization:Bearer ${ESPCI_TOKEN}" "${GITLAB_HTTP_SERVER}/api/v4/projects/2581/repository/files/${CI_PYTHON_CONSTRAINT_FILE}/raw?ref=${CI_PYTHON_CONSTRAINT_BRANCH}"
    mkdir -p ~/.espressif
    mv /tmp/constraint.txt ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
  fi

  # Mirror
  if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then
    export IDF_MIRROR_PREFIX_MAP=
  fi
  # Optimize pip install
  if echo "${CI_RUNNER_TAGS}" | grep "shiny"; then
    export PIP_INDEX_URL="${PIP_INDEX_URL_SHINY}"
  fi
  if [[ "$(uname -m)" == "x86_64" ]] || [[ "$(uname -m)" == "aarch64" ]]; then
    export IDF_PIP_WHEELS_URL=""
  fi

  # install.sh
  if [[ "${CI_JOB_STAGE}" != "target_test" ]]; then
    section_start "running_install_sh" "Running install.sh"
    if [[ "${CI_JOB_STAGE}" == "build_doc" ]]; then
      run_cmd bash install.sh --enable-ci --enable-docs
    else
      run_cmd bash install.sh --enable-ci
    fi
    section_end "running_install_sh"
  else
    section_start "install_python_env" "Install Python environment, skip required tools check"
    run_cmd python tools/idf_tools.py install-python-env --features ci,test-specific
    export IDF_SKIP_TOOLS_CHECK=1
    section_end "install_python_env"
  fi

  section_start "source_export" "Source export.sh"
  source ./export.sh
  section_end "source_export"

  # Eager upgrade of CI dependencies
  # Done after sourcing export.sh so that we could easily invoke the right pip
  section_start "upgrade_ci_dependencies" "Upgrading CI dependencies"
  pip install --upgrade --upgrade-strategy=eager -r $IDF_PATH/tools/requirements/requirements.ci.txt -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
  if [[ "${CI_JOB_STAGE}" == "target_test" ]]; then
    pip install --upgrade --upgrade-strategy=eager -r $IDF_PATH/tools/requirements/requirements.test-specific.txt -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
  fi
  section_end "upgrade_ci_dependencies"

  REEXPORT_NEEDED=0
  if [[ ! -z "$INSTALL_EXTRA_TOOLS" ]]; then
    section_start "installing_optional_tools" "Install optional tools ${INSTALL_EXTRA_TOOLS}"
    run_cmd $IDF_PATH/tools/idf_tools.py --non-interactive install $INSTALL_EXTRA_TOOLS
    section_end "installing_optional_tools"

    REEXPORT_NEEDED=1
  fi

  # Install esp-clang if necessary (esp-clang is separately installed)
  if [[ "$IDF_TOOLCHAIN" == "clang" && -z "$CI_CLANG_DISTRO_URL" ]]; then
    $IDF_PATH/tools/idf_tools.py --non-interactive install esp-clang

    REEXPORT_NEEDED=1
  fi

  if [[ $REEXPORT_NEEDED -eq 1 ]]; then
    section_start "re_source_export" "Re-source export.sh"
    source ./export.sh
    section_end "re_source_export"
  fi

  # Custom clang toolchain
  if [[ "$IDF_TOOLCHAIN" == "clang" && ! -z "$CI_CLANG_DISTRO_URL" ]]; then
    echo "Using custom clang from ${CI_CLANG_DISTRO_URL}"
    wget $CI_CLANG_DISTRO_URL
    ARCH_NAME=$(basename $CI_CLANG_DISTRO_URL)
    tar -x -f $ARCH_NAME
    export PATH=$PWD/esp-clang/bin:$PATH
  fi

  # Custom OpenOCD
  if [[ "$CI_JOB_STAGE" == "target_test" ]]; then
    machine="$(uname -m)"
    if [[ "$machine" == "armv7l" ]] ; then
      OOCD_DISTRO_URL="$OOCD_DISTRO_URL_ARMHF"
    elif [[ "$machine" == "aarch64" ]] ; then
      OOCD_DISTRO_URL="$OOCD_DISTRO_URL_ARM64"
    fi
    if [[ ! -z "$OOCD_DISTRO_URL" ]]; then
      echo "Using custom OpenOCD from ${OOCD_DISTRO_URL}"
      wget $OOCD_DISTRO_URL
      ARCH_NAME=$(basename $OOCD_DISTRO_URL)
      tar -x -f $ARCH_NAME
      export OPENOCD_SCRIPTS=$PWD/openocd-esp32/share/openocd/scripts
      export PATH=$PWD/openocd-esp32/bin:$PATH
    fi
  fi

  if [[ -n "$CI_PYTHON_TOOL_REPO" ]]; then
    git clone --quiet --depth=1 -b ${CI_PYTHON_TOOL_BRANCH} https://gitlab-ci-token:${ESPCI_TOKEN}@${GITLAB_HTTPS_HOST}/espressif/${CI_PYTHON_TOOL_REPO}.git
    pip install ./${CI_PYTHON_TOOL_REPO}
    rm -rf ${CI_PYTHON_TOOL_REPO}
  fi

  info "setup tools and python venv done"

.show_ccache_statistics: &show_ccache_statistics |
  # Show ccache statistics if enabled globally
  section_start "ccache_show_stats" "Show ccache statistics"
  test "$CI_CCACHE_STATS" == 1 && test -n "$(which ccache)" && ccache --show-stats -vv || true
  section_end "ccache_show_stats"

.upload_failed_job_log_artifacts: &upload_failed_job_log_artifacts |
  if [ $CI_JOB_STATUS = "failed" ]; then
    run_cmd idf-ci gitlab upload-artifacts --type log
  fi

.before_script:minimal:
  before_script:
    - *common-before_scripts

.before_script:build:
  before_script:
    - *common-before_scripts
    - *setup_tools_and_idf_python_venv
    - add_gitlab_ssh_keys
    - fetch_submodules
    - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
    - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}

.after_script:build:
  after_script:
    - source tools/ci/utils.sh
    - *show_ccache_statistics
    - *upload_failed_job_log_artifacts

##############################
# Git Strategy Job Templates #
##############################
.git_init: &git_init |
  mkdir -p "${CI_PROJECT_DIR}"
  cd "${CI_PROJECT_DIR}"
  git init

.git_fetch_from_mirror_url_if_exists: &git_fetch_from_mirror_url_if_exists |
  # check if set mirror
  if [ -n "${LOCAL_GITLAB_HTTPS_HOST:-}" ] && [ -n "${ESPCI_TOKEN:-}" ]; then
    MIRROR_REPO_URL="https://bot:${ESPCI_TOKEN}@${LOCAL_GITLAB_HTTPS_HOST}/${CI_PROJECT_PATH}"
  elif [ -n "${LOCAL_GIT_MIRROR:-}" ]; then
    MIRROR_REPO_URL="${LOCAL_GIT_MIRROR}/${CI_PROJECT_PATH}"
  fi

  # fetch from mirror first if set
  if [ -n "${MIRROR_REPO_URL:-}" ]; then
    if git remote -v | grep origin; then
      git remote set-url origin "${MIRROR_REPO_URL}"
    else
      git remote add origin "${MIRROR_REPO_URL}"
    fi
    # mirror url may fail with authentication issue
    git fetch origin --no-recurse-submodules || true
  fi

  # set remote url to CI_REPOSITORY_URL
  if git remote -v | grep origin; then
    git remote set-url origin "${CI_REPOSITORY_URL}"
  else
    git remote add origin "${CI_REPOSITORY_URL}"
  fi

.git_checkout_ci_commit_sha: &git_checkout_ci_commit_sha |
  git checkout $CI_COMMIT_SHA
  eval "git clean ${GIT_CLEAN_FLAGS}"

# git diff requires two commits, with different CI env var
#
# By default, we use git strategy "clone" with depth 1 to speed up the clone process.
# But for jobs requires running `git diff`, we need to fetch more commits to get the correct diffs.
#
# Since there's no way to get the correct git_depth before the job starts,
# we can't set `GIT_DEPTH` in the job definition.
#
# Set git strategy to "none" and fetch manually instead.
.before_script:fetch:git_diff:
  variables:
    GIT_STRATEGY: none
  before_script:
    - *git_init
    - *git_fetch_from_mirror_url_if_exists
    - |
      # Store the diff output in a temporary file
      TEMP_FILE=$(mktemp)
      # merged results pipelines, by default
      if [[ -n $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA ]]; then
        git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_SHA
        git fetch origin $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA

        git diff --name-only $CI_MERGE_REQUEST_TARGET_BRANCH_SHA...$CI_MERGE_REQUEST_SOURCE_BRANCH_SHA > "$TEMP_FILE"
        GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
        git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
      # merge request pipelines, when the mr got conflicts
      elif [[ -n $CI_MERGE_REQUEST_DIFF_BASE_SHA ]]; then
        git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
        git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
        git diff --name-only $CI_MERGE_REQUEST_DIFF_BASE_SHA $CI_COMMIT_SHA > "$TEMP_FILE"
        GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
      # other pipelines, like the protected branches pipelines
      elif [[ "$CI_COMMIT_BEFORE_SHA" != "0000000000000000000000000000000000000000" ]]; then
        git fetch origin $CI_COMMIT_BEFORE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
        git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
        git diff --name-only $CI_COMMIT_BEFORE_SHA $CI_COMMIT_SHA > "$TEMP_FILE"
        GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
      else
        # pipeline source could be web, scheduler, etc.
        git fetch origin $CI_COMMIT_SHA --depth=2 ${GIT_FETCH_EXTRA_FLAGS}
        git diff --name-only $CI_COMMIT_SHA~1 $CI_COMMIT_SHA > "$TEMP_FILE"
        GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
      fi
    - *git_checkout_ci_commit_sha
    - *common-before_scripts
    - *setup_tools_and_idf_python_venv
    - add_gitlab_ssh_keys

# target test runners may locate in different places
# for runners set git mirror, we fetch from the mirror first, then fetch the HEAD commit
.before_script:fetch:target_test:
  variables:
    GIT_STRATEGY: none
  before_script:
    - *git_init
    - *git_fetch_from_mirror_url_if_exists
    - eval "git fetch --depth=1 ${GIT_FETCH_EXTRA_FLAGS} origin ${CI_COMMIT_SHA}"
    - *git_checkout_ci_commit_sha
    - *common-before_scripts
    - *setup_tools_and_idf_python_venv
    - add_gitlab_ssh_keys
    # no submodules

.brew-macos-settings:
  variables:
    GIT_STRATEGY: none  # we do manual git clone to use local mirror
    IDF_CCACHE_ENABLE: "0"
    CCACHE_DIR: "/var/tmp/cache/idf_ccache"
  tags:
    - macos-tart
  image: macos-sequoia-idf-v6.1
  cache: []  # pip cache is created under amd64, and submodules are downloaded with brew mirror, so disable cache here
  before_script:
    # assert LOCAL_GIT_MIRROR is set
    - echo -e "section_start:`date +%s`:check_out\r\e[0Kchecking out from local git mirror, then reset to CI_COMMIT_SHA"
    - |
      if [ -z "${LOCAL_GIT_MIRROR:-}" ]; then
        echo "Error: LOCAL_GIT_MIRROR not set, cannot clone from mirror."
        exit 1
      fi
    - MIRROR_REPO_URL="${LOCAL_GIT_MIRROR}/${CI_PROJECT_PATH}"
    - cd "${CI_PROJECT_DIR}"
    # since .cache exists in CI_PROJECT_DIR, so can't direct `git clone .`
    - git clone -b ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:-${CI_COMMIT_BRANCH}} --depth=1 --recursive --shallow-submodules "${MIRROR_REPO_URL}" tmp
    - mv tmp/.git ./
    - rm -rf tmp
    - git reset --hard
    # set remote url back
    - git remote set-url origin "${CI_REPOSITORY_URL}"
    - eval "git fetch --depth=1 ${GIT_FETCH_EXTRA_FLAGS} origin ${CI_COMMIT_SHA}"
    - git checkout FETCH_HEAD
    - git submodule update --init --recursive --depth=1
    - echo -e "section_end:`date +%s`:check_out\r\e[0K"
    - *common-before_scripts
    - *setup_tools_and_idf_python_venv
  after_script: []  # ccache now is disabled for macos brew runners
  timeout: 30m

#############
# `default` #
#############
default:
  cache:
    # pull only for most of the use cases since it's cache dir.
    # Only set "push" policy for "upload_cache" stage jobs
    - key: pip-cache-${LATEST_GIT_TAG}
      fallback_keys:
        - pip-cache
      paths:
        - .cache/pip
      policy: pull
    - key: submodule-cache-${LATEST_GIT_TAG}
      fallback_keys:
        - submodule-cache
      paths:
        - .cache/submodule_archives
      policy: pull
  before_script:
    - *common-before_scripts
    - *setup_tools_and_idf_python_venv
    - add_gitlab_ssh_keys
    - fetch_submodules
# gitlab bug, setting them here doesn't work
# - expire_in: https://gitlab.com/gitlab-org/gitlab/-/issues/404563
# - when: https://gitlab.com/gitlab-org/gitlab/-/issues/440672
#  artifacts:
#    expire_in: 1 week
#    when: always
  retry:
    max: 2
    when:
      # In case of a runner failure we could hop to another one, or a network error could go away.
      - runner_system_failure
      # Job execution timeout may be caused by a network issue.
      - job_execution_timeout