Merge branch 'ci/stable-build-jobs' into 'master'

Ci/stable build jobs

See merge request espressif/esp-idf!47606

.gitlab/ci/common.yml

@@ -19,6 +19,10 @@ variables:
 
   # Common parameters for the 'make' during CI tests
   MAKEFLAGS: "-j5 --no-keep-going"
+  # By default, CI build jobs request and limit 4 CPU cores and 4 GB of memory
+  # https://github.com/ninja-build/ninja/blob/def9560a0b6d755936e615ce443a0aec45c39bdb/src/ninja.cc#L262
+  # cpu_cores + 2
+  IDF_PY_BUILD_JOBS: "6"
 
 # GitLab-CI environment
   # Thanks to pack-objects cache, clone strategy should behave faster than fetch
@@ -119,6 +123,11 @@ variables:
   # configure cmake related flags
   source tools/ci/configure_ci_environment.sh
 
+  if [[ "$CI_CCACHE_STATS" == 1 ]] && command -v ccache >/dev/null 2>&1 && [[ -n "$CCACHE_STATSLOG" ]]; then
+    mkdir -p "$(dirname "$CCACHE_STATSLOG")"
+    rm -f "$CCACHE_STATSLOG"
+  fi
+
   # add extra python packages
   export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci:$IDF_PATH/tools/esp_app_trace:$IDF_PATH/components/partition_table:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH"
 
@@ -231,7 +240,17 @@ variables:
 .show_ccache_statistics: &show_ccache_statistics |
   # Show ccache statistics if enabled globally
   section_start "ccache_show_stats" "Show ccache statistics"
-  test "$CI_CCACHE_STATS" == 1 && test -n "$(which ccache)" && ccache --show-stats -vv || true
+  if [[ "$CI_CCACHE_STATS" == 1 ]] && command -v ccache >/dev/null 2>&1; then
+    if ccache --help 2>/dev/null | grep -q -- '--show-log-stats'; then
+      if [[ -n "$CCACHE_STATSLOG" && -f "$CCACHE_STATSLOG" ]]; then
+        ccache --show-log-stats -vv
+      else
+        echo "INFO: No per-job ccache statistics were recorded"
+      fi
+    else
+      ccache --show-stats -vv
+    fi
+  fi || true
   section_end "ccache_show_stats"
 
 .upload_failed_job_log_artifacts: &upload_failed_job_log_artifacts |
@@ -246,6 +265,7 @@ variables:
 .after_script:build:
   after_script:
     - source tools/ci/utils.sh
+    - source tools/ci/configure_ci_environment.sh
     - *show_ccache_statistics
     - *upload_failed_job_log_artifacts
 

docs/en/api-guides/build-system.rst

@@ -77,6 +77,14 @@ In the above list, the ``cmake`` command configures the project and generates bu
 
 It's not necessary to run ``cmake`` more than once. After the first build, you only need to run ``ninja`` each time. ``ninja`` will automatically re-invoke ``cmake`` if the project needs reconfiguration.
 
+When using ``idf.py`` with the Ninja generator, you can cap the number of parallel build jobs by setting the ``IDF_PY_BUILD_JOBS`` environment variable. For example:
+
+.. code-block:: bash
+
+    IDF_PY_BUILD_JOBS=6 idf.py build
+
+If you invoke CMake, ``ninja``, or ``make`` directly instead of ``idf.py``, use their native options or environment variables to control parallelism.
+
 If using CMake with ``ninja`` or ``make``, there are also targets for more of the ``idf.py`` sub-commands. For example, running ``make menuconfig`` or ``ninja menuconfig`` in the build directory will work the same as ``idf.py menuconfig``.
 
 .. note::

docs/zh_CN/api-guides/build-system.rst

@@ -77,6 +77,14 @@ idf.py
 
 没有必要多次运行 ``cmake``。第一次构建后，往后每次只需运行 ``ninja`` 即可。如果项目需要重新配置，``ninja`` 会自动重新调用 ``cmake``。
 
+使用 Ninja 生成器配合 ``idf.py`` 时，可以通过设置环境变量 ``IDF_PY_BUILD_JOBS`` 来限制并行构建任务数。例如：
+
+.. code-block:: bash
+
+    IDF_PY_BUILD_JOBS=6 idf.py build
+
+如果不是通过 ``idf.py``，而是直接调用 CMake、``ninja`` 或 ``make``，则请使用它们各自原生的并行控制选项或环境变量。
+
 若在 CMake 中使用 ``ninja`` 或 ``make``，则多数 ``idf.py`` 子命令也会有其对应的目标，例如在构建目录下运行 ``make menuconfig`` 或 ``ninja menuconfig`` 与运行 ``idf.py menuconfig`` 是相同的。
 
 .. note::

tools/ci/configure_ci_environment.sh

@@ -36,9 +36,13 @@ fi
 # https://ccache.dev/manual/latest.html#_configuring_ccache
 # Set ccache base directory to the project checkout path, to cancel out differences between runners
 export CCACHE_BASEDIR="${IDF_PATH}"
+export CCACHE_COMPILERCHECK="${CCACHE_COMPILERCHECK:-content}"
 
 # host mapping volume to share ccache fbetween runner concurrent jobs
-export CCACHE_SLOPPINESS="time_macros"
+export CCACHE_SLOPPINESS="time_macros,file_macro,include_file_mtime,include_file_ctime"
+
+# Keep per-job statistics in the checkout directory while sharing the cache itself.
+export CCACHE_STATSLOG="${CCACHE_STATSLOG:-${IDF_PATH}/.ccache-stats.log}"
 
 # CCACHE_RECACHE Used when invalidating the current cache.
 # could be enabled by MR label "ccache:recache"

tools/idf_py_actions/tools.py

@@ -592,7 +592,20 @@ def run_target(
     if env is None:
         env = {}
 
-    generator_cmd = GENERATORS[args.generator]['command']
+    generator_cmd = list(GENERATORS[args.generator]['command'])
+
+    if args.generator == 'Ninja':
+        parallel_level = os.environ.get('IDF_PY_BUILD_JOBS')
+        if parallel_level:
+            try:
+                jobs = int(parallel_level)
+            except ValueError as e:
+                raise FatalError('Environment variable IDF_PY_BUILD_JOBS must be a positive integer') from e
+
+            if jobs <= 0:
+                raise FatalError('Environment variable IDF_PY_BUILD_JOBS must be a positive integer')
+
+            generator_cmd += ['-j', str(jobs)]
 
     if args.verbose:
         generator_cmd += [GENERATORS[args.generator]['verbose_flag']]