From 27dd31913ae4b30520a0d599ea8cc131f071bd88 Mon Sep 17 00:00:00 2001 From: Shubham Patil Date: Tue, 30 Dec 2025 18:00:33 +0530 Subject: [PATCH] tools: optimize docker image and fix launchpad builds - Earlier fixes removed .git which potentially failed building the examples as esp-idf does git describe during builds. Added them back anad build works now. - Removed the dormant bits from the docker-image - Added ninja-jobs option for better resource utilization during host tool builds - Added back the linux installation as we do need ability to run chip-tool from the container. --- .github/workflows/docker-image.yml | 12 ++++++---- install.sh | 12 ++++++++-- tools/docker/matter_builds/Dockerfile | 33 +++++++++++++++------------ 3 files changed, 35 insertions(+), 22 deletions(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 4747e0389..50a81ea8e 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -34,15 +34,17 @@ jobs: # Disable the job in forks if: ${{ github.repository_owner == 'espressif' }} - runs-on: ubuntu-latest strategy: fail-fast: true matrix: # Platforms to build the image for - # Using a separate runner for each image to work around space issues on a single runner - platform: - - linux/amd64 - - linux/arm64 + # Using native runners for each architecture to avoid slow QEMU emulation + include: + - platform: linux/amd64 + runner: ubuntu-24.04 + - platform: linux/arm64 + runner: ubuntu-24.04-arm + runs-on: ${{ matrix.runner }} steps: # Workaround for disk space issues on the GitHub runner diff --git a/install.sh b/install.sh index 02bfbd2d4..3860c74c6 100755 --- a/install.sh +++ b/install.sh @@ -9,6 +9,10 @@ print_help() { echo " --no-bootstrap Disable sourcing connectedhomeip's scripts/bootstrap.sh," echo " This can be helpful if there's already present connectedhomeip setup" echo " --build-python Build Python environment for running Python test scripts" + echo " --ninja-jobs Number of jobs to use for ninja (default: $(nproc))" + echo " This is used to build the host tools." + echo " This can be helpful in case of slow build machines/docker containers," + echo " or to speed up the build process on faster machines." echo " --help Display this help message" } @@ -21,6 +25,7 @@ echo_log() { NO_HOST_TOOL=false NO_BOOTSTRAP=false BUILD_PYTHON=false +NINJA_JOBS=$(nproc) while [[ "$#" -gt 0 ]]; do case $1 in @@ -33,6 +38,10 @@ while [[ "$#" -gt 0 ]]; do --build-python) BUILD_PYTHON=true ;; + --ninja-jobs) + NINJA_JOBS=${2:-$(nproc)} + shift + ;; --help) print_help exit 1 @@ -63,7 +72,7 @@ fi if [ $NO_HOST_TOOL = false ]; then echo_log "Building host tools" gn --root="${MATTER_PATH}" gen ${MATTER_PATH}/out/host --args='chip_inet_config_enable_ipv4=false' - ninja -C ${MATTER_PATH}/out/host chip-cert chip-tool + ninja -j $NINJA_JOBS -C ${MATTER_PATH}/out/host chip-cert chip-tool echo_log "Host tools built at: ${MATTER_PATH}/out/host" else echo_log "Skip building host tools" @@ -85,7 +94,6 @@ else python3 -m pip install -r ${ESP_MATTER_PATH}/requirements.txt > /dev/null fi - if [ $BUILD_PYTHON = true ]; then echo_log "Building Python testing environment" cd ${MATTER_PATH} diff --git a/tools/docker/matter_builds/Dockerfile b/tools/docker/matter_builds/Dockerfile index 1f3c53499..fb14c797c 100644 --- a/tools/docker/matter_builds/Dockerfile +++ b/tools/docker/matter_builds/Dockerfile @@ -10,6 +10,8 @@ ENV DEBIAN_FRONTEND=noninteractive ENV IDF_PATH=/opt/espressif/esp-idf ENV IDF_TOOLS_PATH=/opt/espressif/tools ENV PATH="${IDF_PATH}/tools:${PATH}" +# Defaulting to 4 as this would be mostly run on resource constrained machines. +ENV NINJA_JOBS=4 # Install minimal dependencies for ESP-IDF and Matter RUN set -x \ @@ -36,7 +38,6 @@ RUN set -x \ wget \ curl \ unzip \ - rsync \ # ESP-IDF requirements flex \ bison \ @@ -59,9 +60,12 @@ RUN set -x \ libmbedtls-dev \ # Additional utilities tree \ - # Cleanup + # Cleanup apt and unnecessary files && apt-get clean \ && rm -rf /var/lib/apt/lists/* \ + && rm -rf /usr/share/doc/* /usr/share/man/* /usr/share/info/* \ + && rm -rf /usr/share/locale/* \ + && rm -rf /var/log/* /var/cache/* \ && git lfs install \ && : # last line @@ -69,11 +73,10 @@ WORKDIR /opt/espressif RUN set -x \ && git clone --depth=1 --branch=${IDF_CHECKOUT_REF} --recurse-submodules --shallow-submodules ${IDF_CLONE_URL} ${IDF_PATH} \ && cd ${IDF_PATH} \ - && ./install.sh \ + && ./install.sh --disable-gdbgui --disable-pytest --disable-ci --disable-docs \ && git rev-parse --short HEAD > /opt/espressif/esp-idf-commit.txt \ - && rm -rf .git \ - && find . -name "__pycache__" -type d -exec rm -rf {} + 2>/dev/null || true \ - && find . -name "*.pyc" -delete 2>/dev/null || true \ + # remove the archives from the tools path + && rm -rf ${IDF_TOOLS_PATH}/dist \ && rm -rf /root/.cache \ && : # last line @@ -90,16 +93,16 @@ RUN set -x \ && git checkout FETCH_HEAD \ && git submodule update --init --depth=1 \ && cd ./connectedhomeip/connectedhomeip \ - && ./scripts/checkout_submodules.py --platform esp32 --shallow \ + && ./scripts/checkout_submodules.py --platform esp32 linux --shallow \ && git rev-parse --short HEAD > /opt/espressif/connectedhomeip-commit.txt \ && cd ${ESP_MATTER_PATH} && git rev-parse --short HEAD > /opt/espressif/esp-matter-commit.txt \ - && . ${IDF_PATH}/export.sh \ - && ./install.sh \ - && rm -rf ${ESP_MATTER_PATH}/.git \ - && rm -rf ${ESP_MATTER_PATH}/.gitmodules \ - && find ${ESP_MATTER_PATH} -name ".git" -exec rm -rf {} + 2>/dev/null || true \ - && find ${ESP_MATTER_PATH} -name "__pycache__" -type d -exec rm -rf {} + 2>/dev/null || true \ - && find ${ESP_MATTER_PATH} -name "*.pyc" -delete 2>/dev/null || true \ + && export IDF_PATH_FORCE=1 && . ${IDF_PATH}/export.sh \ + && cd ${ESP_MATTER_PATH} && ./install.sh --ninja-jobs ${NINJA_JOBS} \ + # Move host tools to /usr/local/bin, strip debug symbols, and remove build directory + && cp ${ESP_MATTER_PATH}/connectedhomeip/connectedhomeip/out/host/chip-cert /usr/local/bin/ \ + && cp ${ESP_MATTER_PATH}/connectedhomeip/connectedhomeip/out/host/chip-tool /usr/local/bin/ \ + && strip /usr/local/bin/chip-cert /usr/local/bin/chip-tool \ + && rm -rf ${ESP_MATTER_PATH}/connectedhomeip/connectedhomeip/out \ && rm -rf /root/.cache /tmp/* \ && : # last line @@ -107,4 +110,4 @@ COPY entrypoint.sh /opt/esp/entrypoint.sh ENTRYPOINT [ "/opt/esp/entrypoint.sh" ] CMD [ "/bin/bash" ] -WORKDIR /opt/espressif/esp-matter \ No newline at end of file +WORKDIR /opt/espressif/esp-matter