diff --git a/tools/docker/matter-builds b/tools/docker/matter-builds
index bd5381f71..daae6a028 100644
--- a/tools/docker/matter-builds
+++ b/tools/docker/matter-builds
@@ -6,14 +6,6 @@ FROM ghcr.io/project-chip/chip-build:${VERSION} as build
 ARG IDF_CLONE_URL=https://github.com/espressif/esp-idf.git
 ARG IDF_CHECKOUT_REF=v5.1
 
-RUN set -x \
-    && apt-get update \
-    && DEBIAN_FRONTEND=noninteractive apt-get install -fy --no-install-recommends \
-    tree \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/ \
-    && : # last line
-
 RUN set -x \
     && mkdir -p /tmp/esp-idf \
     && cd /tmp/esp-idf \
@@ -58,6 +50,14 @@ RUN set -x \
     && ./install.sh \
     && : # last line
 
+RUN set -x \
+    && apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -fy --no-install-recommends \
+    tree \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/ \
+    && : # last line
+
 COPY entrypoint.sh /opt/esp/entrypoint.sh
 ENTRYPOINT [ "/opt/esp/entrypoint.sh" ]
 CMD [ "/bin/bash" ]