diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 1800ab1ac..8a6cfb388 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -169,6 +169,13 @@ build_image:
     - internet
   rules:
     - if: $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH == "main"
+      changes:
+        - tools/docker/chip_idf/**/*
+      variables:
+        DOCKERFILE_CHANGED: "1"
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH == "main"
+      variables:
+        DOCKERFILE_CHANGED: "0"
   before_script:
     - mkdir -p $HOME/.docker
     - echo $DOCKER_AUTH_CONFIG > $HOME/.docker/config.json
@@ -225,11 +232,12 @@ build_image:
     - TAG="chip_${CHIP_SHORT_HASH}_idf_${IDF_VERSION}"
     - echo $TAG
     - |
+      echo "DOCKERFILE_CHANGED=$DOCKERFILE_CHANGED"
       url="https://hub.docker.com/v2/repositories/${DOCKER_IMAGE_NAME}/tags/${TAG}"
-      if wget -q --spider "$url"; then
+      if [ "$DOCKERFILE_CHANGED" != "1" ] && wget -q --spider "$url"; then
           echo "No Differences detected. The docker image with ${TAG} is present in docker hub. Skipping docker build "
       else
-          echo "Differences detected. The docker image with ${TAG} is not present on docker hub. Triggering docker build due to submdoule update or idf version change in CI."
+          echo "Triggering docker build due to: dockerfile_changed=$DOCKERFILE_CHANGED or tag not present on docker hub"
           cd ${ESP_MATTER_PATH}/tools/docker/chip_idf
           docker build --build-arg CHIP_CHECKOUT_REF=${chip_head_commit} --build-arg IDF_CHECKOUT_REF=${IDF_VERSION} --tag ${DOCKER_IMAGE_NAME}:${TAG} .
           docker push ${DOCKER_IMAGE_NAME}:${TAG}
diff --git a/tools/docker/chip_idf/Dockerfile b/tools/docker/chip_idf/Dockerfile
index d2e9acc60..0d749ea48 100644
--- a/tools/docker/chip_idf/Dockerfile
+++ b/tools/docker/chip_idf/Dockerfile
@@ -1,5 +1,5 @@
 ARG VERSION=latest
-FROM ghcr.io/project-chip/chip-build:${VERSION} as build
+FROM ghcr.io/project-chip/chip-build:${VERSION} AS build
 
 # Use IDF_CHECKOUT_REF to specify a tag, a branch or a specific commit ID.
 
@@ -12,6 +12,8 @@ RUN set -x \
     && DEBIAN_FRONTEND=noninteractive apt-get install -fy --fix-missing --no-install-recommends \
         dbus \
         bluez \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* \
     && : # last line
 
 RUN set -x \
@@ -36,7 +38,9 @@ RUN apt-get update && apt-get install -y python3-virtualenv
 # Setup the ESP-IDF
 WORKDIR /opt/espressif/esp-idf
 RUN set -x \
-    && ./install.sh \
+    && ./install.sh --disable-gdbgui --disable-docs \
+    && rm -rf ${IDF_TOOLS_PATH}/dist \
+    && rm -rf /root/.cache \
     && : # last line
 
 ARG CHIP_CLONE_URL=https://github.com/espressif/connectedhomeip.git
@@ -57,6 +61,7 @@ RUN set -x \
     && ./scripts/build_python.sh --enable_ble true --chip_detail_logging true -i out/py-env \
     && bash -c "source out/py-env/bin/activate" \
     && ./scripts/examples/gn_build_example.sh examples/chip-tool out/host \
+    && rm -rf /root/.cache /tmp/* \
     && : # last line
 
 COPY entrypoint.sh /opt/espressif/entrypoint.sh