CI: Add pytest in CI test

.gitlab-ci.yml

@@ -1,10 +1,12 @@
 stages:
     - build
+    - target_test
     - docs
 
 variables:
     ESP_MATTER_PATH: "$CI_PROJECT_DIR"
     IDF_PATH: "$CI_PROJECT_DIR/esp-idf"
+    BR_PATH: "$CI_PROJECT_DIR/esp-thread-br"
     IDF_GITHUB_ASSETS: "dl.espressif.com/github_assets"
     GIT_STRATEGY: fetch
     GIT_SUBMODULE_STRATEGY: none
@@ -65,12 +67,38 @@ variables:
   - pip install python-gitlab
   - python tools/ci/ci_fetch_submodule.py
 
+.setup_ot_br: &setup_ot_br
+  - cd ${CI_PROJECT_DIR}
+  - git clone ssh://git@gitlab.espressif.cn:27227/espressif/esp-thread-br.git
+  - cd ${BR_PATH}/examples/basic_thread_border_router
+  - idf.py set-target esp32s3
+  - idf.py build
+
+.setup_ot_rcp: &setup_ot_rcp
+  - cd ${IDF_PATH}
+  - ./install.sh
+  - . ./export.sh
+  # fetch submodules
+  - export PYTHONPATH=${IDF_PATH}/tools/ci/python_packages/:${PYTHONPATH}
+  - pip install python-gitlab
+  - python tools/ci/ci_fetch_submodule.py
+  - cd examples/openthread/ot_rcp
+  - idf.py set-target esp32h2
+  - idf.py build
+
 .setup_matter: &setup_matter
   - cd ${CI_PROJECT_DIR}
   # Setting up Python environment still spend a pretty long time (15mins -> 5mins).
   - ./install.sh
   - . ./export.sh
 
+.build_chip_tool: &build_chip_tool
+    - cd ${ESP_MATTER_PATH}/connectedhomeip/connectedhomeip
+    - source scripts/activate.sh
+    - cd ${ESP_MATTER_PATH}/connectedhomeip/connectedhomeip/examples/chip-tool
+    - gn gen out
+    - ninja -C out
+
 .build_matter_examples: &build_matter_examples
     - export MATTER_EXAMPLES_PATH=$ESP_MATTER_PATH/connectedhomeip/connectedhomeip/examples
     - cd $MATTER_EXAMPLES_PATH/all-clusters-app/esp32
@@ -84,32 +112,6 @@ variables:
     - cd $MATTER_EXAMPLES_PATH/persistent-storage/esp32
     - idf.py build
 
-.build_examples: &build_examples
-    - cd $ESP_MATTER_PATH/examples/blemesh_bridge
-    - idf.py set-target esp32
-    - idf.py build
-    - idf.py set-target esp32c3
-    - idf.py build
-    - cd $ESP_MATTER_PATH/examples/zap_light
-    - idf.py set-target esp32
-    - idf.py build
-    - cd $ESP_MATTER_PATH/examples/light_switch
-    - idf.py set-target esp32
-    - idf.py build
-    - cd $ESP_MATTER_PATH/examples/light
-    - idf.py set-target esp32
-    - idf.py build
-    - idf.py set-target esp32c3
-    - idf.py build
-    - cd $ESP_MATTER_PATH/examples/controller
-    - idf.py set-target esp32
-    - idf.py build
-    - cd $ESP_MATTER_PATH/examples/esp-now_bridge_light
-    - idf.py set-target esp32
-    - idf.py build
-    - idf.py set-target esp32c3
-    - idf.py build
-
 .build_external_platform_example: &build_external_platform_example
     - rm -rf $ESP_MATTER_PATH/../platform
     - mkdir $ESP_MATTER_PATH/../platform
@@ -122,32 +124,6 @@ variables:
     - idf.py build
     - cp sdkconfig.defaults.backup sdkconfig.defaults
 
-.build_examples_idf_v5_1: &build_examples_idf_v5_1
-    - cd $ESP_MATTER_PATH/examples/zap_light
-    - idf.py --preview set-target esp32h2
-    - idf.py build
-    - idf.py set-target esp32c2
-    - idf.py build
-    - cd $ESP_MATTER_PATH/examples/light_switch
-    - idf.py --preview set-target esp32h2
-    - idf.py build
-    - idf.py --preview set-target esp32c6
-    - idf.py build
-    - idf.py set-target esp32c2
-    - idf.py build
-    - cd $ESP_MATTER_PATH/examples/light
-    - idf.py --preview set-target esp32h2
-    - idf.py build
-    - idf.py --preview set-target esp32c6
-    - idf.py build
-    - idf.py set-target esp32c2
-    - idf.py build
-    - idf.py set-target esp32c3
-    - idf.py build
-    - cd $ESP_MATTER_PATH/examples/zigbee_bridge
-    - idf.py set-target esp32
-    - idf.py build
-
 .build_examples_template:
   stage: build
   image: gitlab.espressif.cn:5050/app-frameworks/esp-matter/build-env:latest
@@ -171,28 +147,149 @@ variables:
     REPOS_PATH: "$CI_PROJECT_DIR/repos"
     IDF_CCACHE_ENABLE: 1
 
-build_esp_matter_examples:
+build_esp_matter_examples_pytest_C6_idf_v5_1:
-    extends:
-        - .build_examples_template
-    script:
-        - *build_examples
-        - *build_external_platform_example
-
-build_esp_matter_examples_idf_v4_4:
-    extends:
-        - .build_examples_template
-    variables:
-        IDF_VERSION: "v4.4.3"
-    script:
-        - *build_examples
-
-build_esp_matter_examples_idf_v5_1:
   extends:
         - .build_examples_template
+  artifacts:
+    paths:
+      - "examples/**/build*/size.json"
+      - "examples/**/build*/build_log.txt"
+      - "examples/**/build*/*.bin"
+      - "examples/**/build*/flasher_args.json"
+      - "examples/**/build*/config/sdkconfig.json"
+      - "examples/**/build*/bootloader/*.bin"
+      - "examples/**/build*/partition_table/*.bin"
+      - "connectedhomeip/connectedhomeip/examples/chip-tool/out/chip-tool"
+    when: always
+    expire_in: 4 days
   variables:
         IDF_VERSION: "ea5e0ff298e6257b31d8e0c81435e6d3937f04c7"
   script:
-        - *build_examples_idf_v5_1
+    - cd ${ESP_MATTER_PATH}
+    - pip install -r tools/ci/requirements-build.txt
+    - python tools/ci/build_apps.py ./examples --pytest_c6
+    - *build_chip_tool
+  after_script:
+    - find . -name "*.bin"
+
+build_esp_matter_examples_pytest_H2_idf_v5_1:
+  extends:
+        - .build_examples_template
+  artifacts:
+    paths:
+      - "examples/**/build*/size.json"
+      - "examples/**/build*/build_log.txt"
+      - "examples/**/build*/*.bin"
+      - "examples/**/build*/flasher_args.json"
+      - "examples/**/build*/config/sdkconfig.json"
+      - "examples/**/build*/bootloader/*.bin"
+      - "examples/**/build*/partition_table/*.bin"
+      - "${IDF_PATH}/examples/openthread/ot_rcp/build/*.bin"
+      - "${IDF_PATH}/examples/openthread/ot_rcp/build/partition_table/*.bin"
+      - "${IDF_PATH}/examples/openthread/ot_rcp/build/bootloader/*.bin"
+      - "${IDF_PATH}/examples/openthread/ot_rcp/build/config/sdkconfig.json"
+      - "${IDF_PATH}/examples/openthread/ot_rcp/build/flasher_args.json"
+      - "${BR_PATH}/examples/basic_thread_border_router/build/*.bin"
+      - "${BR_PATH}/examples/basic_thread_border_router/build/partition_table/*.bin"
+      - "${BR_PATH}/examples/basic_thread_border_router/build/bootloader/*.bin"
+      - "${BR_PATH}/examples/basic_thread_border_router/build/config/sdkconfig.json"
+      - "${BR_PATH}/examples/basic_thread_border_router/build/flasher_args.json"
+      - "connectedhomeip/connectedhomeip/examples/chip-tool/out/chip-tool"
+    when: always
+    expire_in: 4 days
+  variables:
+        IDF_VERSION: "ea5e0ff298e6257b31d8e0c81435e6d3937f04c7"
+  script:
+    - *setup_ot_rcp
+    - *setup_ot_br
+    - cd ${ESP_MATTER_PATH}
+    - pip install -r tools/ci/requirements-build.txt
+    - python tools/ci/build_apps.py ./examples --pytest_h2
+    - *build_chip_tool
+  after_script:
+    - find . -name "*.bin"
+
+build_esp_matter_examples_non_pytest_idf_v5_1:
+  extends:
+        - .build_examples_template
+  artifacts:
+    paths:
+      - "examples/**/build*/size.json"
+      - "examples/**/build*/build_log.txt"
+      - "examples/**/build*/*.bin"
+      - "examples/**/build*/flasher_args.json"
+      - "examples/**/build*/config/sdkconfig.json"
+      - "examples/**/build*/bootloader/*.bin"
+      - "examples/**/build*/partition_table/*.bin"
+    when: always
+    expire_in: 4 days
+  variables:
+        IDF_VERSION: "ea5e0ff298e6257b31d8e0c81435e6d3937f04c7"
+  script:
+    - *build_external_platform_example
+    - cd ${ESP_MATTER_PATH}
+    - pip install -r tools/ci/requirements-build.txt
+    - python tools/ci/build_apps.py ./examples --no_pytest
+  after_script:
+    - find . -name "*.bin"
+
+build_esp_matter_examples_pytest_C3_idf_v4_4:
+  extends:
+        - .build_examples_template
+  artifacts:
+    paths:
+      - "examples/**/build*/size.json"
+      - "examples/**/build*/build_log.txt"
+      - "examples/**/build*/*.bin"
+      - "examples/**/build*/flasher_args.json"
+      - "examples/**/build*/config/sdkconfig.json"
+      - "examples/**/build*/bootloader/*.bin"
+      - "examples/**/build*/partition_table/*.bin"
+      - "connectedhomeip/connectedhomeip/examples/chip-tool/out/chip-tool"
+    when: always
+    expire_in: 4 days
+  variables:
+        IDF_VERSION: "v4.4.3"
+  script:
+    - cd ${ESP_MATTER_PATH}
+    - pip install -r tools/ci/requirements-build.txt
+    - python tools/ci/build_apps.py ./examples --pytest_c3
+    - *build_chip_tool
+  after_script:
+    - find . -name "*.bin"
+
+pytest_esp32c3_esp_matter_dut:
+  stage: target_test
+  image: ${TARGET_TEST_ENV}
+  needs:
+    - build_esp_matter_examples_pytest_C3_idf_v4_4
+  script:
+    - cd ${ESP_MATTER_PATH}
+    - pip install -r tools/ci/requirements-pytest.txt
+    - pytest examples/ --target esp32c3 -m esp_matter_dut --junitxml=XUNIT_RESULT.xml
+  tags: ["esp32c3", "esp_matter_dut"]
+
+pytest_esp32c6_esp_matter_dut:
+  stage: target_test
+  image: ${TARGET_TEST_ENV}
+  needs:
+    - build_esp_matter_examples_pytest_C6_idf_v5_1
+  script:
+    - cd ${ESP_MATTER_PATH}
+    - pip install -r tools/ci/requirements-pytest.txt
+    - pytest examples/ --target esp32c6 -m esp_matter_dut --junitxml=XUNIT_RESULT.xml
+  tags: ["esp32c6", "esp_matter_dut"]
+
+pytest_esp32h2_esp_matter_dut:
+  stage: target_test
+  image: ${TARGET_TEST_ENV}
+  needs:
+    - build_esp_matter_examples_pytest_H2_idf_v5_1
+  script:
+    - cd ${ESP_MATTER_PATH}
+    - pip install -r tools/ci/requirements-pytest.txt
+    - pytest examples/ --target esp32h2 -m esp_matter_dut --junitxml=XUNIT_RESULT.xml
+  tags: ["esp32h2", "esp_matter_dut"]
 
 build_upstream_examples:
     extends:
@@ -203,7 +300,7 @@ build_upstream_examples:
 
 build_docs:
     stage: build
-    image: $CI_DOCKER_REGISTRY/esp-idf-doc-env-v5.0:2-3
+    image: $CI_DOCKER_REGISTRY/esp-idf-doc-env-v5.1:1-1
     tags:
         - build
     variables:
@@ -221,7 +318,7 @@ build_docs:
 
 .deploy_docs_template:
     stage: docs
-    image: $CI_DOCKER_REGISTRY/esp-idf-doc-env-v5.0:2-3
+    image: $CI_DOCKER_REGISTRY/esp-idf-doc-env-v5.1:1-1
     tags:
         - docs
     needs:

conftest.py

@@ -0,0 +1,269 @@
+# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+
+# pylint: disable=W0621  # redefined-outer-name
+
+# This file is a pytest root configuration file and provide the following functionalities:
+# 1. Defines a few fixtures that could be used under the whole project.
+# 2. Defines a few hook functions.
+#
+# IDF is using [pytest](https://github.com/pytest-dev/pytest) and
+# [pytest-embedded plugin](https://github.com/espressif/pytest-embedded) as its example test framework.
+#
+# This is an experimental feature, and if you found any bug or have any question, please report to
+# https://github.com/espressif/pytest-embedded/issues
+
+import logging
+import os
+import xml.etree.ElementTree as ET
+from datetime import datetime
+from typing import Callable, List, Optional, Tuple
+
+import pytest
+from _pytest.config import Config, ExitCode
+from _pytest.fixtures import FixtureRequest
+from _pytest.main import Session
+from _pytest.nodes import Item
+from _pytest.python import Function
+from _pytest.reports import TestReport
+from _pytest.runner import CallInfo
+from _pytest.terminal import TerminalReporter
+from pytest_embedded.plugin import multi_dut_argument, multi_dut_fixture
+from pytest_embedded.utils import find_by_suffix
+
+
+DEFAULT_SDKCONFIG = 'default'
+
+
+##################
+# Help Functions #
+##################
+def format_case_id(target: Optional[str], config: Optional[str], case: str) -> str:
+    return f'{target}.{config}.{case}'
+
+
+def item_marker_names(item: Item) -> List[str]:
+    return [marker.name for marker in item.iter_markers()]
+
+
+############
+# Fixtures #
+############
+@pytest.fixture(scope='session', autouse=True)
+def session_tempdir() -> str:
+    _tmpdir = os.path.join(
+        os.path.dirname(__file__),
+        'pytest_embedded_log',
+        datetime.now().strftime('%Y-%m-%d_%H-%M-%S'),
+    )
+    os.makedirs(_tmpdir, exist_ok=True)
+    return _tmpdir
+
+
+@pytest.fixture
+@multi_dut_argument
+def config(request: FixtureRequest) -> str:
+    return getattr(request, 'param', None) or DEFAULT_SDKCONFIG
+
+
+@pytest.fixture
+def test_func_name(request: FixtureRequest) -> str:
+    return request.node.function.__name__  # type: ignore
+
+
+@pytest.fixture
+def test_case_name(request: FixtureRequest, target: str, config: str) -> str:
+    return format_case_id(target, config, request.node.originalname)
+
+
+@pytest.fixture
+@multi_dut_fixture
+def build_dir(app_path: str, target: Optional[str], config: Optional[str]) -> str:
+    """
+    Check local build dir with the following priority:
+
+    1. build_<target>_<config>
+    2. build_<target>
+    3. build_<config>
+    4. build
+
+    Args:
+        app_path: app path
+        target: target
+        config: config
+
+    Returns:
+        valid build directory
+    """
+
+    check_dirs = []
+    if target is not None and config is not None:
+        check_dirs.append(f'build_{target}_{config}')
+    if target is not None:
+        check_dirs.append(f'build_{target}')
+    if config is not None:
+        check_dirs.append(f'build_{config}')
+    check_dirs.append('build')
+
+    for check_dir in check_dirs:
+        binary_path = os.path.join(app_path, check_dir)
+        if os.path.isdir(binary_path):
+            logging.info(f'find valid binary path: {binary_path}')
+            return check_dir
+
+        logging.warning(
+            'checking binary path: %s... missing... try another place', binary_path
+        )
+
+    recommend_place = check_dirs[0]
+    raise ValueError(
+        f'no build dir valid. Please build the binary via "idf.py -B {recommend_place} build" and run pytest again'
+    )
+
+
+@pytest.fixture(autouse=True)
+@multi_dut_fixture
+def junit_properties(
+    test_case_name: str, record_xml_attribute: Callable[[str, object], None]
+) -> None:
+    """
+    This fixture is autoused and will modify the junit report test case name to <target>.<config>.<case_name>
+    """
+    record_xml_attribute('name', test_case_name)
+
+
+##################
+# Hook functions #
+##################
+_idf_pytest_embedded_key = pytest.StashKey['IdfPytestEmbedded']
+
+
+def pytest_configure(config: Config) -> None:
+    # cli option "--target"
+    target = config.getoption('target') or ''
+
+    help_commands = ['--help', '--fixtures', '--markers', '--version']
+    for cmd in help_commands:
+        if cmd in config.invocation_params.args:
+            target = 'unneeded'
+            break
+
+    assert target, "Must specify target by --target"
+
+    config.stash[_idf_pytest_embedded_key] = IdfPytestEmbedded(
+        target=target,
+    )
+    config.pluginmanager.register(config.stash[_idf_pytest_embedded_key])
+
+
+def pytest_unconfigure(config: Config) -> None:
+    _pytest_embedded = config.stash.get(_idf_pytest_embedded_key, None)
+    if _pytest_embedded:
+        del config.stash[_idf_pytest_embedded_key]
+        config.pluginmanager.unregister(_pytest_embedded)
+
+
+class IdfPytestEmbedded:
+    def __init__(
+        self,
+        target: Optional[str] = None,
+    ):
+        # CLI options to filter the test cases
+        self.target = target
+        self._failed_cases: List[
+            Tuple[str, bool, bool]
+        ] = []  # (test_case_name, is_known_failure_cases, is_xfail)
+
+    @property
+    def failed_cases(self) -> List[str]:
+        return [
+            case
+            for case, is_xfail in self._failed_cases
+            if not is_xfail
+        ]
+
+    @property
+    def xfail_cases(self) -> List[str]:
+        return [case for case, is_xfail in self._failed_cases if is_xfail]
+
+    @pytest.hookimpl(tryfirst=True)
+    def pytest_sessionstart(self, session: Session) -> None:
+        if self.target:
+            self.target = self.target.lower()
+            session.config.option.target = self.target
+
+    @pytest.hookimpl(tryfirst=True)
+    def pytest_collection_modifyitems(self, items: List[Function]) -> None:
+        # sort by file path and callspec.config
+        # implement like this since this is a limitation of pytest, couldn't get fixture values while collecting
+        # https://github.com/pytest-dev/pytest/discussions/9689
+        def _get_param_config(_item: Function) -> str:
+            if hasattr(_item, 'callspec'):
+                return _item.callspec.params.get('config', DEFAULT_SDKCONFIG)  # type: ignore
+            return DEFAULT_SDKCONFIG
+
+        items.sort(key=lambda x: (os.path.dirname(x.path), _get_param_config(x)))
+
+        # set default timeout 10 minutes for each case
+        for item in items:
+            if 'timeout' not in item.keywords:
+                item.add_marker(pytest.mark.timeout(10 * 60))
+
+        # filter all the test cases with "--target"
+        if self.target:
+            items[:] = [
+                item for item in items if self.target in item_marker_names(item)
+            ]
+
+    def pytest_runtest_makereport(
+        self, item: Function, call: CallInfo[None]
+    ) -> Optional[TestReport]:
+        report = TestReport.from_item_and_call(item, call)
+        if report.outcome == 'failed':
+            test_case_name = item.funcargs.get('test_case_name', '')
+            is_xfail = report.keywords.get('xfail', False)
+            self._failed_cases.append((test_case_name, is_xfail))
+
+        return report
+
+    @pytest.hookimpl(trylast=True)
+    def pytest_runtest_teardown(self, item: Function) -> None:
+        """
+        Format the test case generated junit reports
+        """
+        tempdir = item.funcargs.get('test_case_tempdir')
+        if not tempdir:
+            return
+
+        junits = find_by_suffix('.xml', tempdir)
+        if not junits:
+            return
+
+        target = item.funcargs['target']
+        config = item.funcargs['config']
+        for junit in junits:
+            xml = ET.parse(junit)
+            testcases = xml.findall('.//testcase')
+            for case in testcases:
+                case.attrib['name'] = format_case_id(
+                    target, config, case.attrib['name']
+                )
+                if 'file' in case.attrib:
+                    case.attrib['file'] = case.attrib['file'].replace(
+                        '/IDF/', ''
+                    )  # our unity test framework
+            xml.write(junit)
+
+    def pytest_sessionfinish(self, session: Session, exitstatus: int) -> None:
+        if exitstatus != 0:
+            if exitstatus == ExitCode.NO_TESTS_COLLECTED:
+                session.exitstatus = 0
+
+    def pytest_terminal_summary(self, terminalreporter: TerminalReporter) -> None:
+        if self.xfail_cases:
+            terminalreporter.section('xfail cases', bold=True, yellow=True)
+            terminalreporter.line('\n'.join(self.xfail_cases))
+
+        if self.failed_cases:
+            terminalreporter.section('Failed cases', bold=True, red=True)
+            terminalreporter.line('\n'.join(self.failed_cases))

device_hal/device/esp32c2_devkit_m/esp_matter_device.cmake

@@ -5,7 +5,7 @@ endif()
 
 SET(device_type     esp32c2_devkit_m)
 SET(led_type        gpio)
-SET(button_type     hollow_button)
+SET(button_type     iot_button)
 
 SET(extra_components_dirs_append "$ENV{ESP_MATTER_DEVICE_PATH}/../../led_driver"
-                                 "$ENV{ESP_MATTER_DEVICE_PATH}/../../button_driver/button")
+                                 "$ENV{ESP_MATTER_DEVICE_PATH}/../../button_driver/iot_button")

examples/.build-rules.yml

@@ -0,0 +1,49 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+examples/blemesh_bridge:
+  enable:
+    - if: IDF_TARGET in ["esp32", "esp32c3"]
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/zap_light:
+  enable:
+    - if: IDF_TARGET in ["esp32", "esp32h2"]
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/light_switch:
+  enable:
+    - if: IDF_TARGET in ["esp32", "esp32c6", "esp32c2", "esp32h2"]
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/light:
+  enable:
+    - if: IDF_TARGET in ["esp32", "esp32c3", "esp32c2", "esp32c6", "esp32h2"]
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/generic_switch:
+  enable:
+    - if: IDF_TARGET in ["esp32c2", "esp32c6", "esp32h2"]
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/esp-now_bridge_light:
+  enable:
+    - if: IDF_TARGET in ["esp32c3"]
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/controller:
+  enable:
+    - if: IDF_TARGET in ["esp32"]
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/zigbee_bridge:
+  enable:
+    - if: IDF_TARGET in ["esp32"]
+      temporary: true
+      reason: the other targets are not tested yet

examples/pytest_esp_matter_light.py

@@ -0,0 +1,237 @@
+# SPDX-License-Identifier: CC0-1.0
+
+import pathlib
+import pytest
+import time
+import re
+import pexpect
+import subprocess
+import netifaces
+from typing import Tuple
+from pytest_embedded import Dut
+
+CURRENT_DIR_LIGHT = str(pathlib.Path(__file__).parent)+'/light'
+CHIP_TOOL_DIR = str(pathlib.Path(__file__).parent)+'/../connectedhomeip/connectedhomeip/examples/chip-tool'
+OT_BR_EXAMPLE_PATH = str(pathlib.Path(__file__).parent)+'/../esp-thread-br/examples/basic_thread_border_router'
+pytest_build_dir = CURRENT_DIR_LIGHT
+pytest_matter_thread_dir = CURRENT_DIR_LIGHT+'|'+OT_BR_EXAMPLE_PATH
+
+
+@pytest.mark.esp32c3
+@pytest.mark.esp_matter_dut
+@pytest.mark.parametrize(
+    ' count, app_path, target, erase_all', [
+        ( 1, pytest_build_dir, 'esp32c3', 'y'),
+    ],
+    indirect=True,
+)
+
+# Matter over wifi commissioning
+def test_matter_commissioning_c3(dut:Dut) -> None:
+    light = dut
+    # BLE start advertising
+    light.expect(r'chip\[DL\]\: Configuring CHIPoBLE advertising', timeout=20)
+    # Start commissioning
+    time.sleep(5)
+    command = CHIP_TOOL_DIR+'/out/chip-tool pairing ble-wifi 1 ChipTEH2 chiptest123 20202021 3840'
+    out_str = subprocess.getoutput(command)
+    print(out_str)
+    result = re.findall(r'Run command failure', str(out_str))
+    if len(result) != 0:
+      assert False
+    # Use toggle command to turn-off the light
+    time.sleep(3)
+    command = CHIP_TOOL_DIR+'/out/chip-tool onoff toggle 1 1'
+    out_str = subprocess.getoutput(command)
+    print(out_str)
+    result = re.findall(r'Run command failure', str(out_str))
+    if len(result) != 0:
+      assert False
+    # Use toggle command to turn-on the light
+    time.sleep(5)
+    command = CHIP_TOOL_DIR+'/out/chip-tool onoff toggle 1 1'
+    out_str = subprocess.getoutput(command)
+    print(out_str)
+    result = re.findall(r'Run command failure', str(out_str))
+    if len(result) != 0:
+      assert False      
+
+@pytest.mark.esp32c6
+@pytest.mark.esp_matter_dut
+@pytest.mark.parametrize(
+    ' count, app_path, target, erase_all', [
+        ( 1, pytest_build_dir, 'esp32c6', 'y'),
+    ],
+    indirect=True,
+)
+
+# Matter over wifi commissioning
+def test_matter_commissioning_c6(dut:Dut) -> None:
+    light = dut
+    # BLE start advertising
+    light.expect(r'chip\[DL\]\: Configuring CHIPoBLE advertising', timeout=20)
+    # Start commissioning
+    time.sleep(5)
+    command = CHIP_TOOL_DIR+'/out/chip-tool pairing ble-wifi 1 ChipTEH2 chiptest123 20202021 3840'
+    out_str = subprocess.getoutput(command)
+    print(out_str)
+    result = re.findall(r'Run command failure', str(out_str))
+    if len(result) != 0:
+      assert False
+    # Use toggle command to turn-off the light
+    time.sleep(3)
+    command = CHIP_TOOL_DIR+'/out/chip-tool onoff toggle 1 1'
+    out_str = subprocess.getoutput(command)
+    print(out_str)
+    result = re.findall(r'Run command failure', str(out_str))
+    if len(result) != 0:
+      assert False
+    # Use toggle command to turn-on the light
+    time.sleep(5)
+    command = CHIP_TOOL_DIR+'/out/chip-tool onoff toggle 1 1'
+    out_str = subprocess.getoutput(command)
+    print(out_str)
+    result = re.findall(r'Run command failure', str(out_str))
+    if len(result) != 0:
+      assert False   
+
+
+# get the host interface name
+def get_host_interface_name() -> str:
+    interfaces = netifaces.interfaces()
+    interface_name = [s for s in interfaces if 'wl' in s][0]
+    return str(interface_name)
+
+
+# reset host interface
+def reset_host_interface() -> None:
+    interface_name = get_host_interface_name()
+    flag = False
+    try:
+        command = 'ifconfig ' + interface_name + ' down'
+        subprocess.call(command, shell=True, timeout=5)
+        time.sleep(1)
+        command = 'ifconfig ' + interface_name + ' up'
+        subprocess.call(command, shell=True, timeout=10)
+        time.sleep(1)
+        flag = True
+    finally:
+        time.sleep(1)
+        assert flag
+
+
+# set interface sysctl options
+def set_interface_sysctl_options() -> None:
+    interface_name = get_host_interface_name()
+    flag = False
+    try:
+        command = 'sysctl -w net/ipv6/conf/' + interface_name + '/accept_ra=2'
+        subprocess.call(command, shell=True, timeout=5)
+        time.sleep(1)
+        command = 'sysctl -w net/ipv6/conf/' + interface_name + '/accept_ra_rt_info_max_plen=128'
+        subprocess.call(command, shell=True, timeout=5)
+        time.sleep(1)
+        command = 'sysctl -w net.ipv6.conf.all.forwarding=1'
+        subprocess.call(command, shell=True, timeout=5)
+        time.sleep(1)
+        flag = True
+    finally:
+        time.sleep(2)
+        assert flag
+
+
+# initialize interface ipv6 address
+def init_interface_ipv6_address() -> None:
+    interface_name = get_host_interface_name()
+    flag = False
+    try:
+        command = 'ip -6 route | grep ' + interface_name + " | grep ra | awk {'print $1'} | xargs -I {} ip -6 route del {}"
+        subprocess.call(command, shell=True, timeout=5)
+        time.sleep(0.5)
+        subprocess.call(command, shell=True, timeout=5)
+        time.sleep(1)
+        command = 'ip -6 address show dev ' + interface_name + \
+            " scope global | grep 'inet6' | awk {'print $2'} | xargs -I {} ip -6 addr del {} dev " + interface_name
+        subprocess.call(command, shell=True, timeout=5)
+        time.sleep(1)
+        flag = True
+    finally:
+        time.sleep(1)
+        assert flag
+
+
+def fixture_Init_interface() -> bool:
+    print('Init interface')
+    init_interface_ipv6_address()
+    reset_host_interface()
+    time.sleep(30)
+    set_interface_sysctl_options()
+    return True
+
+
+@pytest.mark.esp32h2
+@pytest.mark.esp32s3
+@pytest.mark.esp_matter_dut
+@pytest.mark.parametrize(
+    'count, app_path, target, erase_all', [
+        ( 2, pytest_matter_thread_dir, 'esp32h2|esp32s3', 'y|n'),
+    ],
+    indirect=True,
+)
+
+# Matter over thread commissioning
+def test_matter_commissioning_h2(dut:Tuple[Dut, Dut]) -> None:
+    ot_br = dut[1]
+    light = dut[0]
+    # For matter over thread commissioning need to reset host interface
+    fixture_Init_interface()
+    # BLE start advertising
+    light.expect(r'chip\[DL\]\: Configuring CHIPoBLE advertising', timeout=20)
+    # flash ot_br
+    ot_br.expect('OpenThread attached to netif', timeout=30)
+    time.sleep(2)
+    ot_br.write('factoryreset')
+    ot_br.expect('OpenThread attached to netif', timeout=30)
+    time.sleep(2)
+    ot_br.write('log level 3')
+    ot_br.expect('Done', timeout=5)
+    time.sleep(2)
+    # wifi connect -s ChipTEH2 -p chiptest123
+    ot_br.write('wifi connect -s ChipTEH2 -p chiptest123')
+    ot_br.expect('wifi sta is connected successfully', timeout=5)
+    time.sleep(2)
+    # start an ot network
+    ot_br.write('ifconfig up')
+    ot_br.expect('netif up', timeout=5)
+    time.sleep(2)
+    ot_br.write('thread start')
+    ot_br.expect('Role detached -> leader', timeout=20)
+    time.sleep(2)
+   # get dataset
+    ot_br.write('dataset active -x')
+    dataset=ot_br.expect(r'\n(\w{202}\r)', timeout=5)[1].decode()
+    print(dataset)
+    # Start commissioning
+    time.sleep(2)
+    command = CHIP_TOOL_DIR+"/out/chip-tool pairing ble-thread 1 hex:{} ".format(dataset.strip())+"20202021 3840"
+    out_str = subprocess.getoutput(command)
+    print(out_str)
+    result = re.findall(r'Run command failure', str(out_str))
+    if len(result) != 0:
+      assert False
+    # Use toggle command to turn-off the light
+    time.sleep(2)
+    command = CHIP_TOOL_DIR+'/out/chip-tool onoff toggle 1 1'
+    out_str = subprocess.getoutput(command)
+    print(out_str)
+    result = re.findall(r'Run command failure', str(out_str))
+    if len(result) != 0:
+      assert False
+    # Use toggle command to turn-on the light
+    time.sleep(2)
+    command = CHIP_TOOL_DIR+'/out/chip-tool onoff toggle 1 1'
+    out_str = subprocess.getoutput(command)
+    print(out_str)
+    result = re.findall(r'Run command failure', str(out_str))
+    if len(result) != 0:
+      assert False

pytest.ini

@@ -0,0 +1,39 @@
+[pytest]
+
+python_files = pytest_*.py
+
+# ignore PytestExperimentalApiWarning for record_xml_attribute
+# set traceback to "short" to prevent the overwhelming tracebacks
+addopts =
+  -s
+  --embedded-services esp,idf
+  --tb short
+  --strict-markers
+  --skip-check-coredump y
+  --logfile-extension ".txt"
+
+# ignore DeprecationWarning
+filterwarnings =
+  ignore::_pytest.warning_types.PytestExperimentalApiWarning
+
+markers =
+  # target markers
+  esp32c3: support esp32c3 target
+  esp32c6: support esp32c6 target
+  esp32h2: support esp32h2 target
+  esp32s3: support esp32s3 target
+  # env markers
+  esp_matter_dut: esp matter runner which have single dut
+  
+# log related
+log_cli = True
+log_cli_level = INFO
+log_cli_format = %(asctime)s %(levelname)s %(message)s
+log_cli_date_format = %Y-%m-%d %H:%M:%S
+
+# junit related
+junit_family = xunit1
+
+## log all to `system-out` when case fail
+junit_logging = stdout
+junit_log_passing_tests = False

tools/ci/build_apps.py

@@ -0,0 +1,179 @@
+# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+
+# SPDX-License-Identifier: CC0-1.0
+
+# This file is used in CI generate binary files for different kinds of apps
+
+import argparse
+import sys
+from pathlib import Path
+from typing import List
+
+from idf_build_apps import LOGGER, App, build_apps, find_apps, setup_logging
+
+# from idf_ci_utils import IDF_PATH, get_pytest_app_paths, get_pytest_cases, get_ttfw_app_paths
+
+PROJECT_ROOT = Path(__file__).parent.parent.parent.absolute()
+DEF_APP_PATH = PROJECT_ROOT / 'examples'
+APPS_BUILD_PER_JOB = 30
+PYTEST_C6_APPS = [
+    {"target": "esp32c6", "name": "light"},
+]
+MAINFEST_FILES = [
+    str(PROJECT_ROOT / 'examples' / '.build-rules.yml'),
+]
+
+PYTEST_H2_APPS = [
+    {"target": "esp32h2", "name": "light"},
+]
+MAINFEST_FILES = [
+    str(PROJECT_ROOT / 'examples' / '.build-rules.yml'),
+]
+
+PYTEST_C3_APPS = [
+    {"target": "esp32c3", "name": "light"},
+]
+MAINFEST_FILES = [
+    str(PROJECT_ROOT / 'examples' / '.build-rules.yml'),
+]
+
+def _is_c6_pytest_app(app: App) -> bool:
+    print(app.name , app.target)
+    for pytest_app in PYTEST_C6_APPS:
+        print(pytest_app["name"] , pytest_app["target"])
+        if app.name == pytest_app["name"] and app.target == pytest_app["target"]:
+            return True
+    return False
+
+def _is_h2_pytest_app(app: App) -> bool:
+    for pytest_app in PYTEST_H2_APPS:
+        if app.name == pytest_app["name"] and app.target == pytest_app["target"]:
+            return True
+    return False
+
+def _is_c3_pytest_app(app: App) -> bool:
+    for pytest_app in PYTEST_C3_APPS:
+        if app.name == pytest_app["name"] and app.target == pytest_app["target"]:
+            return True
+    return False
+
+def get_cmake_apps(
+    paths: List[str],
+    target: str,
+    config_rules_str: List[str],
+) -> List[App]:
+    apps = find_apps(
+        paths,
+        recursive=True,
+        target=target,
+        build_dir='build_@t_@w',
+        config_rules_str=config_rules_str,
+        build_log_path='build_log.txt',
+        size_json_path='size.json',
+        check_warnings=False,
+        preserve=True,
+        manifest_files=MAINFEST_FILES,
+    )
+    return apps
+
+
+def main(args: argparse.Namespace) -> None:
+    apps = get_cmake_apps(args.paths, args.target, args.config)
+
+    # no_pytest and only_pytest can not be both True
+    assert not (args.no_pytest and args.pytest_c6 and args.pytest_h2 and args.pytest_c3)
+    if args.no_pytest:
+        apps_for_build = [app for app in apps if not (_is_c6_pytest_app(app) or _is_h2_pytest_app(app))]
+    elif args.pytest_c6:
+        apps_for_build = [app for app in apps if _is_c6_pytest_app(app)]
+    elif args.pytest_h2:
+        apps_for_build = [app for app in apps if _is_h2_pytest_app(app)]
+    elif args.pytest_c3:
+        apps_for_build = [app for app in apps if _is_c3_pytest_app(app)]
+    else:
+        apps_for_build = apps[:]
+
+    LOGGER.info('Found %d apps after filtering', len(apps_for_build))
+    LOGGER.info(
+        'Suggest setting the parallel count to %d for this build job',
+        len(apps_for_build) // APPS_BUILD_PER_JOB + 1,
+    )
+
+    ret_code = build_apps(
+        apps_for_build,
+        parallel_count=args.parallel_count,
+        parallel_index=args.parallel_index,
+        dry_run=False,
+        collect_size_info=args.collect_size_info,
+        # build_verbose=0,
+        keep_going=True,
+        ignore_warning_strs=[r".*"],
+        copy_sdkconfig=True,
+    )
+
+    sys.exit(ret_code)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description='Build all the apps for different test types. Will auto remove those non-test apps binaries',
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument('paths', nargs='*', help='Paths to the apps to build.')
+    parser.add_argument(
+        '-t',
+        '--target',
+        default='all',
+        help='Build apps for given target. could pass "all" to get apps for all targets',
+    )
+    parser.add_argument(
+        '--config',
+        default=['sdkconfig.ci=default', 'sdkconfig.ci.*=', '=default'],
+        action='append',
+        help='Adds configurations (sdkconfig file names) to build. This can either be '
+        'FILENAME[=NAME] or FILEPATTERN. FILENAME is the name of the sdkconfig file, '
+        'relative to the project directory, to be used. Optional NAME can be specified, '
+        'which can be used as a name of this configuration. FILEPATTERN is the name of '
+        'the sdkconfig file, relative to the project directory, with at most one wildcard. '
+        'The part captured by the wildcard is used as the name of the configuration.',
+    )
+    parser.add_argument(
+        '--parallel-count', default=1, type=int, help='Number of parallel build jobs.'
+    )
+    parser.add_argument(
+        '--parallel-index',
+        default=1,
+        type=int,
+        help='Index (1-based) of the job, out of the number specified by --parallel-count.',
+    )
+    parser.add_argument(
+        '--no_pytest',
+        action="store_true",
+        help='Exclude pytest apps, definded in PYTEST_H2_APPS and PYTEST_C6_APPS',
+    )
+    parser.add_argument(
+        '--pytest_c6',
+        action="store_true",
+        help='Only build pytest apps, definded in PYTEST_C6_APPS',
+    )
+    parser.add_argument(
+        '--pytest_h2',
+        action="store_true",
+        help='Only build pytest apps, definded in PYTEST_H2_APPS',
+    )
+    parser.add_argument(
+        '--pytest_c3',
+        action="store_true",
+        help='Only build pytest apps, definded in PYTEST_C3_APPS',
+    )
+    parser.add_argument(
+        '--collect-size-info',
+        type=argparse.FileType('w'),
+        help='If specified, the test case name and size info json will be written to this file',
+    )
+
+    arguments = parser.parse_args()
+    if not arguments.paths:
+        arguments.paths = [DEF_APP_PATH]
+    setup_logging(verbose=1)  # Info
+    main(arguments)

tools/ci/requirements-build.txt

@@ -0,0 +1 @@
+idf_build_apps

tools/ci/requirements-pytest.txt

@@ -0,0 +1,6 @@
+pytest-embedded-serial-esp~=1.0
+pytest-embedded-idf~=1.0
+pytest-embedded-qemu~=1.0
+pytest-timeout
+netifaces
+esptool>=4.5