From 14d41f39b634d72a2cdd788c4f9461e1c6128391 Mon Sep 17 00:00:00 2001 From: Abudl Rehman Date: Tue, 9 May 2023 16:27:21 +0800 Subject: [PATCH] CI: Add pytest in CI test --- .gitlab-ci.yml | 245 +++++++++++----- conftest.py | 269 ++++++++++++++++++ .../esp32c2_devkit_m/esp_matter_device.cmake | 4 +- examples/.build-rules.yml | 49 ++++ examples/pytest_esp_matter_light.py | 237 +++++++++++++++ pytest.ini | 39 +++ tools/ci/build_apps.py | 179 ++++++++++++ tools/ci/requirements-build.txt | 1 + tools/ci/requirements-pytest.txt | 6 + 9 files changed, 953 insertions(+), 76 deletions(-) create mode 100644 conftest.py create mode 100644 examples/.build-rules.yml create mode 100644 examples/pytest_esp_matter_light.py create mode 100644 pytest.ini create mode 100644 tools/ci/build_apps.py create mode 100644 tools/ci/requirements-build.txt create mode 100644 tools/ci/requirements-pytest.txt diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fa8b9d8d8..bb6d8c3c8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,10 +1,12 @@ stages: - build + - target_test - docs variables: ESP_MATTER_PATH: "$CI_PROJECT_DIR" IDF_PATH: "$CI_PROJECT_DIR/esp-idf" + BR_PATH: "$CI_PROJECT_DIR/esp-thread-br" IDF_GITHUB_ASSETS: "dl.espressif.com/github_assets" GIT_STRATEGY: fetch GIT_SUBMODULE_STRATEGY: none @@ -65,12 +67,38 @@ variables: - pip install python-gitlab - python tools/ci/ci_fetch_submodule.py +.setup_ot_br: &setup_ot_br + - cd ${CI_PROJECT_DIR} + - git clone ssh://git@gitlab.espressif.cn:27227/espressif/esp-thread-br.git + - cd ${BR_PATH}/examples/basic_thread_border_router + - idf.py set-target esp32s3 + - idf.py build + +.setup_ot_rcp: &setup_ot_rcp + - cd ${IDF_PATH} + - ./install.sh + - . ./export.sh + # fetch submodules + - export PYTHONPATH=${IDF_PATH}/tools/ci/python_packages/:${PYTHONPATH} + - pip install python-gitlab + - python tools/ci/ci_fetch_submodule.py + - cd examples/openthread/ot_rcp + - idf.py set-target esp32h2 + - idf.py build + .setup_matter: &setup_matter - cd ${CI_PROJECT_DIR} # Setting up Python environment still spend a pretty long time (15mins -> 5mins). - ./install.sh - . ./export.sh +.build_chip_tool: &build_chip_tool + - cd ${ESP_MATTER_PATH}/connectedhomeip/connectedhomeip + - source scripts/activate.sh + - cd ${ESP_MATTER_PATH}/connectedhomeip/connectedhomeip/examples/chip-tool + - gn gen out + - ninja -C out + .build_matter_examples: &build_matter_examples - export MATTER_EXAMPLES_PATH=$ESP_MATTER_PATH/connectedhomeip/connectedhomeip/examples - cd $MATTER_EXAMPLES_PATH/all-clusters-app/esp32 @@ -84,32 +112,6 @@ variables: - cd $MATTER_EXAMPLES_PATH/persistent-storage/esp32 - idf.py build -.build_examples: &build_examples - - cd $ESP_MATTER_PATH/examples/blemesh_bridge - - idf.py set-target esp32 - - idf.py build - - idf.py set-target esp32c3 - - idf.py build - - cd $ESP_MATTER_PATH/examples/zap_light - - idf.py set-target esp32 - - idf.py build - - cd $ESP_MATTER_PATH/examples/light_switch - - idf.py set-target esp32 - - idf.py build - - cd $ESP_MATTER_PATH/examples/light - - idf.py set-target esp32 - - idf.py build - - idf.py set-target esp32c3 - - idf.py build - - cd $ESP_MATTER_PATH/examples/controller - - idf.py set-target esp32 - - idf.py build - - cd $ESP_MATTER_PATH/examples/esp-now_bridge_light - - idf.py set-target esp32 - - idf.py build - - idf.py set-target esp32c3 - - idf.py build - .build_external_platform_example: &build_external_platform_example - rm -rf $ESP_MATTER_PATH/../platform - mkdir $ESP_MATTER_PATH/../platform @@ -122,32 +124,6 @@ variables: - idf.py build - cp sdkconfig.defaults.backup sdkconfig.defaults -.build_examples_idf_v5_1: &build_examples_idf_v5_1 - - cd $ESP_MATTER_PATH/examples/zap_light - - idf.py --preview set-target esp32h2 - - idf.py build - - idf.py set-target esp32c2 - - idf.py build - - cd $ESP_MATTER_PATH/examples/light_switch - - idf.py --preview set-target esp32h2 - - idf.py build - - idf.py --preview set-target esp32c6 - - idf.py build - - idf.py set-target esp32c2 - - idf.py build - - cd $ESP_MATTER_PATH/examples/light - - idf.py --preview set-target esp32h2 - - idf.py build - - idf.py --preview set-target esp32c6 - - idf.py build - - idf.py set-target esp32c2 - - idf.py build - - idf.py set-target esp32c3 - - idf.py build - - cd $ESP_MATTER_PATH/examples/zigbee_bridge - - idf.py set-target esp32 - - idf.py build - .build_examples_template: stage: build image: gitlab.espressif.cn:5050/app-frameworks/esp-matter/build-env:latest @@ -171,28 +147,149 @@ variables: REPOS_PATH: "$CI_PROJECT_DIR/repos" IDF_CCACHE_ENABLE: 1 -build_esp_matter_examples: - extends: +build_esp_matter_examples_pytest_C6_idf_v5_1: + extends: - .build_examples_template - script: - - *build_examples - - *build_external_platform_example - -build_esp_matter_examples_idf_v4_4: - extends: - - .build_examples_template - variables: - IDF_VERSION: "v4.4.3" - script: - - *build_examples - -build_esp_matter_examples_idf_v5_1: - extends: - - .build_examples_template - variables: + artifacts: + paths: + - "examples/**/build*/size.json" + - "examples/**/build*/build_log.txt" + - "examples/**/build*/*.bin" + - "examples/**/build*/flasher_args.json" + - "examples/**/build*/config/sdkconfig.json" + - "examples/**/build*/bootloader/*.bin" + - "examples/**/build*/partition_table/*.bin" + - "connectedhomeip/connectedhomeip/examples/chip-tool/out/chip-tool" + when: always + expire_in: 4 days + variables: IDF_VERSION: "ea5e0ff298e6257b31d8e0c81435e6d3937f04c7" - script: - - *build_examples_idf_v5_1 + script: + - cd ${ESP_MATTER_PATH} + - pip install -r tools/ci/requirements-build.txt + - python tools/ci/build_apps.py ./examples --pytest_c6 + - *build_chip_tool + after_script: + - find . -name "*.bin" + +build_esp_matter_examples_pytest_H2_idf_v5_1: + extends: + - .build_examples_template + artifacts: + paths: + - "examples/**/build*/size.json" + - "examples/**/build*/build_log.txt" + - "examples/**/build*/*.bin" + - "examples/**/build*/flasher_args.json" + - "examples/**/build*/config/sdkconfig.json" + - "examples/**/build*/bootloader/*.bin" + - "examples/**/build*/partition_table/*.bin" + - "${IDF_PATH}/examples/openthread/ot_rcp/build/*.bin" + - "${IDF_PATH}/examples/openthread/ot_rcp/build/partition_table/*.bin" + - "${IDF_PATH}/examples/openthread/ot_rcp/build/bootloader/*.bin" + - "${IDF_PATH}/examples/openthread/ot_rcp/build/config/sdkconfig.json" + - "${IDF_PATH}/examples/openthread/ot_rcp/build/flasher_args.json" + - "${BR_PATH}/examples/basic_thread_border_router/build/*.bin" + - "${BR_PATH}/examples/basic_thread_border_router/build/partition_table/*.bin" + - "${BR_PATH}/examples/basic_thread_border_router/build/bootloader/*.bin" + - "${BR_PATH}/examples/basic_thread_border_router/build/config/sdkconfig.json" + - "${BR_PATH}/examples/basic_thread_border_router/build/flasher_args.json" + - "connectedhomeip/connectedhomeip/examples/chip-tool/out/chip-tool" + when: always + expire_in: 4 days + variables: + IDF_VERSION: "ea5e0ff298e6257b31d8e0c81435e6d3937f04c7" + script: + - *setup_ot_rcp + - *setup_ot_br + - cd ${ESP_MATTER_PATH} + - pip install -r tools/ci/requirements-build.txt + - python tools/ci/build_apps.py ./examples --pytest_h2 + - *build_chip_tool + after_script: + - find . -name "*.bin" + +build_esp_matter_examples_non_pytest_idf_v5_1: + extends: + - .build_examples_template + artifacts: + paths: + - "examples/**/build*/size.json" + - "examples/**/build*/build_log.txt" + - "examples/**/build*/*.bin" + - "examples/**/build*/flasher_args.json" + - "examples/**/build*/config/sdkconfig.json" + - "examples/**/build*/bootloader/*.bin" + - "examples/**/build*/partition_table/*.bin" + when: always + expire_in: 4 days + variables: + IDF_VERSION: "ea5e0ff298e6257b31d8e0c81435e6d3937f04c7" + script: + - *build_external_platform_example + - cd ${ESP_MATTER_PATH} + - pip install -r tools/ci/requirements-build.txt + - python tools/ci/build_apps.py ./examples --no_pytest + after_script: + - find . -name "*.bin" + +build_esp_matter_examples_pytest_C3_idf_v4_4: + extends: + - .build_examples_template + artifacts: + paths: + - "examples/**/build*/size.json" + - "examples/**/build*/build_log.txt" + - "examples/**/build*/*.bin" + - "examples/**/build*/flasher_args.json" + - "examples/**/build*/config/sdkconfig.json" + - "examples/**/build*/bootloader/*.bin" + - "examples/**/build*/partition_table/*.bin" + - "connectedhomeip/connectedhomeip/examples/chip-tool/out/chip-tool" + when: always + expire_in: 4 days + variables: + IDF_VERSION: "v4.4.3" + script: + - cd ${ESP_MATTER_PATH} + - pip install -r tools/ci/requirements-build.txt + - python tools/ci/build_apps.py ./examples --pytest_c3 + - *build_chip_tool + after_script: + - find . -name "*.bin" + +pytest_esp32c3_esp_matter_dut: + stage: target_test + image: ${TARGET_TEST_ENV} + needs: + - build_esp_matter_examples_pytest_C3_idf_v4_4 + script: + - cd ${ESP_MATTER_PATH} + - pip install -r tools/ci/requirements-pytest.txt + - pytest examples/ --target esp32c3 -m esp_matter_dut --junitxml=XUNIT_RESULT.xml + tags: ["esp32c3", "esp_matter_dut"] + +pytest_esp32c6_esp_matter_dut: + stage: target_test + image: ${TARGET_TEST_ENV} + needs: + - build_esp_matter_examples_pytest_C6_idf_v5_1 + script: + - cd ${ESP_MATTER_PATH} + - pip install -r tools/ci/requirements-pytest.txt + - pytest examples/ --target esp32c6 -m esp_matter_dut --junitxml=XUNIT_RESULT.xml + tags: ["esp32c6", "esp_matter_dut"] + +pytest_esp32h2_esp_matter_dut: + stage: target_test + image: ${TARGET_TEST_ENV} + needs: + - build_esp_matter_examples_pytest_H2_idf_v5_1 + script: + - cd ${ESP_MATTER_PATH} + - pip install -r tools/ci/requirements-pytest.txt + - pytest examples/ --target esp32h2 -m esp_matter_dut --junitxml=XUNIT_RESULT.xml + tags: ["esp32h2", "esp_matter_dut"] build_upstream_examples: extends: @@ -203,7 +300,7 @@ build_upstream_examples: build_docs: stage: build - image: $CI_DOCKER_REGISTRY/esp-idf-doc-env-v5.0:2-3 + image: $CI_DOCKER_REGISTRY/esp-idf-doc-env-v5.1:1-1 tags: - build variables: @@ -221,7 +318,7 @@ build_docs: .deploy_docs_template: stage: docs - image: $CI_DOCKER_REGISTRY/esp-idf-doc-env-v5.0:2-3 + image: $CI_DOCKER_REGISTRY/esp-idf-doc-env-v5.1:1-1 tags: - docs needs: diff --git a/conftest.py b/conftest.py new file mode 100644 index 000000000..5f2f02890 --- /dev/null +++ b/conftest.py @@ -0,0 +1,269 @@ +# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +# pylint: disable=W0621 # redefined-outer-name + +# This file is a pytest root configuration file and provide the following functionalities: +# 1. Defines a few fixtures that could be used under the whole project. +# 2. Defines a few hook functions. +# +# IDF is using [pytest](https://github.com/pytest-dev/pytest) and +# [pytest-embedded plugin](https://github.com/espressif/pytest-embedded) as its example test framework. +# +# This is an experimental feature, and if you found any bug or have any question, please report to +# https://github.com/espressif/pytest-embedded/issues + +import logging +import os +import xml.etree.ElementTree as ET +from datetime import datetime +from typing import Callable, List, Optional, Tuple + +import pytest +from _pytest.config import Config, ExitCode +from _pytest.fixtures import FixtureRequest +from _pytest.main import Session +from _pytest.nodes import Item +from _pytest.python import Function +from _pytest.reports import TestReport +from _pytest.runner import CallInfo +from _pytest.terminal import TerminalReporter +from pytest_embedded.plugin import multi_dut_argument, multi_dut_fixture +from pytest_embedded.utils import find_by_suffix + + +DEFAULT_SDKCONFIG = 'default' + + +################## +# Help Functions # +################## +def format_case_id(target: Optional[str], config: Optional[str], case: str) -> str: + return f'{target}.{config}.{case}' + + +def item_marker_names(item: Item) -> List[str]: + return [marker.name for marker in item.iter_markers()] + + +############ +# Fixtures # +############ +@pytest.fixture(scope='session', autouse=True) +def session_tempdir() -> str: + _tmpdir = os.path.join( + os.path.dirname(__file__), + 'pytest_embedded_log', + datetime.now().strftime('%Y-%m-%d_%H-%M-%S'), + ) + os.makedirs(_tmpdir, exist_ok=True) + return _tmpdir + + +@pytest.fixture +@multi_dut_argument +def config(request: FixtureRequest) -> str: + return getattr(request, 'param', None) or DEFAULT_SDKCONFIG + + +@pytest.fixture +def test_func_name(request: FixtureRequest) -> str: + return request.node.function.__name__ # type: ignore + + +@pytest.fixture +def test_case_name(request: FixtureRequest, target: str, config: str) -> str: + return format_case_id(target, config, request.node.originalname) + + +@pytest.fixture +@multi_dut_fixture +def build_dir(app_path: str, target: Optional[str], config: Optional[str]) -> str: + """ + Check local build dir with the following priority: + + 1. build__ + 2. build_ + 3. build_ + 4. build + + Args: + app_path: app path + target: target + config: config + + Returns: + valid build directory + """ + + check_dirs = [] + if target is not None and config is not None: + check_dirs.append(f'build_{target}_{config}') + if target is not None: + check_dirs.append(f'build_{target}') + if config is not None: + check_dirs.append(f'build_{config}') + check_dirs.append('build') + + for check_dir in check_dirs: + binary_path = os.path.join(app_path, check_dir) + if os.path.isdir(binary_path): + logging.info(f'find valid binary path: {binary_path}') + return check_dir + + logging.warning( + 'checking binary path: %s... missing... try another place', binary_path + ) + + recommend_place = check_dirs[0] + raise ValueError( + f'no build dir valid. Please build the binary via "idf.py -B {recommend_place} build" and run pytest again' + ) + + +@pytest.fixture(autouse=True) +@multi_dut_fixture +def junit_properties( + test_case_name: str, record_xml_attribute: Callable[[str, object], None] +) -> None: + """ + This fixture is autoused and will modify the junit report test case name to .. + """ + record_xml_attribute('name', test_case_name) + + +################## +# Hook functions # +################## +_idf_pytest_embedded_key = pytest.StashKey['IdfPytestEmbedded'] + + +def pytest_configure(config: Config) -> None: + # cli option "--target" + target = config.getoption('target') or '' + + help_commands = ['--help', '--fixtures', '--markers', '--version'] + for cmd in help_commands: + if cmd in config.invocation_params.args: + target = 'unneeded' + break + + assert target, "Must specify target by --target" + + config.stash[_idf_pytest_embedded_key] = IdfPytestEmbedded( + target=target, + ) + config.pluginmanager.register(config.stash[_idf_pytest_embedded_key]) + + +def pytest_unconfigure(config: Config) -> None: + _pytest_embedded = config.stash.get(_idf_pytest_embedded_key, None) + if _pytest_embedded: + del config.stash[_idf_pytest_embedded_key] + config.pluginmanager.unregister(_pytest_embedded) + + +class IdfPytestEmbedded: + def __init__( + self, + target: Optional[str] = None, + ): + # CLI options to filter the test cases + self.target = target + self._failed_cases: List[ + Tuple[str, bool, bool] + ] = [] # (test_case_name, is_known_failure_cases, is_xfail) + + @property + def failed_cases(self) -> List[str]: + return [ + case + for case, is_xfail in self._failed_cases + if not is_xfail + ] + + @property + def xfail_cases(self) -> List[str]: + return [case for case, is_xfail in self._failed_cases if is_xfail] + + @pytest.hookimpl(tryfirst=True) + def pytest_sessionstart(self, session: Session) -> None: + if self.target: + self.target = self.target.lower() + session.config.option.target = self.target + + @pytest.hookimpl(tryfirst=True) + def pytest_collection_modifyitems(self, items: List[Function]) -> None: + # sort by file path and callspec.config + # implement like this since this is a limitation of pytest, couldn't get fixture values while collecting + # https://github.com/pytest-dev/pytest/discussions/9689 + def _get_param_config(_item: Function) -> str: + if hasattr(_item, 'callspec'): + return _item.callspec.params.get('config', DEFAULT_SDKCONFIG) # type: ignore + return DEFAULT_SDKCONFIG + + items.sort(key=lambda x: (os.path.dirname(x.path), _get_param_config(x))) + + # set default timeout 10 minutes for each case + for item in items: + if 'timeout' not in item.keywords: + item.add_marker(pytest.mark.timeout(10 * 60)) + + # filter all the test cases with "--target" + if self.target: + items[:] = [ + item for item in items if self.target in item_marker_names(item) + ] + + def pytest_runtest_makereport( + self, item: Function, call: CallInfo[None] + ) -> Optional[TestReport]: + report = TestReport.from_item_and_call(item, call) + if report.outcome == 'failed': + test_case_name = item.funcargs.get('test_case_name', '') + is_xfail = report.keywords.get('xfail', False) + self._failed_cases.append((test_case_name, is_xfail)) + + return report + + @pytest.hookimpl(trylast=True) + def pytest_runtest_teardown(self, item: Function) -> None: + """ + Format the test case generated junit reports + """ + tempdir = item.funcargs.get('test_case_tempdir') + if not tempdir: + return + + junits = find_by_suffix('.xml', tempdir) + if not junits: + return + + target = item.funcargs['target'] + config = item.funcargs['config'] + for junit in junits: + xml = ET.parse(junit) + testcases = xml.findall('.//testcase') + for case in testcases: + case.attrib['name'] = format_case_id( + target, config, case.attrib['name'] + ) + if 'file' in case.attrib: + case.attrib['file'] = case.attrib['file'].replace( + '/IDF/', '' + ) # our unity test framework + xml.write(junit) + + def pytest_sessionfinish(self, session: Session, exitstatus: int) -> None: + if exitstatus != 0: + if exitstatus == ExitCode.NO_TESTS_COLLECTED: + session.exitstatus = 0 + + def pytest_terminal_summary(self, terminalreporter: TerminalReporter) -> None: + if self.xfail_cases: + terminalreporter.section('xfail cases', bold=True, yellow=True) + terminalreporter.line('\n'.join(self.xfail_cases)) + + if self.failed_cases: + terminalreporter.section('Failed cases', bold=True, red=True) + terminalreporter.line('\n'.join(self.failed_cases)) diff --git a/device_hal/device/esp32c2_devkit_m/esp_matter_device.cmake b/device_hal/device/esp32c2_devkit_m/esp_matter_device.cmake index 5348059d3..e5db709cc 100644 --- a/device_hal/device/esp32c2_devkit_m/esp_matter_device.cmake +++ b/device_hal/device/esp32c2_devkit_m/esp_matter_device.cmake @@ -5,7 +5,7 @@ endif() SET(device_type esp32c2_devkit_m) SET(led_type gpio) -SET(button_type hollow_button) +SET(button_type iot_button) SET(extra_components_dirs_append "$ENV{ESP_MATTER_DEVICE_PATH}/../../led_driver" - "$ENV{ESP_MATTER_DEVICE_PATH}/../../button_driver/button") + "$ENV{ESP_MATTER_DEVICE_PATH}/../../button_driver/iot_button") diff --git a/examples/.build-rules.yml b/examples/.build-rules.yml new file mode 100644 index 000000000..c4ba55d24 --- /dev/null +++ b/examples/.build-rules.yml @@ -0,0 +1,49 @@ +# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps + +examples/blemesh_bridge: + enable: + - if: IDF_TARGET in ["esp32", "esp32c3"] + temporary: true + reason: the other targets are not tested yet + +examples/zap_light: + enable: + - if: IDF_TARGET in ["esp32", "esp32h2"] + temporary: true + reason: the other targets are not tested yet + +examples/light_switch: + enable: + - if: IDF_TARGET in ["esp32", "esp32c6", "esp32c2", "esp32h2"] + temporary: true + reason: the other targets are not tested yet + +examples/light: + enable: + - if: IDF_TARGET in ["esp32", "esp32c3", "esp32c2", "esp32c6", "esp32h2"] + temporary: true + reason: the other targets are not tested yet + +examples/generic_switch: + enable: + - if: IDF_TARGET in ["esp32c2", "esp32c6", "esp32h2"] + temporary: true + reason: the other targets are not tested yet + +examples/esp-now_bridge_light: + enable: + - if: IDF_TARGET in ["esp32c3"] + temporary: true + reason: the other targets are not tested yet + +examples/controller: + enable: + - if: IDF_TARGET in ["esp32"] + temporary: true + reason: the other targets are not tested yet + +examples/zigbee_bridge: + enable: + - if: IDF_TARGET in ["esp32"] + temporary: true + reason: the other targets are not tested yet diff --git a/examples/pytest_esp_matter_light.py b/examples/pytest_esp_matter_light.py new file mode 100644 index 000000000..e628f7d80 --- /dev/null +++ b/examples/pytest_esp_matter_light.py @@ -0,0 +1,237 @@ +# SPDX-License-Identifier: CC0-1.0 + +import pathlib +import pytest +import time +import re +import pexpect +import subprocess +import netifaces +from typing import Tuple +from pytest_embedded import Dut + +CURRENT_DIR_LIGHT = str(pathlib.Path(__file__).parent)+'/light' +CHIP_TOOL_DIR = str(pathlib.Path(__file__).parent)+'/../connectedhomeip/connectedhomeip/examples/chip-tool' +OT_BR_EXAMPLE_PATH = str(pathlib.Path(__file__).parent)+'/../esp-thread-br/examples/basic_thread_border_router' +pytest_build_dir = CURRENT_DIR_LIGHT +pytest_matter_thread_dir = CURRENT_DIR_LIGHT+'|'+OT_BR_EXAMPLE_PATH + + +@pytest.mark.esp32c3 +@pytest.mark.esp_matter_dut +@pytest.mark.parametrize( + ' count, app_path, target, erase_all', [ + ( 1, pytest_build_dir, 'esp32c3', 'y'), + ], + indirect=True, +) + +# Matter over wifi commissioning +def test_matter_commissioning_c3(dut:Dut) -> None: + light = dut + # BLE start advertising + light.expect(r'chip\[DL\]\: Configuring CHIPoBLE advertising', timeout=20) + # Start commissioning + time.sleep(5) + command = CHIP_TOOL_DIR+'/out/chip-tool pairing ble-wifi 1 ChipTEH2 chiptest123 20202021 3840' + out_str = subprocess.getoutput(command) + print(out_str) + result = re.findall(r'Run command failure', str(out_str)) + if len(result) != 0: + assert False + # Use toggle command to turn-off the light + time.sleep(3) + command = CHIP_TOOL_DIR+'/out/chip-tool onoff toggle 1 1' + out_str = subprocess.getoutput(command) + print(out_str) + result = re.findall(r'Run command failure', str(out_str)) + if len(result) != 0: + assert False + # Use toggle command to turn-on the light + time.sleep(5) + command = CHIP_TOOL_DIR+'/out/chip-tool onoff toggle 1 1' + out_str = subprocess.getoutput(command) + print(out_str) + result = re.findall(r'Run command failure', str(out_str)) + if len(result) != 0: + assert False + +@pytest.mark.esp32c6 +@pytest.mark.esp_matter_dut +@pytest.mark.parametrize( + ' count, app_path, target, erase_all', [ + ( 1, pytest_build_dir, 'esp32c6', 'y'), + ], + indirect=True, +) + +# Matter over wifi commissioning +def test_matter_commissioning_c6(dut:Dut) -> None: + light = dut + # BLE start advertising + light.expect(r'chip\[DL\]\: Configuring CHIPoBLE advertising', timeout=20) + # Start commissioning + time.sleep(5) + command = CHIP_TOOL_DIR+'/out/chip-tool pairing ble-wifi 1 ChipTEH2 chiptest123 20202021 3840' + out_str = subprocess.getoutput(command) + print(out_str) + result = re.findall(r'Run command failure', str(out_str)) + if len(result) != 0: + assert False + # Use toggle command to turn-off the light + time.sleep(3) + command = CHIP_TOOL_DIR+'/out/chip-tool onoff toggle 1 1' + out_str = subprocess.getoutput(command) + print(out_str) + result = re.findall(r'Run command failure', str(out_str)) + if len(result) != 0: + assert False + # Use toggle command to turn-on the light + time.sleep(5) + command = CHIP_TOOL_DIR+'/out/chip-tool onoff toggle 1 1' + out_str = subprocess.getoutput(command) + print(out_str) + result = re.findall(r'Run command failure', str(out_str)) + if len(result) != 0: + assert False + + +# get the host interface name +def get_host_interface_name() -> str: + interfaces = netifaces.interfaces() + interface_name = [s for s in interfaces if 'wl' in s][0] + return str(interface_name) + + +# reset host interface +def reset_host_interface() -> None: + interface_name = get_host_interface_name() + flag = False + try: + command = 'ifconfig ' + interface_name + ' down' + subprocess.call(command, shell=True, timeout=5) + time.sleep(1) + command = 'ifconfig ' + interface_name + ' up' + subprocess.call(command, shell=True, timeout=10) + time.sleep(1) + flag = True + finally: + time.sleep(1) + assert flag + + +# set interface sysctl options +def set_interface_sysctl_options() -> None: + interface_name = get_host_interface_name() + flag = False + try: + command = 'sysctl -w net/ipv6/conf/' + interface_name + '/accept_ra=2' + subprocess.call(command, shell=True, timeout=5) + time.sleep(1) + command = 'sysctl -w net/ipv6/conf/' + interface_name + '/accept_ra_rt_info_max_plen=128' + subprocess.call(command, shell=True, timeout=5) + time.sleep(1) + command = 'sysctl -w net.ipv6.conf.all.forwarding=1' + subprocess.call(command, shell=True, timeout=5) + time.sleep(1) + flag = True + finally: + time.sleep(2) + assert flag + + +# initialize interface ipv6 address +def init_interface_ipv6_address() -> None: + interface_name = get_host_interface_name() + flag = False + try: + command = 'ip -6 route | grep ' + interface_name + " | grep ra | awk {'print $1'} | xargs -I {} ip -6 route del {}" + subprocess.call(command, shell=True, timeout=5) + time.sleep(0.5) + subprocess.call(command, shell=True, timeout=5) + time.sleep(1) + command = 'ip -6 address show dev ' + interface_name + \ + " scope global | grep 'inet6' | awk {'print $2'} | xargs -I {} ip -6 addr del {} dev " + interface_name + subprocess.call(command, shell=True, timeout=5) + time.sleep(1) + flag = True + finally: + time.sleep(1) + assert flag + + +def fixture_Init_interface() -> bool: + print('Init interface') + init_interface_ipv6_address() + reset_host_interface() + time.sleep(30) + set_interface_sysctl_options() + return True + + +@pytest.mark.esp32h2 +@pytest.mark.esp32s3 +@pytest.mark.esp_matter_dut +@pytest.mark.parametrize( + 'count, app_path, target, erase_all', [ + ( 2, pytest_matter_thread_dir, 'esp32h2|esp32s3', 'y|n'), + ], + indirect=True, +) + +# Matter over thread commissioning +def test_matter_commissioning_h2(dut:Tuple[Dut, Dut]) -> None: + ot_br = dut[1] + light = dut[0] + # For matter over thread commissioning need to reset host interface + fixture_Init_interface() + # BLE start advertising + light.expect(r'chip\[DL\]\: Configuring CHIPoBLE advertising', timeout=20) + # flash ot_br + ot_br.expect('OpenThread attached to netif', timeout=30) + time.sleep(2) + ot_br.write('factoryreset') + ot_br.expect('OpenThread attached to netif', timeout=30) + time.sleep(2) + ot_br.write('log level 3') + ot_br.expect('Done', timeout=5) + time.sleep(2) + # wifi connect -s ChipTEH2 -p chiptest123 + ot_br.write('wifi connect -s ChipTEH2 -p chiptest123') + ot_br.expect('wifi sta is connected successfully', timeout=5) + time.sleep(2) + # start an ot network + ot_br.write('ifconfig up') + ot_br.expect('netif up', timeout=5) + time.sleep(2) + ot_br.write('thread start') + ot_br.expect('Role detached -> leader', timeout=20) + time.sleep(2) + # get dataset + ot_br.write('dataset active -x') + dataset=ot_br.expect(r'\n(\w{202}\r)', timeout=5)[1].decode() + print(dataset) + # Start commissioning + time.sleep(2) + command = CHIP_TOOL_DIR+"/out/chip-tool pairing ble-thread 1 hex:{} ".format(dataset.strip())+"20202021 3840" + out_str = subprocess.getoutput(command) + print(out_str) + result = re.findall(r'Run command failure', str(out_str)) + if len(result) != 0: + assert False + # Use toggle command to turn-off the light + time.sleep(2) + command = CHIP_TOOL_DIR+'/out/chip-tool onoff toggle 1 1' + out_str = subprocess.getoutput(command) + print(out_str) + result = re.findall(r'Run command failure', str(out_str)) + if len(result) != 0: + assert False + # Use toggle command to turn-on the light + time.sleep(2) + command = CHIP_TOOL_DIR+'/out/chip-tool onoff toggle 1 1' + out_str = subprocess.getoutput(command) + print(out_str) + result = re.findall(r'Run command failure', str(out_str)) + if len(result) != 0: + assert False diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 000000000..bca0a4d21 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,39 @@ +[pytest] + +python_files = pytest_*.py + +# ignore PytestExperimentalApiWarning for record_xml_attribute +# set traceback to "short" to prevent the overwhelming tracebacks +addopts = + -s + --embedded-services esp,idf + --tb short + --strict-markers + --skip-check-coredump y + --logfile-extension ".txt" + +# ignore DeprecationWarning +filterwarnings = + ignore::_pytest.warning_types.PytestExperimentalApiWarning + +markers = + # target markers + esp32c3: support esp32c3 target + esp32c6: support esp32c6 target + esp32h2: support esp32h2 target + esp32s3: support esp32s3 target + # env markers + esp_matter_dut: esp matter runner which have single dut + +# log related +log_cli = True +log_cli_level = INFO +log_cli_format = %(asctime)s %(levelname)s %(message)s +log_cli_date_format = %Y-%m-%d %H:%M:%S + +# junit related +junit_family = xunit1 + +## log all to `system-out` when case fail +junit_logging = stdout +junit_log_passing_tests = False diff --git a/tools/ci/build_apps.py b/tools/ci/build_apps.py new file mode 100644 index 000000000..b3071817a --- /dev/null +++ b/tools/ci/build_apps.py @@ -0,0 +1,179 @@ +# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD + +# SPDX-License-Identifier: CC0-1.0 + +# This file is used in CI generate binary files for different kinds of apps + +import argparse +import sys +from pathlib import Path +from typing import List + +from idf_build_apps import LOGGER, App, build_apps, find_apps, setup_logging + +# from idf_ci_utils import IDF_PATH, get_pytest_app_paths, get_pytest_cases, get_ttfw_app_paths + +PROJECT_ROOT = Path(__file__).parent.parent.parent.absolute() +DEF_APP_PATH = PROJECT_ROOT / 'examples' +APPS_BUILD_PER_JOB = 30 +PYTEST_C6_APPS = [ + {"target": "esp32c6", "name": "light"}, +] +MAINFEST_FILES = [ + str(PROJECT_ROOT / 'examples' / '.build-rules.yml'), +] + +PYTEST_H2_APPS = [ + {"target": "esp32h2", "name": "light"}, +] +MAINFEST_FILES = [ + str(PROJECT_ROOT / 'examples' / '.build-rules.yml'), +] + +PYTEST_C3_APPS = [ + {"target": "esp32c3", "name": "light"}, +] +MAINFEST_FILES = [ + str(PROJECT_ROOT / 'examples' / '.build-rules.yml'), +] + +def _is_c6_pytest_app(app: App) -> bool: + print(app.name , app.target) + for pytest_app in PYTEST_C6_APPS: + print(pytest_app["name"] , pytest_app["target"]) + if app.name == pytest_app["name"] and app.target == pytest_app["target"]: + return True + return False + +def _is_h2_pytest_app(app: App) -> bool: + for pytest_app in PYTEST_H2_APPS: + if app.name == pytest_app["name"] and app.target == pytest_app["target"]: + return True + return False + +def _is_c3_pytest_app(app: App) -> bool: + for pytest_app in PYTEST_C3_APPS: + if app.name == pytest_app["name"] and app.target == pytest_app["target"]: + return True + return False + +def get_cmake_apps( + paths: List[str], + target: str, + config_rules_str: List[str], +) -> List[App]: + apps = find_apps( + paths, + recursive=True, + target=target, + build_dir='build_@t_@w', + config_rules_str=config_rules_str, + build_log_path='build_log.txt', + size_json_path='size.json', + check_warnings=False, + preserve=True, + manifest_files=MAINFEST_FILES, + ) + return apps + + +def main(args: argparse.Namespace) -> None: + apps = get_cmake_apps(args.paths, args.target, args.config) + + # no_pytest and only_pytest can not be both True + assert not (args.no_pytest and args.pytest_c6 and args.pytest_h2 and args.pytest_c3) + if args.no_pytest: + apps_for_build = [app for app in apps if not (_is_c6_pytest_app(app) or _is_h2_pytest_app(app))] + elif args.pytest_c6: + apps_for_build = [app for app in apps if _is_c6_pytest_app(app)] + elif args.pytest_h2: + apps_for_build = [app for app in apps if _is_h2_pytest_app(app)] + elif args.pytest_c3: + apps_for_build = [app for app in apps if _is_c3_pytest_app(app)] + else: + apps_for_build = apps[:] + + LOGGER.info('Found %d apps after filtering', len(apps_for_build)) + LOGGER.info( + 'Suggest setting the parallel count to %d for this build job', + len(apps_for_build) // APPS_BUILD_PER_JOB + 1, + ) + + ret_code = build_apps( + apps_for_build, + parallel_count=args.parallel_count, + parallel_index=args.parallel_index, + dry_run=False, + collect_size_info=args.collect_size_info, + # build_verbose=0, + keep_going=True, + ignore_warning_strs=[r".*"], + copy_sdkconfig=True, + ) + + sys.exit(ret_code) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Build all the apps for different test types. Will auto remove those non-test apps binaries', + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument('paths', nargs='*', help='Paths to the apps to build.') + parser.add_argument( + '-t', + '--target', + default='all', + help='Build apps for given target. could pass "all" to get apps for all targets', + ) + parser.add_argument( + '--config', + default=['sdkconfig.ci=default', 'sdkconfig.ci.*=', '=default'], + action='append', + help='Adds configurations (sdkconfig file names) to build. This can either be ' + 'FILENAME[=NAME] or FILEPATTERN. FILENAME is the name of the sdkconfig file, ' + 'relative to the project directory, to be used. Optional NAME can be specified, ' + 'which can be used as a name of this configuration. FILEPATTERN is the name of ' + 'the sdkconfig file, relative to the project directory, with at most one wildcard. ' + 'The part captured by the wildcard is used as the name of the configuration.', + ) + parser.add_argument( + '--parallel-count', default=1, type=int, help='Number of parallel build jobs.' + ) + parser.add_argument( + '--parallel-index', + default=1, + type=int, + help='Index (1-based) of the job, out of the number specified by --parallel-count.', + ) + parser.add_argument( + '--no_pytest', + action="store_true", + help='Exclude pytest apps, definded in PYTEST_H2_APPS and PYTEST_C6_APPS', + ) + parser.add_argument( + '--pytest_c6', + action="store_true", + help='Only build pytest apps, definded in PYTEST_C6_APPS', + ) + parser.add_argument( + '--pytest_h2', + action="store_true", + help='Only build pytest apps, definded in PYTEST_H2_APPS', + ) + parser.add_argument( + '--pytest_c3', + action="store_true", + help='Only build pytest apps, definded in PYTEST_C3_APPS', + ) + parser.add_argument( + '--collect-size-info', + type=argparse.FileType('w'), + help='If specified, the test case name and size info json will be written to this file', + ) + + arguments = parser.parse_args() + if not arguments.paths: + arguments.paths = [DEF_APP_PATH] + setup_logging(verbose=1) # Info + main(arguments) \ No newline at end of file diff --git a/tools/ci/requirements-build.txt b/tools/ci/requirements-build.txt new file mode 100644 index 000000000..da4c75951 --- /dev/null +++ b/tools/ci/requirements-build.txt @@ -0,0 +1 @@ +idf_build_apps diff --git a/tools/ci/requirements-pytest.txt b/tools/ci/requirements-pytest.txt new file mode 100644 index 000000000..c90d19f6f --- /dev/null +++ b/tools/ci/requirements-pytest.txt @@ -0,0 +1,6 @@ +pytest-embedded-serial-esp~=1.0 +pytest-embedded-idf~=1.0 +pytest-embedded-qemu~=1.0 +pytest-timeout +netifaces +esptool>=4.5