diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a49e9a874..05600d30f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -525,9 +525,19 @@ pytest_esp32c6_esp_matter_dut: - cd ${ESP_MATTER_PATH} - rm -rf connectedhomeip/connectedhomeip - ln -s ${CHIP_SUBMODULE_PATH} connectedhomeip/connectedhomeip + - cd connectedhomeip/connectedhomeip + - source out/py-env/bin/activate + - cd ${ESP_MATTER_PATH} + - cp ${ESP_MATTER_PATH}/tools/ci/extended_color_light_wifi_pics_code.txt $ESP_MATTER_PATH/connectedhomeip/connectedhomeip/src/python_testing/ - pip install -r tools/ci/requirements-pytest.txt - - pytest examples/ --target esp32c6 -m esp_matter_dut --junitxml=XUNIT_RESULT.xml + - python3 -c "import os; os.environ['TEST_CHUNK'] = '$TEST_CHUNK'" + - pytest examples/ --target esp32c6 -m esp_matter_dut --junitxml=XUNIT_RESULT.xml --certification-json=$ESP_MATTER_PATH/tools/ci/certification_test_commands.json tags: ["esp32c6", "esp_matter_dut"] + parallel: + matrix: + - TEST_CHUNK: + - "1" + - "2" pytest_esp32c2_esp_matter_dut: stage: target_test diff --git a/conftest.py b/conftest.py index 5f2f02890..b4681f518 100644 --- a/conftest.py +++ b/conftest.py @@ -120,6 +120,17 @@ def build_dir(app_path: str, target: Optional[str], config: Optional[str]) -> st f'no build dir valid. Please build the binary via "idf.py -B {recommend_place} build" and run pytest again' ) +def pytest_addoption(parser): + parser.addoption( + "--certification-json", + action="store", + default="certification_test_commands.json", + help="Path to the certification test commands JSON file", + ) + +@pytest.fixture(scope="session") +def certification_tests(request): + return request.config.getoption("--certification-json") @pytest.fixture(autouse=True) @multi_dut_fixture @@ -207,7 +218,7 @@ class IdfPytestEmbedded: # set default timeout 10 minutes for each case for item in items: if 'timeout' not in item.keywords: - item.add_marker(pytest.mark.timeout(10 * 60)) + item.add_marker(pytest.mark.timeout(40 * 60)) # filter all the test cases with "--target" if self.target: diff --git a/examples/pytest_esp_matter_light.py b/examples/pytest_esp_matter_light.py index 6c4f023b6..9d39b3e55 100644 --- a/examples/pytest_esp_matter_light.py +++ b/examples/pytest_esp_matter_light.py @@ -11,7 +11,11 @@ from typing import Tuple from pytest_embedded import Dut import os import yaml +import sys +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../tools/ci'))) +from pytest_cert_helper import * +from gitlab_api import GitLabAPI CURRENT_DIR_LIGHT = str(pathlib.Path(__file__).parent)+'/light' CHIP_TOOL_EXE = str(pathlib.Path(__file__).parent)+ '/../connectedhomeip/connectedhomeip/out/host/chip-tool' @@ -20,6 +24,9 @@ OT_DATASET_HEXSTR = '0e08000000000001000035060004001fffe00708fdb824be22185de50c0 pytest_build_dir = CURRENT_DIR_LIGHT pytest_matter_thread_dir = CURRENT_DIR_LIGHT+'|'+OT_BR_EXAMPLE_PATH +gitlab_api = GitLabAPI() +PYTEST_SSID = gitlab_api.ci_gitlab_pytest_ssid +PYTEST_PASSPHRASE = gitlab_api.ci_gitlab_pytest_passphrase @pytest.mark.esp32c3 @pytest.mark.esp_matter_dut @@ -37,7 +44,7 @@ def test_matter_commissioning_c3(dut:Dut) -> None: light.expect(r'chip\[DL\]\: Configuring CHIPoBLE advertising', timeout=20) # Start commissioning time.sleep(5) - command = CHIP_TOOL_EXE + ' pairing ble-wifi 1 ChipTEH2 chiptest123 20202021 3840' + command = CHIP_TOOL_EXE + f" pairing ble-wifi 1 {PYTEST_SSID} {PYTEST_PASSPHRASE} 20202021 3840" out_str = subprocess.getoutput(command) print(out_str) result = re.findall(r'Run command failure', str(out_str)) @@ -76,7 +83,7 @@ def test_matter_commissioning_c2(dut:Dut) -> None: light.expect(r'chip\[DL\]\: Configuring CHIPoBLE advertising', timeout=20) # Start commissioning time.sleep(5) - command = CHIP_TOOL_EXE + ' pairing ble-wifi 1 ChipTEH2 chiptest123 20202021 3840' + command = CHIP_TOOL_EXE + f" pairing ble-wifi 1 {PYTEST_SSID} {PYTEST_PASSPHRASE} 20202021 3840" out_str = subprocess.getoutput(command) print(out_str) result = re.findall(r'Run command failure', str(out_str)) @@ -109,13 +116,13 @@ def test_matter_commissioning_c2(dut:Dut) -> None: ) # Matter over wifi commissioning -def test_matter_commissioning_c6(dut:Dut) -> None: +def test_matter_commissioning_c6(dut:Dut, certification_tests: str) -> None: light = dut # BLE start advertising light.expect(r'chip\[DL\]\: Configuring CHIPoBLE advertising', timeout=20) # Start commissioning time.sleep(5) - command = CHIP_TOOL_EXE + ' pairing ble-wifi 1 ChipTEH2 chiptest123 20202021 3840' + command = CHIP_TOOL_EXE + f" pairing ble-wifi 1 {PYTEST_SSID} {PYTEST_PASSPHRASE} 20202021 3840" out_str = subprocess.getoutput(command) print(out_str) result = re.findall(r'Run command failure', str(out_str)) @@ -134,6 +141,11 @@ def test_matter_commissioning_c6(dut:Dut) -> None: command = CHIP_TOOL_EXE + ' onoff toggle 1 1' out_str = subprocess.getoutput(command) print(out_str) + + light.write('matter esp factoryreset') + time.sleep(10) + run_python_certification_tests(light, certification_tests) + result = re.findall(r'Run command failure', str(out_str)) if len(result) != 0: assert False @@ -244,7 +256,7 @@ def test_matter_commissioning_h2(dut:Tuple[Dut, Dut]) -> None: ot_br.expect(r'chip\[DL\]\: Configuring CHIPoBLE advertising', timeout=20) # Start commissioning OTBR time.sleep(2) - command = CHIP_TOOL_EXE + ' pairing ble-wifi 1 ChipTEH2 chiptest123 20202021 3584' + command = CHIP_TOOL_EXE + f" pairing ble-wifi 1 {PYTEST_SSID} {PYTEST_PASSPHRASE} 20202021 3584" out_str = subprocess.getoutput(command) print(out_str) result = re.findall(r'Run command failure', str(out_str)) diff --git a/tools/ci/certification_test_commands.json b/tools/ci/certification_test_commands.json new file mode 100644 index 000000000..632265268 --- /dev/null +++ b/tools/ci/certification_test_commands.json @@ -0,0 +1,138 @@ +{ + "common_args": "-m ble-wifi --wifi-ssid {WIFI_SSID} --wifi-passphrase {WIFI_PASSPHRASE} --discriminator 3840 --passcode 20202021 --int-arg use_pase_only:0", + "test_cases": { + "TC_LVL_2_3": { + "script": "TC_LVL_2_3.py", + "args": "--endpoint 1 --PICS extended_color_light_wifi_pics_code.txt" + }, + "TC_ACE_1_2": { + "script": "TC_ACE_1_2.py", + "args": "--timeout 300" + }, + "TC_ACE_1_3": { + "script": "TC_ACE_1_3.py", + "args": "" + }, + "TC_ACE_1_5": { + "script": "TC_ACE_1_5.py", + "args": "--timeout 300" + }, + "TC_ACE_2_1": { + "script": "TC_AccessChecker.py", + "test_case": "test_TC_ACE_2_1", + "args": "--timeout 300" + }, + "TC_ACE_2_2": { + "script": "TC_AccessChecker.py", + "test_case": "test_TC_ACE_2_2", + "args": "--timeout 300" + }, + "TC_CGEN_2_4": { + "script": "TC_CGEN_2_4.py", + "args": "--timeout 300" + }, + "TC_CNET_1_4": { + "script": "TC_CNET_1_4.py", + "args": "--timeout 300" + }, + "TC_CNET_4_4": { + "script": "TC_CNET_4_4.py", + "args": "--timeout 300" + }, + "TC_DESC_2_2": { + "script": "TC_DeviceBasicComposition.py", + "test_case": "test_TC_DESC_2_2", + "args": "--timeout 300" + }, + "TC_DGGEN_2_4": { + "script": "TC_DGGEN_2_4.py", + "args": "--timeout 300" + }, + "TC_DGGEN_3_2": { + "script": "TC_DGGEN_3_2.py", + "args": "" + }, + "TC_DT_1_1": { + "script": "TC_DeviceBasicComposition.py", + "test_case": "test_TC_DT_1_1", + "args": "--timeout 300" + }, + "TC_OPCREDS_3_1": { + "script": "TC_OPCREDS_3_1.py", + "args": "--timeout 300" + }, + "TC_OPCREDS_3_2": { + "script": "TC_OPCREDS_3_2.py", + "args": "--timeout 300" + }, + "TC_SM_1_1": { + "script": "TC_DeviceBasicComposition.py", + "test_case": "test_TC_SM_1_1", + "args": "--timeout 300" + }, + "TC_SM_1_2": { + "script": "TC_DeviceBasicComposition.py", + "test_case": "test_TC_SM_1_2", + "args": "--timeout 300" + }, + "TC_ACE_1_4": { + "script": "TC_ACE_1_4.py", + "args": "--int-arg PIXIT.ACE.APPENDPOINT:1 PIXIT.ACE.APPDEVTYPEID:269 use_pase_only:0 --string-arg PIXIT.ACE.APPCLUSTER:OnOff PIXIT.ACE.APPATTRIBUTE:OnOff" + }, + "TC_IDM_4_2": { + "script": "TC_IDM_4_2.py", + "args": "" + }, + "TC_IDM_12_1": { + "script": "TC_DeviceBasicComposition.py", + "test_case": "test_TC_IDM_12_1", + "args": "" + }, + "TC_IDM_11_1": { + "script": "TC_DeviceBasicComposition.py", + "test_case": "test_TC_IDM_11_1", + "args": "" + }, + "TC_IDM_10_5": { + "script": "TC_DeviceConformance.py", + "test_case": "test_TC_IDM_10_5", + "args": "" + }, + "TC_IDM_10_3": { + "script": "TC_DeviceConformance.py", + "test_case": "test_TC_IDM_10_3", + "args": "" + }, + "TC_IDM_10_2": { + "script": "TC_DeviceConformance.py", + "test_case": "test_TC_IDM_10_2", + "args": "--bool-arg allow_provisional:true" + }, + "TC_IDM_10_1": { + "script": "TC_DeviceBasicComposition.py", + "test_case": "test_TC_IDM_10_1", + "args": "" + }, + "TC_IDM_1_4": { + "script": "TC_IDM_1_4.py", + "args": "--hex-arg PIXIT.DGGEN.TEST_EVENT_TRIGGER_KEY:000102030405060708090a0b0c0d0e0f" + }, + "TC_IDM_1_2": { + "script": "TC_IDM_1_2.py", + "args": "--PICS extended_color_light_wifi_pics_code.txt" + }, + "TC_RR_1_1": { + "script": "TC_RR_1_1.py", + "args": "--timeout 100000" + }, + "TC_SC_3_6": { + "script": "TC_SC_3_6.py", + "args": "--PICS extended_color_light_wifi_pics_code.txt" + }, + "TC_SC_3_6_post": { + "script": "TC_SC_3_6.py", + "args": "--PICS extended_color_light_wifi_pics_code.txt --bool-arg: post_cert_test:true" + } + } +} + diff --git a/tools/ci/extended_color_light_wifi_pics_code.txt b/tools/ci/extended_color_light_wifi_pics_code.txt new file mode 100644 index 000000000..1cb421fa1 --- /dev/null +++ b/tools/ci/extended_color_light_wifi_pics_code.txt @@ -0,0 +1,594 @@ +ACL.S=1 +ACL.C=0 +ACL.S.A0000=1 +ACL.S.A0001=0 +ACL.S.A0002=1 +ACL.S.A0003=1 +ACL.S.A0004=1 +ACL.S.A0005=0 +ACL.S.A0006=0 +ACL.S.E00=1 +ACL.S.E01=0 +ACL.S.E02=0 +ACL.S.C01.Tx=0 +ACL.S.C00.Rsp=0 +ACL.S.F00=0 +ACL.S.F01=0 +ACL.S=1 +ACL.C=0 +APPDEVICE.S=1 +ACL.S.A0000=1 +CADMIN.S=1 +CADMIN.C=0 +CADMIN.S.A0000=1 +CADMIN.S.A0001=1 +CADMIN.S.A0002=1 +CADMIN.S.C00.Rsp=1 +CADMIN.S.C01.Rsp=0 +CADMIN.S.C02.Rsp=1 +CADMIN.S.F00=0 +CADMIN.C.A0000=0 +CADMIN.C.A0001=0 +CADMIN.C.A0002=0 +CADMIN.C.C00.Tx=0 +CADMIN.C.C01.Tx=0 +CADMIN.C.C02.Tx=0 +CADMIN.M.UserInterfaceDisplay=0 +CADMIN.M.AudioInterface=0 +MCORE.DT_SW_COMP=0 +MCORE.COM.BLE=1 +MCORE.COM.WIFI=1 +MCORE.COM.ETH=0 +MCORE.COM.THR=0 +MCORE.COM.WIRELESS=1 +MCORE.ROLE.COMMISSIONER=0 +MCORE.ROLE.COMMISSIONEE=1 +MCORE.ROLE.CONTROLLER=0 +MCORE.DLOG.S.UTCTIMESTAMP=0 +MCORE.DLOG.S.TIMESINCEBOOT=0 +MCORE.DD.CHIP_DEV=0 +MCORE.DD.DEV_LOCK=0 +MCORE.DD.DEV_BARRIER=0 +MCORE.DD.IE=0 +MCORE.DD.QR=1 +MCORE.DD.MANUAL_PC=1 +MCORE.DD.NFC=0 +MCORE.DD.UI=0 +MCORE.DD.COMM_DISCOVERY=0 +MCORE.DD.CONCATENATED_QR_CODE=1 +MCORE.DD.CTRL_CONCATENATED_QR_CODE_1=0 +MCORE.DD.CTRL_CONCATENATED_QR_CODE_1_INORDER=0 +MCORE.DD.CTRL_CONCATENATED_QR_CODE_2=0 +MCORE.DD.DISCOVERY_BLE=0 +MCORE.DD.DISCOVERY_PAF=0 +MCORE.DD.DISCOVERY_IP=0 +MCORE.DD.DISCOVERY_SOFTAP=0 +MCORE.DD.STANDARD_COMM_FLOW=1 +MCORE.DD.NON_CONCURRENT_CONNECTION=0 +MCORE.DD.USER_INTENT_COMM_FLOW=0 +MCORE.DD.CUSTOM_COMM_FLOW=0 +MCORE.DD.MANUAL_PC_COMMISSIONING=0 +MCORE.DD.11_MANUAL_PC=0 +MCORE.DD.21_MANUAL_PC=0 +MCORE.DD.PHYSICAL_TAMPERING=0 +MCORE.DD.SCAN_NFC=0 +MCORE.DD.QR_COMMISSIONING=0 +MCORE.DD.SCAN_QR_CODE=0 +MCORE.DD.EXTENDED_DISCOVERY=0 +MCORE.DD.COMMISSIONING_SUBTYPE_V=1 +MCORE.DD.COMMISSIONING_SUBTYPE_T=0 +MCORE.DD.TXT_KEY_VP=1 +MCORE.DD.TXT_KEY_DT=0 +MCORE.DD.TXT_KEY_DN=0 +MCORE.DD.TXT_KEY_RI=0 +MCORE.DD.TXT_KEY_PH=1 +MCORE.DD.TXT_KEY_PI=1 +MCORE.COM.PAF=0 +MCORE.DD.ESF_TC_COMMISSIONER=0 +MCORE.SC.VENDOR_SUBTYPE=1 +MCORE.SC.DEVTYPE_SUBTYPE=0 +MCORE.SC.VP_KEY=1 +MCORE.SC.DT_KEY=0 +MCORE.SC.DN_KEY=0 +MCORE.SC.RI_KEY=0 +MCORE.SC.PH_KEY=1 +MCORE.SC.PI_KEY=1 +MCORE.SC.SII_OP_DISCOVERY_KEY=1 +MCORE.SC.SAI_OP_DISCOVERY_KEY=1 +MCORE.SC.SAT_OP_DISCOVERY_KEY=1 +MCORE.SC.T_KEY=1 +MCORE.SC.SII_COMM_DISCOVERY_KEY=1 +MCORE.SC.SAI_COMM_DISCOVERY_KEY=1 +MCORE.SC.EXTENDED_DISCOVERY=0 +MCORE.SC.SIT_ICD=0 +MCORE.SC.TCP=0 +MCORE.IDM.S=1 +MCORE.IDM.C=0 +MCORE.IDM.C.InvokeRequest=0 +MCORE.IDM.C.ReadRequest=0 +MCORE.IDM.C.WriteRequest=0 +MCORE.IDM.C.SubscribeRequest=0 +MCORE.IDM.C.InvokeRequest.BatchCommands=0 +MCORE.IDM.C.ReadRequest.Attribute.DataType_Bool=0 +MCORE.IDM.C.ReadRequest.Attribute.DataType_String=0 +MCORE.IDM.C.ReadRequest.Attribute.DataType_UnsignedInteger=0 +MCORE.IDM.C.ReadRequest.Attribute.DataType_SignedInteger=0 +MCORE.IDM.C.ReadRequest.Attribute.DataType_Struct=0 +MCORE.IDM.C.ReadRequest.Attribute.DataType_FloatingPoint=0 +MCORE.IDM.C.ReadRequest.Attribute.DataType_List=0 +MCORE.IDM.C.ReadRequest.Attribute.DataType_OctetString=0 +MCORE.IDM.C.ReadRequest.Attribute.DataType_Enum=0 +MCORE.IDM.C.ReadRequest.Attribute.DataType_Bitmap=0 +MCORE.IDM.C.WriteRequest.Attribute.DataType_Bool=0 +MCORE.IDM.C.WriteRequest.Attribute.DataType_String=0 +MCORE.IDM.C.WriteRequest.Attribute.DataType_UnsignedInteger=0 +MCORE.IDM.C.WriteRequest.Attribute.DataType_SignedInteger=0 +MCORE.IDM.C.WriteRequest.Attribute.DataType_Struct=0 +MCORE.IDM.C.WriteRequest.Attribute.DataType_FloatingPoint=0 +MCORE.IDM.C.WriteRequest.Attribute.DataType_List=0 +MCORE.IDM.C.WriteRequest.Attribute.DataType_OctetString=0 +MCORE.IDM.C.WriteRequest.Attribute.DataType_Enum=0 +MCORE.IDM.C.WriteRequest.Attribute.DataType_Bitmap=0 +MCORE.IDM.S.Attribute_W.DataType_Bool=0 +MCORE.IDM.S.Attribute_W.DataType_String=1 +MCORE.IDM.S.Attribute_W.DataType_UnsignedInteger=1 +MCORE.IDM.S.Attribute_W.DataType_SignedInteger=0 +MCORE.IDM.S.Attribute_W.DataType_Struct=0 +MCORE.IDM.S.Attribute_W.DataType_FloatingPoint=0 +MCORE.IDM.S.Attribute_W.DataType_List=1 +MCORE.IDM.S.Attribute_W.DataType_OctetString=0 +MCORE.IDM.S.Attribute_W.DataType_Enum=1 +MCORE.IDM.S.Attribute_W.DataType_Bitmap=1 +MCORE.IDM.C.SubscribeRequest.Attribute.DataType_Bool=0 +MCORE.IDM.C.SubscribeRequest.Attribute.DataType_String=0 +MCORE.IDM.C.SubscribeRequest.Attribute.DataType_UnsignedInteger=0 +MCORE.IDM.C.SubscribeRequest.Attribute.DataType_Integer=0 +MCORE.IDM.C.SubscribeRequest.Attribute.DataType_FloatingPoint=0 +MCORE.IDM.C.SubscribeRequest.Attribute.DataType_List=0 +MCORE.IDM.S.LargeData=0 +MCORE.IDM.C.SubscribeEvent=0 +MCORE.IDM.C.ReadEvent=0 +MCORE.IDM.C.SubscribeRequest.MultipleAttributes=0 +MCORE.IDM.S.PersistentSubscription=0 +MCORE.BRIDGE=0 +MCORE.BRIDGE.BatInfo=0 +MCORE.BRIDGE.OtherControl=0 +MCORE.BRIDGE.AllowDeviceRename=0 +MCORE.BRIDGECLIENT=0 +MCORE.DEVLIST.UseDevices=0 +MCORE.DEVLIST.UseDeviceName=0 +MCORE.DEVLIST.UseDeviceState=0 +MCORE.DEVLIST.UseBatInfo=0 +MCORE.BDX.Sender=0 +MCORE.BDX.Receiver=1 +MCORE.BDX.SynchronousSender=0 +MCORE.BDX.SynchronousReceiver=1 +MCORE.BDX.AsynchronousSender=0 +MCORE.BDX.AsynchronousReceiver=1 +MCORE.BDX.Driver=0 +MCORE.BDX.Initiator=1 +MCORE.BDX.Responder=0 +MCORE.BDX.BlockQueryWithSkip=0 +MCORE.OTA.Requestor=1 +MCORE.OTA.Provider=0 +MCORE.OTA.HTTPS=0 +MCORE.OTA.RequestorConsent=0 +MCORE.OTA.Resume=0 +MCORE.OTA.VendorSpecific=0 +MCORE.ACL.Administrator=0 +MCORE.OTA.Retry=0 +MCORE.FS=0 +MCORE.SM.S=1 +MCORE.DT.S=1 +MCORE.G.MULTIENDPOINT=0 +BINFO.S=1 +BINFO.S.A0000=1 +BINFO.S.A0001=1 +BINFO.S.A0002=1 +BINFO.S.A0003=1 +BINFO.S.A0004=1 +BINFO.S.A0005=1 +BINFO.S.A0006=1 +BINFO.S.A0007=1 +BINFO.S.A0008=1 +BINFO.S.A0009=1 +BINFO.S.A000a=1 +BINFO.S.A000b=0 +BINFO.S.A000c=0 +BINFO.S.A000d=0 +BINFO.S.A000e=0 +BINFO.S.A000f=0 +BINFO.S.A0010=0 +BINFO.S.A0011=0 +BINFO.S.A0012=1 +BINFO.S.A0013=1 +BINFO.S.A0014=0 +BINFO.S.A0015=1 +BINFO.S.A0016=1 +BINFO.S.A0017=0 +BINFO.S.A0018=1 +BINFO.S.E00=1 +BINFO.S.E01=0 +BINFO.S.E02=0 +BINFO.S.E03=0 +BINFO.S.M.DeviceConfigurationChange=0 +BINFO.C.A0005=0 +DESC.S=1 +DESC.S.A0000=1 +DESC.S.A0001=1 +DESC.S.A0002=1 +DESC.S.A0003=1 +DESC.S.A0004=0 +DESC.S.A0005=0 +DESC.S.F00=0 +CGEN.S=1 +CGEN.C=0 +CGEN.S.A0000=1 +CGEN.S.A0001=1 +CGEN.S.A0002=1 +CGEN.S.A0003=1 +CGEN.S.A0004=1 +CGEN.S.A0005=0 +CGEN.S.A0006=0 +CGEN.S.A0007=0 +CGEN.S.A0008=0 +CGEN.S.A0009=0 +CGEN.S.C01.Tx=1 +CGEN.S.C03.Tx=1 +CGEN.S.C05.Tx=1 +CGEN.S.C07.Tx=0 +CGEN.S.C00.Rsp=1 +CGEN.S.C02.Rsp=1 +CGEN.S.C04.Rsp=1 +CGEN.S.C06.Rsp=0 +CGEN.S.F00=0 +{PICS_ESF_TC_CLIENT}=0 +DGGEN.S=1 +DGGEN.C=0 +DGGEN.S.A0000=1 +DGGEN.S.A0001=1 +DGGEN.S.A0002=1 +DGGEN.S.A0003=0 +DGGEN.S.A0004=0 +DGGEN.S.A0005=0 +DGGEN.S.A0006=0 +DGGEN.S.A0007=0 +DGGEN.S.A0008=1 +DGGEN.S.E00=0 +DGGEN.S.E01=0 +DGGEN.S.E02=0 +DGGEN.S.E03=1 +DGGEN.S.C02.Tx=1 +DGGEN.S.C04.Tx=0 +DGGEN.S.C00.Rsp=1 +DGGEN.S.C01.Rsp=1 +DGGEN.S.C03.Rsp=0 +DGGEN.S.F00=0 +DGGEN.C.C00.Tx=0 +DGGEN.C.C01.Tx=0 +G.S=1 +G.C=0 +GRPKEY.S=1 +GRPKEY.C=0 +GRPKEY.S.A0000=1 +GRPKEY.S.A0001=1 +GRPKEY.S.A0002=1 +GRPKEY.S.A0003=1 +GRPKEY.S.C02.Tx=1 +GRPKEY.S.C05.Tx=1 +GRPKEY.S.C00.Rsp=1 +GRPKEY.S.C01.Rsp=1 +GRPKEY.S.C03.Rsp=1 +GRPKEY.S.C04.Rsp=1 +G.S.C00.Rsp=1 +G.S.C01.Rsp=1 +G.S.C04.Rsp=1 +G.S.F00=1 +GRPKEY.S.F00=0 +GRPKEY.C.A0000=0 +GRPKEY.C.A0001=0 +G.C.C00.Tx=0 +G.C.C01.Tx=0 +GRPKEY.C.C00.Tx=0 +GRPKEY.C.C01.Tx=0 +GRPKEY.C.C03.Tx=0 +GRPKEY.C.C04.Tx=0 +CNET.S=1 +CNET.C=0 +CNET.S.A0000=1 +CNET.S.A0001=1 +CNET.S.A0002=1 +CNET.S.A0003=1 +CNET.S.A0004=1 +CNET.S.A0005=1 +CNET.S.A0006=1 +CNET.S.A0007=1 +CNET.S.A0008=1 +CNET.S.A0009=0 +CNET.S.A000a=0 +CNET.S.C01.Tx=1 +CNET.S.C05.Tx=1 +CNET.S.C07.Tx=1 +CNET.S.C00.Rsp=1 +CNET.S.C02.Rsp=1 +CNET.S.C03.Rsp=0 +CNET.S.C04.Rsp=1 +CNET.S.C06.Rsp=1 +CNET.S.C08.Rsp=1 +CNET.S.F00=1 +CNET.S.F01=0 +CNET.S.F02=0 +CNET.C.C00.Tx=0 +CNET.C.C02.Tx=0 +CNET.C.C03.Tx=0 +CNET.C.C04.Tx=0 +CNET.C.C06.Tx=0 +CNET.C.C08.Tx=0 +CNET.C.F00=0 +CNET.C.F01=0 +CNET.C.F02=0 +CC.S=1 +CC.C=0 +CC.S.A0000=0 +CC.S.A0001=0 +CC.S.A0002=1 +CC.S.A0003=1 +CC.S.A0004=1 +CC.S.A0005=0 +CC.S.A0006=0 +CC.S.A0007=1 +CC.S.A0008=1 +CC.S.A000f=1 +CC.S.A0010=1 +CC.S.A0011=0 +CC.S.A0012=0 +CC.S.A0013=0 +CC.S.A0015=0 +CC.S.A0016=0 +CC.S.A0017=0 +CC.S.A0019=0 +CC.S.A001a=0 +CC.S.A001b=0 +CC.S.A0020=0 +CC.S.A0021=0 +CC.S.A0022=0 +CC.S.A0024=0 +CC.S.A0025=0 +CC.S.A0026=0 +CC.S.A0028=0 +CC.S.A0029=0 +CC.S.A002a=0 +CC.S.A0030=0 +CC.S.A0031=0 +CC.S.A0032=0 +CC.S.A0033=0 +CC.S.A0034=0 +CC.S.A0036=0 +CC.S.A0037=0 +CC.S.A0038=0 +CC.S.A003a=0 +CC.S.A003b=0 +CC.S.A003c=0 +CC.S.A4000=0 +CC.S.A4001=1 +CC.S.A4002=0 +CC.S.A4003=0 +CC.S.A4004=0 +CC.S.A4005=0 +CC.S.A4006=0 +CC.S.A400a=1 +CC.S.A400b=1 +CC.S.A400c=1 +CC.S.A400d=1 +CC.S.A4010=1 +CC.S.C00.Rsp=0 +CC.S.C01.Rsp=0 +CC.S.C02.Rsp=0 +CC.S.C03.Rsp=0 +CC.S.C04.Rsp=0 +CC.S.C05.Rsp=0 +CC.S.C06.Rsp=0 +CC.S.C07.Rsp=1 +CC.S.C08.Rsp=1 +CC.S.C09.Rsp=1 +CC.S.C0a.Rsp=1 +CC.S.C40.Rsp=0 +CC.S.C41.Rsp=0 +CC.S.C42.Rsp=0 +CC.S.C43.Rsp=0 +CC.S.C44.Rsp=0 +CC.S.C47.Rsp=1 +CC.S.C4b.Rsp=1 +CC.S.C4c.Rsp=1 +CC.S.F00=0 +CC.S.F01=0 +CC.S.F02=0 +CC.S.F03=1 +CC.S.F04=1 +CC.C.A0000=0 +CC.C.A0001=0 +CC.C.A0002=0 +CC.C.A0003=0 +CC.C.A0004=0 +CC.C.A0005=0 +CC.C.A0006=0 +CC.C.A0007=0 +CC.C.A0008=0 +CC.C.A000f=0 +CC.C.A0010=0 +CC.C.A0011=0 +CC.C.A0012=0 +CC.C.A0013=0 +CC.C.A0015=0 +CC.C.A0016=0 +CC.C.A0017=0 +CC.C.A0019=0 +CC.C.A001a=0 +CC.C.A001b=0 +CC.C.A0020=0 +CC.C.A0021=0 +CC.C.A0022=0 +CC.C.A0024=0 +CC.C.A0025=0 +CC.C.A0026=0 +CC.C.A0028=0 +CC.C.A0029=0 +CC.C.A002a=0 +CC.C.A0030=0 +CC.C.A0031=0 +CC.C.A0032=0 +CC.C.A0033=0 +CC.C.A0034=0 +CC.C.A0036=0 +CC.C.A0037=0 +CC.C.A0038=0 +CC.C.A003a=0 +CC.C.A003b=0 +CC.C.A003c=0 +CC.C.A4000=0 +CC.C.A4001=0 +CC.C.A4002=0 +CC.C.A4003=0 +CC.C.A4004=0 +CC.C.A4005=0 +CC.C.A4006=0 +CC.C.A400a=0 +CC.C.A400b=0 +CC.C.A400c=0 +CC.C.A400d=0 +CC.C.A4010=0 +CC.C.C00.Tx=0 +CC.C.C01.Tx=0 +CC.C.C02.Tx=0 +CC.C.C03.Tx=0 +CC.C.C04.Tx=0 +CC.C.C05.Tx=0 +CC.C.C06.Tx=0 +CC.C.C07.Tx=0 +CC.C.C08.Tx=0 +CC.C.C09.Tx=0 +CC.C.C0a.Tx=0 +CC.C.C40.Tx=0 +CC.C.C41.Tx=0 +CC.C.C42.Tx=0 +CC.C.C43.Tx=0 +CC.C.C44.Tx=0 +CC.C.C47.Tx=0 +CC.C.C4b.Tx=0 +CC.C.C4c.Tx=0 +CC.C.F00=0 +CC.C.F01=0 +CC.C.F02=0 +CC.C.F03=0 +CC.C.F04=0 +G.S=1 +G.C=0 +G.S.A0000=1 +G.S.C00.Tx=1 +G.S.C01.Tx=1 +G.S.C02.Tx=1 +G.S.C03.Tx=1 +G.S.C00.Rsp=1 +G.S.C01.Rsp=1 +G.S.C02.Rsp=1 +G.S.C03.Rsp=1 +G.S.C04.Rsp=1 +G.S.C05.Rsp=1 +G.S.F00=1 +G.C.C00.Tx=0 +G.C.C01.Tx=0 +G.C.C02.Tx=0 +G.C.C03.Tx=0 +G.C.C04.Tx=0 +G.C.C05.Tx=0 +I.S=1 +I.C=0 +I.S.A0000=1 +I.S.A0001=1 +I.S.C00.Rsp=1 +I.S.C40.Rsp=1 +I.C.C00.Tx=0 +I.C.C40.Tx=0 +LVL.S=1 +LVL.C=0 +LVL.S.A0000=1 +LVL.S.A0001=1 +LVL.S.A0002=1 +LVL.S.A0003=1 +LVL.S.A0004=0 +LVL.S.A0005=0 +LVL.S.A0006=0 +LVL.S.A000f=1 +LVL.S.A0010=0 +LVL.S.A0011=1 +LVL.S.A0012=0 +LVL.S.A0013=0 +LVL.S.A0014=0 +LVL.S.A4000=1 +LVL.S.C00.Rsp=1 +LVL.S.C01.Rsp=1 +LVL.S.C02.Rsp=1 +LVL.S.C03.Rsp=1 +LVL.S.C04.Rsp=1 +LVL.S.C05.Rsp=1 +LVL.S.C06.Rsp=1 +LVL.S.C07.Rsp=1 +LVL.S.C08.Rsp=0 +LVL.S.F00=1 +LVL.S.F01=1 +LVL.S.F02=0 +LVL.S.M.VarRate=0 +OO.S=1 +OO.C=0 +OO.S.A0000=1 +OO.S.A4000=1 +OO.S.A4001=1 +OO.S.A4002=1 +OO.S.A4003=1 +OO.S.C00.Rsp=1 +OO.S.C01.Rsp=1 +OO.S.C02.Rsp=1 +OO.S.C40.Rsp=1 +OO.S.C41.Rsp=1 +OO.S.C42.Rsp=1 +OO.S.F00=1 +OO.S.F01=0 +OO.S.F02=0 +OO.M.ManuallyControlled=1 +OO.C.C00.Tx=0 +OO.C.C01.Tx=0 +OO.C.C02.Tx=0 +OO.C.C40.Tx=0 +OO.C.C41.Tx=0 +OO.C.C42.Tx=0 +S.S=1 +S.C=0 +S.S.A0001=1 +S.S.A0002=1 +S.S.C00.Tx=1 +S.S.C01.Tx=1 +S.S.C02.Tx=1 +S.S.C03.Tx=1 +S.S.C04.Tx=1 +S.S.C06.Tx=1 +S.S.C40.Tx=0 +S.S.C00.Rsp=1 +S.S.C01.Rsp=1 +S.S.C02.Rsp=1 +S.S.C03.Rsp=1 +S.S.C04.Rsp=1 +S.S.C05.Rsp=1 +S.S.C06.Rsp=1 +S.S.C40.Rsp=0 +S.S.F00=0 +S.C.C00.Tx=0 +S.C.C01.Tx=0 +S.C.C02.Tx=0 +S.C.C03.Tx=0 +S.C.C04.Tx=0 +S.C.C05.Tx=0 +S.C.C06.Tx=0 +S.C.C40.Tx=0 +PICS_SDK_CI_ONLY=0 +PICS_SKIP_SAMPLE_APP=1 +PICS_USER_PROMPT=1 diff --git a/tools/ci/gitlab_api.py b/tools/ci/gitlab_api.py index 5eeb595af..2524b77a2 100644 --- a/tools/ci/gitlab_api.py +++ b/tools/ci/gitlab_api.py @@ -14,6 +14,8 @@ class GitLabAPI: self.ci_merge_request_iid = os.getenv("CI_MERGE_REQUEST_IID") self.ci_pipeline_id = os.getenv("CI_PIPELINE_ID") self.ci_commit_ref_name = os.getenv("CI_COMMIT_REF_NAME") + self.ci_gitlab_pytest_ssid = os.getenv("CI_GITLAB_PYTEST_SSID") + self.ci_gitlab_pytest_passphrase = os.getenv("CI_GITLAB_PYTEST_PASSPHRASE") if not all([self.gitlab_api_url, self.gitlab_token, self.ci_project_id, self.ci_pipeline_id]): raise ValueError("Required GitLab environment variables are not set") diff --git a/tools/ci/pytest_cert_helper.py b/tools/ci/pytest_cert_helper.py new file mode 100644 index 000000000..6e731bb36 --- /dev/null +++ b/tools/ci/pytest_cert_helper.py @@ -0,0 +1,137 @@ +# SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO LTD + +# SPDX-License-Identifier: Apache-2.0 + +import pathlib +import re +import subprocess +from pytest_embedded import Dut +import os +from prettytable import PrettyTable +import json +import time +from gitlab_api import GitLabAPI +from results_formatter import ResultsFormatter + +PYTEST_PATH = str((pathlib.Path(__file__).parent / '../../connectedhomeip/connectedhomeip/src/python_testing').resolve()) + +gitlab_api = GitLabAPI() +PYTEST_SSID = gitlab_api.ci_gitlab_pytest_ssid +PYTEST_PASSPHRASE = gitlab_api.ci_gitlab_pytest_passphrase + +def load_test_commands(certification_tests: str): + if not PYTEST_SSID or not PYTEST_PASSPHRASE: + raise ValueError("CI_GITLAB_PYTEST_SSID and CI_GITLAB_PYTEST_PASSPHRASE must be set as environment variables") + + json_path = os.path.join(os.path.abspath(certification_tests)) + with open(json_path, 'r') as f: + data = json.load(f) + + common_args_template = data.get("common_args", "") + common_args = common_args_template.format(WIFI_SSID=PYTEST_SSID, WIFI_PASSPHRASE=PYTEST_PASSPHRASE) + + test_commands = [] + test_cases = data.get("test_cases", {}) + + for test_case_name, test_config in test_cases.items(): + script = test_config["script"] + args = test_config.get("args", "") + + if "test_case" in test_config: + test_param = f"--tests {test_config['test_case']}" + storage_path = f"--storage-path logs/{test_param}.json" + command = f"python3 {script} {common_args} {storage_path} {test_param} {args}".strip() + else: + storage_path = f"--storage-path logs/{test_case_name}.json" + command = f"python3 {script} {common_args} {storage_path} {args}".strip() + + test_commands.append({ + "name": test_case_name, + "command": command + }) + + return test_commands + +def clean_environment(): + clean_up_command = "rm -rf /tmp/chip_*" + subprocess.getoutput(clean_up_command) + cleanup_log_file = "rm -rf logs" + subprocess.getoutput(cleanup_log_file) + +def execute_test_command(full_command, dut:Dut, retry_attempts=2): + light = dut + for attempt in range(retry_attempts): + print(f"Attempt {attempt + 1} for command: {full_command}") + test_out_str = subprocess.getoutput(full_command) + print(f"Test output: {test_out_str}") + if "INFO:root:Final result: PASS !" in test_out_str: + print(f"Test passed on attempt {attempt + 1}.") + clean_environment() + time.sleep(5) + return True + else: + print(f"Test failed on attempt {attempt + 1}.") + time.sleep(10) + if attempt < retry_attempts - 1: + clean_environment() + light.write('matter esp factoryreset') + time.sleep(10) + return False + +def generate_markdown_results(results_table, chunk_id=None): + summary_title = f"Python Certification Test Results {chunk_id}" if chunk_id else "Test Results" + markdown_results = [ + "", + f"
{summary_title}", + "", + "| Test Case | Result |", + "|-----------|--------|" + ] + for test_case_name, result in results_table._rows: + markdown_results.append(f"| {test_case_name.strip()} | {result.strip()} |") + markdown_results.extend(["", "
", ""]) + return "\n".join(markdown_results) + +def update_mr_description_with_results(markdown_content, chunk_id=None): + try: + gitlab_api = GitLabAPI() + description = gitlab_api.fetch_merge_request_description() + updated_description = ResultsFormatter.update_cert_test_results_section(description, markdown_content, chunk_id=chunk_id) + gitlab_api.update_merge_request_description(updated_description) + print("Successfully updated MR description with test results.") + except Exception as e: + print(f"Failed to update MR description: {e}") + +def run_python_certification_tests(dut:Dut, certification_tests:str) -> None: + light = dut + test_commands = load_test_commands(certification_tests) + + num_commands = len(test_commands) + mid_index = (num_commands+1) // 2 + + # Split the test commands into two chunks to parallelize the tests + command_chunks = { + "1": test_commands[:mid_index], + "2": test_commands[mid_index:] + } + + results_table = PrettyTable() + results_table.field_names = ["Test Case", "Result"] + + test_chunk_key = os.getenv("TEST_CHUNK", "1") + selected_commands = command_chunks.get(test_chunk_key, []) + + for index, test_item in enumerate(selected_commands, start=1): + test_case_name = test_item["name"] + test_command = test_item["command"] + print(f"Executing command {index} in chunk {test_chunk_key}.") + full_command = f"cd {PYTEST_PATH} && {test_command}" + result = execute_test_command(full_command, light) + test_case_marker = "PASS" if result else "FAIL" + results_table.add_row([test_case_name, test_case_marker]) + print("Resetting the device for the next test case...") + light.write('matter esp factoryreset') + time.sleep(10) + + markdown_content = generate_markdown_results(results_table, chunk_id=test_chunk_key) + update_mr_description_with_results(markdown_content, chunk_id=test_chunk_key) diff --git a/tools/ci/requirements-pytest.txt b/tools/ci/requirements-pytest.txt index 3683ab373..d79a41b74 100644 --- a/tools/ci/requirements-pytest.txt +++ b/tools/ci/requirements-pytest.txt @@ -5,3 +5,5 @@ pytest-timeout netifaces esptool>=4.5 tabulate +prettytable +requests diff --git a/tools/ci/results_formatter.py b/tools/ci/results_formatter.py index fb8daf7f4..a41d8c12c 100644 --- a/tools/ci/results_formatter.py +++ b/tools/ci/results_formatter.py @@ -73,3 +73,22 @@ class ResultsFormatter: def format_heap_dump(parsed_logs): headers = ["State", "Current Free Memory", "Largest Free Block", "Min. Ever Free Size"] return tabulate(parsed_logs, headers=headers, tablefmt="grid") + + @staticmethod + def update_cert_test_results_section(description, markdown_content, chunk_id=None): + # Use chunk-specific markers + marker_id = f" {chunk_id}" if chunk_id else "" + marker_start = f"" + marker_end = f"" + cert_section = f"{marker_start}\n{markdown_content}\n{marker_end}" + + if marker_start in description and marker_end in description: + updated_description = re.sub( + rf"{re.escape(marker_start)}.*?{re.escape(marker_end)}", + cert_section, + description, + flags=re.DOTALL, + ) + else: + updated_description = description.strip() + "\n\n" + cert_section + return updated_description