tools/ci : Added the support to execute python certification tests in CI.

- Added the support of the certification tests in CI for esp32c6.
- Added pytest_cert_helper script to execute certification tests.
- Added the json file to maintain and modify a list of tests to be run in the CI.
- Added the PICS file for extended_color_light for wifi for the certification tests.
- Added the support to post the test results on gitlab MR.
- Added the support to use certification test json file as an argument to pytests.
- Added support to use gitlab environment variables like ssid and password in pytests.

.gitlab-ci.yml

@@ -525,9 +525,19 @@ pytest_esp32c6_esp_matter_dut:
     - cd ${ESP_MATTER_PATH}
     - rm -rf connectedhomeip/connectedhomeip
     - ln -s ${CHIP_SUBMODULE_PATH} connectedhomeip/connectedhomeip
+    - cd connectedhomeip/connectedhomeip
+    - source out/py-env/bin/activate
+    - cd ${ESP_MATTER_PATH}
+    - cp ${ESP_MATTER_PATH}/tools/ci/extended_color_light_wifi_pics_code.txt $ESP_MATTER_PATH/connectedhomeip/connectedhomeip/src/python_testing/
     - pip install -r tools/ci/requirements-pytest.txt
-    - pytest examples/ --target esp32c6 -m esp_matter_dut --junitxml=XUNIT_RESULT.xml
+    - python3 -c "import os; os.environ['TEST_CHUNK'] = '$TEST_CHUNK'"
+    - pytest examples/ --target esp32c6 -m esp_matter_dut --junitxml=XUNIT_RESULT.xml --certification-json=$ESP_MATTER_PATH/tools/ci/certification_test_commands.json
   tags: ["esp32c6", "esp_matter_dut"]
+  parallel:
+    matrix:
+      - TEST_CHUNK:
+          - "1"
+          - "2"
 
 pytest_esp32c2_esp_matter_dut:
   stage: target_test

conftest.py

@@ -120,6 +120,17 @@ def build_dir(app_path: str, target: Optional[str], config: Optional[str]) -> st
         f'no build dir valid. Please build the binary via "idf.py -B {recommend_place} build" and run pytest again'
     )
 
+def pytest_addoption(parser):
+    parser.addoption(
+        "--certification-json",
+        action="store",
+        default="certification_test_commands.json",
+        help="Path to the certification test commands JSON file",
+    )
+
+@pytest.fixture(scope="session")
+def certification_tests(request):
+    return request.config.getoption("--certification-json")
 
 @pytest.fixture(autouse=True)
 @multi_dut_fixture
@@ -207,7 +218,7 @@ class IdfPytestEmbedded:
         # set default timeout 10 minutes for each case
         for item in items:
             if 'timeout' not in item.keywords:
-                item.add_marker(pytest.mark.timeout(10 * 60))
+                item.add_marker(pytest.mark.timeout(40 * 60))
 
         # filter all the test cases with "--target"
         if self.target:

examples/pytest_esp_matter_light.py

@@ -11,7 +11,11 @@ from typing import Tuple
 from pytest_embedded import Dut
 import os
 import yaml
+import sys
 
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../tools/ci')))
+from pytest_cert_helper import *
+from gitlab_api import GitLabAPI
 
 CURRENT_DIR_LIGHT = str(pathlib.Path(__file__).parent)+'/light'
 CHIP_TOOL_EXE = str(pathlib.Path(__file__).parent)+ '/../connectedhomeip/connectedhomeip/out/host/chip-tool'
@@ -20,6 +24,9 @@ OT_DATASET_HEXSTR = '0e08000000000001000035060004001fffe00708fdb824be22185de50c0
 pytest_build_dir = CURRENT_DIR_LIGHT
 pytest_matter_thread_dir = CURRENT_DIR_LIGHT+'|'+OT_BR_EXAMPLE_PATH
 
+gitlab_api = GitLabAPI()
+PYTEST_SSID = gitlab_api.ci_gitlab_pytest_ssid
+PYTEST_PASSPHRASE = gitlab_api.ci_gitlab_pytest_passphrase
 
 @pytest.mark.esp32c3
 @pytest.mark.esp_matter_dut
@@ -37,7 +44,7 @@ def test_matter_commissioning_c3(dut:Dut) -> None:
     light.expect(r'chip\[DL\]\: Configuring CHIPoBLE advertising', timeout=20)
     # Start commissioning
     time.sleep(5)
-    command = CHIP_TOOL_EXE + ' pairing ble-wifi 1 ChipTEH2 chiptest123 20202021 3840'
+    command = CHIP_TOOL_EXE + f" pairing ble-wifi 1 {PYTEST_SSID} {PYTEST_PASSPHRASE} 20202021 3840"
     out_str = subprocess.getoutput(command)
     print(out_str)
     result = re.findall(r'Run command failure', str(out_str))
@@ -76,7 +83,7 @@ def test_matter_commissioning_c2(dut:Dut) -> None:
     light.expect(r'chip\[DL\]\: Configuring CHIPoBLE advertising', timeout=20)
     # Start commissioning
     time.sleep(5)
-    command = CHIP_TOOL_EXE + ' pairing ble-wifi 1 ChipTEH2 chiptest123 20202021 3840'
+    command = CHIP_TOOL_EXE + f" pairing ble-wifi 1 {PYTEST_SSID} {PYTEST_PASSPHRASE} 20202021 3840"
     out_str = subprocess.getoutput(command)
     print(out_str)
     result = re.findall(r'Run command failure', str(out_str))
@@ -109,13 +116,13 @@ def test_matter_commissioning_c2(dut:Dut) -> None:
 )
 
 # Matter over wifi commissioning
-def test_matter_commissioning_c6(dut:Dut) -> None:
+def test_matter_commissioning_c6(dut:Dut, certification_tests: str) -> None:
     light = dut
     # BLE start advertising
     light.expect(r'chip\[DL\]\: Configuring CHIPoBLE advertising', timeout=20)
     # Start commissioning
     time.sleep(5)
-    command = CHIP_TOOL_EXE + ' pairing ble-wifi 1 ChipTEH2 chiptest123 20202021 3840'
+    command = CHIP_TOOL_EXE + f" pairing ble-wifi 1 {PYTEST_SSID} {PYTEST_PASSPHRASE} 20202021 3840"
     out_str = subprocess.getoutput(command)
     print(out_str)
     result = re.findall(r'Run command failure', str(out_str))
@@ -134,6 +141,11 @@ def test_matter_commissioning_c6(dut:Dut) -> None:
     command = CHIP_TOOL_EXE + ' onoff toggle 1 1'
     out_str = subprocess.getoutput(command)
     print(out_str)
+
+    light.write('matter esp factoryreset')
+    time.sleep(10)
+    run_python_certification_tests(light, certification_tests)
+
     result = re.findall(r'Run command failure', str(out_str))
     if len(result) != 0:
       assert False
@@ -244,7 +256,7 @@ def test_matter_commissioning_h2(dut:Tuple[Dut, Dut]) -> None:
     ot_br.expect(r'chip\[DL\]\: Configuring CHIPoBLE advertising', timeout=20)
     # Start commissioning OTBR
     time.sleep(2)
-    command = CHIP_TOOL_EXE + ' pairing ble-wifi 1 ChipTEH2 chiptest123 20202021 3584'
+    command = CHIP_TOOL_EXE + f" pairing ble-wifi 1 {PYTEST_SSID} {PYTEST_PASSPHRASE} 20202021 3584"
     out_str = subprocess.getoutput(command)
     print(out_str)
     result = re.findall(r'Run command failure', str(out_str))

tools/ci/certification_test_commands.json

@@ -0,0 +1,138 @@
+{
+  "common_args": "-m ble-wifi --wifi-ssid {WIFI_SSID} --wifi-passphrase {WIFI_PASSPHRASE} --discriminator 3840 --passcode 20202021 --int-arg use_pase_only:0",
+  "test_cases": {
+    "TC_LVL_2_3": {
+      "script": "TC_LVL_2_3.py",
+      "args": "--endpoint 1 --PICS extended_color_light_wifi_pics_code.txt"
+    },
+    "TC_ACE_1_2": {
+      "script": "TC_ACE_1_2.py",
+      "args": "--timeout 300"
+    },
+    "TC_ACE_1_3": {
+      "script": "TC_ACE_1_3.py",
+      "args": ""
+    },
+    "TC_ACE_1_5": {
+      "script": "TC_ACE_1_5.py",
+      "args": "--timeout 300"
+    },
+    "TC_ACE_2_1": {
+      "script": "TC_AccessChecker.py",
+      "test_case": "test_TC_ACE_2_1",
+      "args": "--timeout 300"
+    },
+    "TC_ACE_2_2": {
+      "script": "TC_AccessChecker.py",
+      "test_case": "test_TC_ACE_2_2",
+      "args": "--timeout 300"
+    },
+    "TC_CGEN_2_4": {
+      "script": "TC_CGEN_2_4.py",
+      "args": "--timeout 300"
+    },
+    "TC_CNET_1_4": {
+      "script": "TC_CNET_1_4.py",
+      "args": "--timeout 300"
+    },
+    "TC_CNET_4_4": {
+      "script": "TC_CNET_4_4.py",
+      "args": "--timeout 300"
+    },
+    "TC_DESC_2_2": {
+      "script": "TC_DeviceBasicComposition.py",
+      "test_case": "test_TC_DESC_2_2",
+      "args": "--timeout 300"
+    },
+    "TC_DGGEN_2_4": {
+      "script": "TC_DGGEN_2_4.py",
+      "args": "--timeout 300"
+    },
+    "TC_DGGEN_3_2": {
+      "script": "TC_DGGEN_3_2.py",
+      "args": ""
+    },
+    "TC_DT_1_1": {
+      "script": "TC_DeviceBasicComposition.py",
+      "test_case": "test_TC_DT_1_1",
+      "args": "--timeout 300"
+    },
+    "TC_OPCREDS_3_1": {
+      "script": "TC_OPCREDS_3_1.py",
+      "args": "--timeout 300"
+    },
+    "TC_OPCREDS_3_2": {
+      "script": "TC_OPCREDS_3_2.py",
+      "args": "--timeout 300"
+    },
+    "TC_SM_1_1": {
+      "script": "TC_DeviceBasicComposition.py",
+      "test_case": "test_TC_SM_1_1",
+      "args": "--timeout 300"
+    },
+    "TC_SM_1_2": {
+      "script": "TC_DeviceBasicComposition.py",
+      "test_case": "test_TC_SM_1_2",
+      "args": "--timeout 300"
+    },
+    "TC_ACE_1_4": {
+      "script": "TC_ACE_1_4.py",
+      "args": "--int-arg PIXIT.ACE.APPENDPOINT:1 PIXIT.ACE.APPDEVTYPEID:269 use_pase_only:0 --string-arg PIXIT.ACE.APPCLUSTER:OnOff PIXIT.ACE.APPATTRIBUTE:OnOff"
+    },
+    "TC_IDM_4_2": {
+      "script": "TC_IDM_4_2.py",
+      "args": ""
+    },
+    "TC_IDM_12_1": {
+      "script": "TC_DeviceBasicComposition.py",
+      "test_case": "test_TC_IDM_12_1",
+      "args": ""
+    },
+    "TC_IDM_11_1": {
+      "script": "TC_DeviceBasicComposition.py",
+      "test_case": "test_TC_IDM_11_1",
+      "args": ""
+    },
+    "TC_IDM_10_5": {
+      "script": "TC_DeviceConformance.py",
+      "test_case": "test_TC_IDM_10_5",
+      "args": ""
+    },
+    "TC_IDM_10_3": {
+      "script": "TC_DeviceConformance.py",
+      "test_case": "test_TC_IDM_10_3",
+      "args": ""
+    },
+    "TC_IDM_10_2": {
+      "script": "TC_DeviceConformance.py",
+      "test_case": "test_TC_IDM_10_2",
+      "args": "--bool-arg allow_provisional:true"
+    },
+    "TC_IDM_10_1": {
+      "script": "TC_DeviceBasicComposition.py",
+      "test_case": "test_TC_IDM_10_1",
+      "args": ""
+    },
+    "TC_IDM_1_4": {
+      "script": "TC_IDM_1_4.py",
+      "args": "--hex-arg PIXIT.DGGEN.TEST_EVENT_TRIGGER_KEY:000102030405060708090a0b0c0d0e0f"
+    },
+    "TC_IDM_1_2": {
+      "script": "TC_IDM_1_2.py",
+      "args": "--PICS extended_color_light_wifi_pics_code.txt"
+    },
+    "TC_RR_1_1": {
+      "script": "TC_RR_1_1.py",
+      "args": "--timeout 100000"
+    },
+    "TC_SC_3_6": {
+      "script": "TC_SC_3_6.py",
+      "args": "--PICS extended_color_light_wifi_pics_code.txt"
+    },
+    "TC_SC_3_6_post": {
+      "script": "TC_SC_3_6.py",
+      "args": "--PICS extended_color_light_wifi_pics_code.txt --bool-arg: post_cert_test:true"
+    }
+  }
+}
+

tools/ci/extended_color_light_wifi_pics_code.txt

@@ -0,0 +1,594 @@
+ACL.S=1
+ACL.C=0
+ACL.S.A0000=1
+ACL.S.A0001=0
+ACL.S.A0002=1
+ACL.S.A0003=1
+ACL.S.A0004=1
+ACL.S.A0005=0
+ACL.S.A0006=0
+ACL.S.E00=1
+ACL.S.E01=0
+ACL.S.E02=0
+ACL.S.C01.Tx=0
+ACL.S.C00.Rsp=0
+ACL.S.F00=0
+ACL.S.F01=0
+ACL.S=1
+ACL.C=0
+APPDEVICE.S=1
+ACL.S.A0000=1
+CADMIN.S=1
+CADMIN.C=0
+CADMIN.S.A0000=1
+CADMIN.S.A0001=1
+CADMIN.S.A0002=1
+CADMIN.S.C00.Rsp=1
+CADMIN.S.C01.Rsp=0
+CADMIN.S.C02.Rsp=1
+CADMIN.S.F00=0
+CADMIN.C.A0000=0
+CADMIN.C.A0001=0
+CADMIN.C.A0002=0
+CADMIN.C.C00.Tx=0
+CADMIN.C.C01.Tx=0
+CADMIN.C.C02.Tx=0
+CADMIN.M.UserInterfaceDisplay=0
+CADMIN.M.AudioInterface=0
+MCORE.DT_SW_COMP=0
+MCORE.COM.BLE=1
+MCORE.COM.WIFI=1
+MCORE.COM.ETH=0
+MCORE.COM.THR=0
+MCORE.COM.WIRELESS=1
+MCORE.ROLE.COMMISSIONER=0
+MCORE.ROLE.COMMISSIONEE=1
+MCORE.ROLE.CONTROLLER=0
+MCORE.DLOG.S.UTCTIMESTAMP=0
+MCORE.DLOG.S.TIMESINCEBOOT=0
+MCORE.DD.CHIP_DEV=0
+MCORE.DD.DEV_LOCK=0
+MCORE.DD.DEV_BARRIER=0
+MCORE.DD.IE=0
+MCORE.DD.QR=1
+MCORE.DD.MANUAL_PC=1
+MCORE.DD.NFC=0
+MCORE.DD.UI=0
+MCORE.DD.COMM_DISCOVERY=0
+MCORE.DD.CONCATENATED_QR_CODE=1
+MCORE.DD.CTRL_CONCATENATED_QR_CODE_1=0
+MCORE.DD.CTRL_CONCATENATED_QR_CODE_1_INORDER=0
+MCORE.DD.CTRL_CONCATENATED_QR_CODE_2=0
+MCORE.DD.DISCOVERY_BLE=0
+MCORE.DD.DISCOVERY_PAF=0
+MCORE.DD.DISCOVERY_IP=0
+MCORE.DD.DISCOVERY_SOFTAP=0
+MCORE.DD.STANDARD_COMM_FLOW=1
+MCORE.DD.NON_CONCURRENT_CONNECTION=0
+MCORE.DD.USER_INTENT_COMM_FLOW=0
+MCORE.DD.CUSTOM_COMM_FLOW=0
+MCORE.DD.MANUAL_PC_COMMISSIONING=0
+MCORE.DD.11_MANUAL_PC=0
+MCORE.DD.21_MANUAL_PC=0
+MCORE.DD.PHYSICAL_TAMPERING=0
+MCORE.DD.SCAN_NFC=0
+MCORE.DD.QR_COMMISSIONING=0
+MCORE.DD.SCAN_QR_CODE=0
+MCORE.DD.EXTENDED_DISCOVERY=0
+MCORE.DD.COMMISSIONING_SUBTYPE_V=1
+MCORE.DD.COMMISSIONING_SUBTYPE_T=0
+MCORE.DD.TXT_KEY_VP=1
+MCORE.DD.TXT_KEY_DT=0
+MCORE.DD.TXT_KEY_DN=0
+MCORE.DD.TXT_KEY_RI=0
+MCORE.DD.TXT_KEY_PH=1
+MCORE.DD.TXT_KEY_PI=1
+MCORE.COM.PAF=0
+MCORE.DD.ESF_TC_COMMISSIONER=0
+MCORE.SC.VENDOR_SUBTYPE=1
+MCORE.SC.DEVTYPE_SUBTYPE=0
+MCORE.SC.VP_KEY=1
+MCORE.SC.DT_KEY=0
+MCORE.SC.DN_KEY=0
+MCORE.SC.RI_KEY=0
+MCORE.SC.PH_KEY=1
+MCORE.SC.PI_KEY=1
+MCORE.SC.SII_OP_DISCOVERY_KEY=1
+MCORE.SC.SAI_OP_DISCOVERY_KEY=1
+MCORE.SC.SAT_OP_DISCOVERY_KEY=1
+MCORE.SC.T_KEY=1
+MCORE.SC.SII_COMM_DISCOVERY_KEY=1
+MCORE.SC.SAI_COMM_DISCOVERY_KEY=1
+MCORE.SC.EXTENDED_DISCOVERY=0
+MCORE.SC.SIT_ICD=0
+MCORE.SC.TCP=0
+MCORE.IDM.S=1
+MCORE.IDM.C=0
+MCORE.IDM.C.InvokeRequest=0
+MCORE.IDM.C.ReadRequest=0
+MCORE.IDM.C.WriteRequest=0
+MCORE.IDM.C.SubscribeRequest=0
+MCORE.IDM.C.InvokeRequest.BatchCommands=0
+MCORE.IDM.C.ReadRequest.Attribute.DataType_Bool=0
+MCORE.IDM.C.ReadRequest.Attribute.DataType_String=0
+MCORE.IDM.C.ReadRequest.Attribute.DataType_UnsignedInteger=0
+MCORE.IDM.C.ReadRequest.Attribute.DataType_SignedInteger=0
+MCORE.IDM.C.ReadRequest.Attribute.DataType_Struct=0
+MCORE.IDM.C.ReadRequest.Attribute.DataType_FloatingPoint=0
+MCORE.IDM.C.ReadRequest.Attribute.DataType_List=0
+MCORE.IDM.C.ReadRequest.Attribute.DataType_OctetString=0
+MCORE.IDM.C.ReadRequest.Attribute.DataType_Enum=0
+MCORE.IDM.C.ReadRequest.Attribute.DataType_Bitmap=0
+MCORE.IDM.C.WriteRequest.Attribute.DataType_Bool=0
+MCORE.IDM.C.WriteRequest.Attribute.DataType_String=0
+MCORE.IDM.C.WriteRequest.Attribute.DataType_UnsignedInteger=0
+MCORE.IDM.C.WriteRequest.Attribute.DataType_SignedInteger=0
+MCORE.IDM.C.WriteRequest.Attribute.DataType_Struct=0
+MCORE.IDM.C.WriteRequest.Attribute.DataType_FloatingPoint=0
+MCORE.IDM.C.WriteRequest.Attribute.DataType_List=0
+MCORE.IDM.C.WriteRequest.Attribute.DataType_OctetString=0
+MCORE.IDM.C.WriteRequest.Attribute.DataType_Enum=0
+MCORE.IDM.C.WriteRequest.Attribute.DataType_Bitmap=0
+MCORE.IDM.S.Attribute_W.DataType_Bool=0
+MCORE.IDM.S.Attribute_W.DataType_String=1
+MCORE.IDM.S.Attribute_W.DataType_UnsignedInteger=1
+MCORE.IDM.S.Attribute_W.DataType_SignedInteger=0
+MCORE.IDM.S.Attribute_W.DataType_Struct=0
+MCORE.IDM.S.Attribute_W.DataType_FloatingPoint=0
+MCORE.IDM.S.Attribute_W.DataType_List=1
+MCORE.IDM.S.Attribute_W.DataType_OctetString=0
+MCORE.IDM.S.Attribute_W.DataType_Enum=1
+MCORE.IDM.S.Attribute_W.DataType_Bitmap=1
+MCORE.IDM.C.SubscribeRequest.Attribute.DataType_Bool=0
+MCORE.IDM.C.SubscribeRequest.Attribute.DataType_String=0
+MCORE.IDM.C.SubscribeRequest.Attribute.DataType_UnsignedInteger=0
+MCORE.IDM.C.SubscribeRequest.Attribute.DataType_Integer=0
+MCORE.IDM.C.SubscribeRequest.Attribute.DataType_FloatingPoint=0
+MCORE.IDM.C.SubscribeRequest.Attribute.DataType_List=0
+MCORE.IDM.S.LargeData=0
+MCORE.IDM.C.SubscribeEvent=0
+MCORE.IDM.C.ReadEvent=0
+MCORE.IDM.C.SubscribeRequest.MultipleAttributes=0
+MCORE.IDM.S.PersistentSubscription=0
+MCORE.BRIDGE=0
+MCORE.BRIDGE.BatInfo=0
+MCORE.BRIDGE.OtherControl=0
+MCORE.BRIDGE.AllowDeviceRename=0
+MCORE.BRIDGECLIENT=0
+MCORE.DEVLIST.UseDevices=0
+MCORE.DEVLIST.UseDeviceName=0
+MCORE.DEVLIST.UseDeviceState=0
+MCORE.DEVLIST.UseBatInfo=0
+MCORE.BDX.Sender=0
+MCORE.BDX.Receiver=1
+MCORE.BDX.SynchronousSender=0
+MCORE.BDX.SynchronousReceiver=1
+MCORE.BDX.AsynchronousSender=0
+MCORE.BDX.AsynchronousReceiver=1
+MCORE.BDX.Driver=0
+MCORE.BDX.Initiator=1
+MCORE.BDX.Responder=0
+MCORE.BDX.BlockQueryWithSkip=0
+MCORE.OTA.Requestor=1
+MCORE.OTA.Provider=0
+MCORE.OTA.HTTPS=0
+MCORE.OTA.RequestorConsent=0
+MCORE.OTA.Resume=0
+MCORE.OTA.VendorSpecific=0
+MCORE.ACL.Administrator=0
+MCORE.OTA.Retry=0
+MCORE.FS=0
+MCORE.SM.S=1
+MCORE.DT.S=1
+MCORE.G.MULTIENDPOINT=0
+BINFO.S=1
+BINFO.S.A0000=1
+BINFO.S.A0001=1
+BINFO.S.A0002=1
+BINFO.S.A0003=1
+BINFO.S.A0004=1
+BINFO.S.A0005=1
+BINFO.S.A0006=1
+BINFO.S.A0007=1
+BINFO.S.A0008=1
+BINFO.S.A0009=1
+BINFO.S.A000a=1
+BINFO.S.A000b=0
+BINFO.S.A000c=0
+BINFO.S.A000d=0
+BINFO.S.A000e=0
+BINFO.S.A000f=0
+BINFO.S.A0010=0
+BINFO.S.A0011=0
+BINFO.S.A0012=1
+BINFO.S.A0013=1
+BINFO.S.A0014=0
+BINFO.S.A0015=1
+BINFO.S.A0016=1
+BINFO.S.A0017=0
+BINFO.S.A0018=1
+BINFO.S.E00=1
+BINFO.S.E01=0
+BINFO.S.E02=0
+BINFO.S.E03=0
+BINFO.S.M.DeviceConfigurationChange=0
+BINFO.C.A0005=0
+DESC.S=1
+DESC.S.A0000=1
+DESC.S.A0001=1
+DESC.S.A0002=1
+DESC.S.A0003=1
+DESC.S.A0004=0
+DESC.S.A0005=0
+DESC.S.F00=0
+CGEN.S=1
+CGEN.C=0
+CGEN.S.A0000=1
+CGEN.S.A0001=1
+CGEN.S.A0002=1
+CGEN.S.A0003=1
+CGEN.S.A0004=1
+CGEN.S.A0005=0
+CGEN.S.A0006=0
+CGEN.S.A0007=0
+CGEN.S.A0008=0
+CGEN.S.A0009=0
+CGEN.S.C01.Tx=1
+CGEN.S.C03.Tx=1
+CGEN.S.C05.Tx=1
+CGEN.S.C07.Tx=0
+CGEN.S.C00.Rsp=1
+CGEN.S.C02.Rsp=1
+CGEN.S.C04.Rsp=1
+CGEN.S.C06.Rsp=0
+CGEN.S.F00=0
+{PICS_ESF_TC_CLIENT}=0
+DGGEN.S=1
+DGGEN.C=0
+DGGEN.S.A0000=1
+DGGEN.S.A0001=1
+DGGEN.S.A0002=1
+DGGEN.S.A0003=0
+DGGEN.S.A0004=0
+DGGEN.S.A0005=0
+DGGEN.S.A0006=0
+DGGEN.S.A0007=0
+DGGEN.S.A0008=1
+DGGEN.S.E00=0
+DGGEN.S.E01=0
+DGGEN.S.E02=0
+DGGEN.S.E03=1
+DGGEN.S.C02.Tx=1
+DGGEN.S.C04.Tx=0
+DGGEN.S.C00.Rsp=1
+DGGEN.S.C01.Rsp=1
+DGGEN.S.C03.Rsp=0
+DGGEN.S.F00=0
+DGGEN.C.C00.Tx=0
+DGGEN.C.C01.Tx=0
+G.S=1
+G.C=0
+GRPKEY.S=1
+GRPKEY.C=0
+GRPKEY.S.A0000=1
+GRPKEY.S.A0001=1
+GRPKEY.S.A0002=1
+GRPKEY.S.A0003=1
+GRPKEY.S.C02.Tx=1
+GRPKEY.S.C05.Tx=1
+GRPKEY.S.C00.Rsp=1
+GRPKEY.S.C01.Rsp=1
+GRPKEY.S.C03.Rsp=1
+GRPKEY.S.C04.Rsp=1
+G.S.C00.Rsp=1
+G.S.C01.Rsp=1
+G.S.C04.Rsp=1
+G.S.F00=1
+GRPKEY.S.F00=0
+GRPKEY.C.A0000=0
+GRPKEY.C.A0001=0
+G.C.C00.Tx=0
+G.C.C01.Tx=0
+GRPKEY.C.C00.Tx=0
+GRPKEY.C.C01.Tx=0
+GRPKEY.C.C03.Tx=0
+GRPKEY.C.C04.Tx=0
+CNET.S=1
+CNET.C=0
+CNET.S.A0000=1
+CNET.S.A0001=1
+CNET.S.A0002=1
+CNET.S.A0003=1
+CNET.S.A0004=1
+CNET.S.A0005=1
+CNET.S.A0006=1
+CNET.S.A0007=1
+CNET.S.A0008=1
+CNET.S.A0009=0
+CNET.S.A000a=0
+CNET.S.C01.Tx=1
+CNET.S.C05.Tx=1
+CNET.S.C07.Tx=1
+CNET.S.C00.Rsp=1
+CNET.S.C02.Rsp=1
+CNET.S.C03.Rsp=0
+CNET.S.C04.Rsp=1
+CNET.S.C06.Rsp=1
+CNET.S.C08.Rsp=1
+CNET.S.F00=1
+CNET.S.F01=0
+CNET.S.F02=0
+CNET.C.C00.Tx=0
+CNET.C.C02.Tx=0
+CNET.C.C03.Tx=0
+CNET.C.C04.Tx=0
+CNET.C.C06.Tx=0
+CNET.C.C08.Tx=0
+CNET.C.F00=0
+CNET.C.F01=0
+CNET.C.F02=0
+CC.S=1
+CC.C=0
+CC.S.A0000=0
+CC.S.A0001=0
+CC.S.A0002=1
+CC.S.A0003=1
+CC.S.A0004=1
+CC.S.A0005=0
+CC.S.A0006=0
+CC.S.A0007=1
+CC.S.A0008=1
+CC.S.A000f=1
+CC.S.A0010=1
+CC.S.A0011=0
+CC.S.A0012=0
+CC.S.A0013=0
+CC.S.A0015=0
+CC.S.A0016=0
+CC.S.A0017=0
+CC.S.A0019=0
+CC.S.A001a=0
+CC.S.A001b=0
+CC.S.A0020=0
+CC.S.A0021=0
+CC.S.A0022=0
+CC.S.A0024=0
+CC.S.A0025=0
+CC.S.A0026=0
+CC.S.A0028=0
+CC.S.A0029=0
+CC.S.A002a=0
+CC.S.A0030=0
+CC.S.A0031=0
+CC.S.A0032=0
+CC.S.A0033=0
+CC.S.A0034=0
+CC.S.A0036=0
+CC.S.A0037=0
+CC.S.A0038=0
+CC.S.A003a=0
+CC.S.A003b=0
+CC.S.A003c=0
+CC.S.A4000=0
+CC.S.A4001=1
+CC.S.A4002=0
+CC.S.A4003=0
+CC.S.A4004=0
+CC.S.A4005=0
+CC.S.A4006=0
+CC.S.A400a=1
+CC.S.A400b=1
+CC.S.A400c=1
+CC.S.A400d=1
+CC.S.A4010=1
+CC.S.C00.Rsp=0
+CC.S.C01.Rsp=0
+CC.S.C02.Rsp=0
+CC.S.C03.Rsp=0
+CC.S.C04.Rsp=0
+CC.S.C05.Rsp=0
+CC.S.C06.Rsp=0
+CC.S.C07.Rsp=1
+CC.S.C08.Rsp=1
+CC.S.C09.Rsp=1
+CC.S.C0a.Rsp=1
+CC.S.C40.Rsp=0
+CC.S.C41.Rsp=0
+CC.S.C42.Rsp=0
+CC.S.C43.Rsp=0
+CC.S.C44.Rsp=0
+CC.S.C47.Rsp=1
+CC.S.C4b.Rsp=1
+CC.S.C4c.Rsp=1
+CC.S.F00=0
+CC.S.F01=0
+CC.S.F02=0
+CC.S.F03=1
+CC.S.F04=1
+CC.C.A0000=0
+CC.C.A0001=0
+CC.C.A0002=0
+CC.C.A0003=0
+CC.C.A0004=0
+CC.C.A0005=0
+CC.C.A0006=0
+CC.C.A0007=0
+CC.C.A0008=0
+CC.C.A000f=0
+CC.C.A0010=0
+CC.C.A0011=0
+CC.C.A0012=0
+CC.C.A0013=0
+CC.C.A0015=0
+CC.C.A0016=0
+CC.C.A0017=0
+CC.C.A0019=0
+CC.C.A001a=0
+CC.C.A001b=0
+CC.C.A0020=0
+CC.C.A0021=0
+CC.C.A0022=0
+CC.C.A0024=0
+CC.C.A0025=0
+CC.C.A0026=0
+CC.C.A0028=0
+CC.C.A0029=0
+CC.C.A002a=0
+CC.C.A0030=0
+CC.C.A0031=0
+CC.C.A0032=0
+CC.C.A0033=0
+CC.C.A0034=0
+CC.C.A0036=0
+CC.C.A0037=0
+CC.C.A0038=0
+CC.C.A003a=0
+CC.C.A003b=0
+CC.C.A003c=0
+CC.C.A4000=0
+CC.C.A4001=0
+CC.C.A4002=0
+CC.C.A4003=0
+CC.C.A4004=0
+CC.C.A4005=0
+CC.C.A4006=0
+CC.C.A400a=0
+CC.C.A400b=0
+CC.C.A400c=0
+CC.C.A400d=0
+CC.C.A4010=0
+CC.C.C00.Tx=0
+CC.C.C01.Tx=0
+CC.C.C02.Tx=0
+CC.C.C03.Tx=0
+CC.C.C04.Tx=0
+CC.C.C05.Tx=0
+CC.C.C06.Tx=0
+CC.C.C07.Tx=0
+CC.C.C08.Tx=0
+CC.C.C09.Tx=0
+CC.C.C0a.Tx=0
+CC.C.C40.Tx=0
+CC.C.C41.Tx=0
+CC.C.C42.Tx=0
+CC.C.C43.Tx=0
+CC.C.C44.Tx=0
+CC.C.C47.Tx=0
+CC.C.C4b.Tx=0
+CC.C.C4c.Tx=0
+CC.C.F00=0
+CC.C.F01=0
+CC.C.F02=0
+CC.C.F03=0
+CC.C.F04=0
+G.S=1
+G.C=0
+G.S.A0000=1
+G.S.C00.Tx=1
+G.S.C01.Tx=1
+G.S.C02.Tx=1
+G.S.C03.Tx=1
+G.S.C00.Rsp=1
+G.S.C01.Rsp=1
+G.S.C02.Rsp=1
+G.S.C03.Rsp=1
+G.S.C04.Rsp=1
+G.S.C05.Rsp=1
+G.S.F00=1
+G.C.C00.Tx=0
+G.C.C01.Tx=0
+G.C.C02.Tx=0
+G.C.C03.Tx=0
+G.C.C04.Tx=0
+G.C.C05.Tx=0
+I.S=1
+I.C=0
+I.S.A0000=1
+I.S.A0001=1
+I.S.C00.Rsp=1
+I.S.C40.Rsp=1
+I.C.C00.Tx=0
+I.C.C40.Tx=0
+LVL.S=1
+LVL.C=0
+LVL.S.A0000=1
+LVL.S.A0001=1
+LVL.S.A0002=1
+LVL.S.A0003=1
+LVL.S.A0004=0
+LVL.S.A0005=0
+LVL.S.A0006=0
+LVL.S.A000f=1
+LVL.S.A0010=0
+LVL.S.A0011=1
+LVL.S.A0012=0
+LVL.S.A0013=0
+LVL.S.A0014=0
+LVL.S.A4000=1
+LVL.S.C00.Rsp=1
+LVL.S.C01.Rsp=1
+LVL.S.C02.Rsp=1
+LVL.S.C03.Rsp=1
+LVL.S.C04.Rsp=1
+LVL.S.C05.Rsp=1
+LVL.S.C06.Rsp=1
+LVL.S.C07.Rsp=1
+LVL.S.C08.Rsp=0
+LVL.S.F00=1
+LVL.S.F01=1
+LVL.S.F02=0
+LVL.S.M.VarRate=0
+OO.S=1
+OO.C=0
+OO.S.A0000=1
+OO.S.A4000=1
+OO.S.A4001=1
+OO.S.A4002=1
+OO.S.A4003=1
+OO.S.C00.Rsp=1
+OO.S.C01.Rsp=1
+OO.S.C02.Rsp=1
+OO.S.C40.Rsp=1
+OO.S.C41.Rsp=1
+OO.S.C42.Rsp=1
+OO.S.F00=1
+OO.S.F01=0
+OO.S.F02=0
+OO.M.ManuallyControlled=1
+OO.C.C00.Tx=0
+OO.C.C01.Tx=0
+OO.C.C02.Tx=0
+OO.C.C40.Tx=0
+OO.C.C41.Tx=0
+OO.C.C42.Tx=0
+S.S=1
+S.C=0
+S.S.A0001=1
+S.S.A0002=1
+S.S.C00.Tx=1
+S.S.C01.Tx=1
+S.S.C02.Tx=1
+S.S.C03.Tx=1
+S.S.C04.Tx=1
+S.S.C06.Tx=1
+S.S.C40.Tx=0
+S.S.C00.Rsp=1
+S.S.C01.Rsp=1
+S.S.C02.Rsp=1
+S.S.C03.Rsp=1
+S.S.C04.Rsp=1
+S.S.C05.Rsp=1
+S.S.C06.Rsp=1
+S.S.C40.Rsp=0
+S.S.F00=0
+S.C.C00.Tx=0
+S.C.C01.Tx=0
+S.C.C02.Tx=0
+S.C.C03.Tx=0
+S.C.C04.Tx=0
+S.C.C05.Tx=0
+S.C.C06.Tx=0
+S.C.C40.Tx=0
+PICS_SDK_CI_ONLY=0
+PICS_SKIP_SAMPLE_APP=1
+PICS_USER_PROMPT=1

tools/ci/gitlab_api.py

@@ -14,6 +14,8 @@ class GitLabAPI:
         self.ci_merge_request_iid = os.getenv("CI_MERGE_REQUEST_IID")
         self.ci_pipeline_id = os.getenv("CI_PIPELINE_ID")
         self.ci_commit_ref_name = os.getenv("CI_COMMIT_REF_NAME")
+        self.ci_gitlab_pytest_ssid = os.getenv("CI_GITLAB_PYTEST_SSID")
+        self.ci_gitlab_pytest_passphrase = os.getenv("CI_GITLAB_PYTEST_PASSPHRASE")
         
         if not all([self.gitlab_api_url, self.gitlab_token, self.ci_project_id, self.ci_pipeline_id]):
             raise ValueError("Required GitLab environment variables are not set")

tools/ci/pytest_cert_helper.py

@@ -0,0 +1,137 @@
+# SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO LTD
+
+# SPDX-License-Identifier: Apache-2.0
+
+import pathlib
+import re
+import subprocess
+from pytest_embedded import Dut
+import os
+from prettytable import PrettyTable
+import json
+import time
+from gitlab_api import GitLabAPI
+from results_formatter import ResultsFormatter
+
+PYTEST_PATH = str((pathlib.Path(__file__).parent / '../../connectedhomeip/connectedhomeip/src/python_testing').resolve())
+
+gitlab_api = GitLabAPI()
+PYTEST_SSID = gitlab_api.ci_gitlab_pytest_ssid
+PYTEST_PASSPHRASE = gitlab_api.ci_gitlab_pytest_passphrase
+
+def load_test_commands(certification_tests: str):
+    if not PYTEST_SSID or not PYTEST_PASSPHRASE:
+        raise ValueError("CI_GITLAB_PYTEST_SSID and CI_GITLAB_PYTEST_PASSPHRASE must be set as environment variables")
+
+    json_path = os.path.join(os.path.abspath(certification_tests))
+    with open(json_path, 'r') as f:
+        data = json.load(f)
+
+    common_args_template = data.get("common_args", "")
+    common_args = common_args_template.format(WIFI_SSID=PYTEST_SSID, WIFI_PASSPHRASE=PYTEST_PASSPHRASE)
+
+    test_commands = []
+    test_cases = data.get("test_cases", {})
+
+    for test_case_name, test_config in test_cases.items():
+        script = test_config["script"]
+        args = test_config.get("args", "")
+
+        if "test_case" in test_config:
+            test_param = f"--tests {test_config['test_case']}"
+            storage_path = f"--storage-path logs/{test_param}.json"
+            command = f"python3 {script} {common_args} {storage_path} {test_param} {args}".strip()
+        else:
+            storage_path = f"--storage-path logs/{test_case_name}.json"
+            command = f"python3 {script} {common_args} {storage_path} {args}".strip()
+
+        test_commands.append({
+            "name": test_case_name,
+            "command": command
+        })
+
+    return test_commands
+
+def clean_environment():
+    clean_up_command = "rm -rf /tmp/chip_*"
+    subprocess.getoutput(clean_up_command)
+    cleanup_log_file = "rm -rf logs"
+    subprocess.getoutput(cleanup_log_file)
+
+def execute_test_command(full_command, dut:Dut, retry_attempts=2):
+    light = dut
+    for attempt in range(retry_attempts):
+        print(f"Attempt {attempt + 1} for command: {full_command}")
+        test_out_str = subprocess.getoutput(full_command)
+        print(f"Test output: {test_out_str}")
+        if "INFO:root:Final result: PASS !" in test_out_str:
+            print(f"Test passed on attempt {attempt + 1}.")
+            clean_environment()
+            time.sleep(5)
+            return True
+        else:
+            print(f"Test failed on attempt {attempt + 1}.")
+            time.sleep(10)
+            if attempt < retry_attempts - 1:
+                clean_environment()
+                light.write('matter esp factoryreset')
+                time.sleep(10)
+    return False
+
+def generate_markdown_results(results_table, chunk_id=None):
+    summary_title = f"Python Certification Test Results {chunk_id}" if chunk_id else "Test Results"
+    markdown_results = [
+        "<!-- Expandable Section -->",
+        f"<details><summary>{summary_title}</summary>",
+        "",
+        "| Test Case | Result |",
+        "|-----------|--------|"
+    ]
+    for test_case_name, result in results_table._rows:
+        markdown_results.append(f"| {test_case_name.strip()} | {result.strip()} |")
+    markdown_results.extend(["", "</details>", "<!-- End Expandable Section -->"])
+    return "\n".join(markdown_results)
+
+def update_mr_description_with_results(markdown_content, chunk_id=None):
+    try:
+        gitlab_api = GitLabAPI()
+        description = gitlab_api.fetch_merge_request_description()
+        updated_description = ResultsFormatter.update_cert_test_results_section(description, markdown_content, chunk_id=chunk_id)
+        gitlab_api.update_merge_request_description(updated_description)
+        print("Successfully updated MR description with test results.")
+    except Exception as e:
+        print(f"Failed to update MR description: {e}")
+
+def run_python_certification_tests(dut:Dut, certification_tests:str) -> None:
+    light = dut
+    test_commands = load_test_commands(certification_tests)
+
+    num_commands = len(test_commands)
+    mid_index = (num_commands+1) // 2
+
+    # Split the test commands into two chunks to parallelize the tests
+    command_chunks = {
+        "1": test_commands[:mid_index],
+        "2": test_commands[mid_index:]
+    }
+
+    results_table = PrettyTable()
+    results_table.field_names = ["Test Case", "Result"]
+
+    test_chunk_key = os.getenv("TEST_CHUNK", "1")
+    selected_commands = command_chunks.get(test_chunk_key, [])
+
+    for index, test_item in enumerate(selected_commands, start=1):
+        test_case_name = test_item["name"]
+        test_command = test_item["command"]
+        print(f"Executing command {index} in chunk {test_chunk_key}.")
+        full_command = f"cd {PYTEST_PATH} && {test_command}"
+        result = execute_test_command(full_command, light)
+        test_case_marker = "PASS" if result else "FAIL"
+        results_table.add_row([test_case_name, test_case_marker])
+        print("Resetting the device for the next test case...")
+        light.write('matter esp factoryreset')
+        time.sleep(10)
+
+    markdown_content = generate_markdown_results(results_table, chunk_id=test_chunk_key)
+    update_mr_description_with_results(markdown_content, chunk_id=test_chunk_key)

tools/ci/requirements-pytest.txt

@@ -5,3 +5,5 @@ pytest-timeout
 netifaces
 esptool>=4.5
 tabulate
+prettytable
+requests

tools/ci/results_formatter.py

@@ -73,3 +73,22 @@ class ResultsFormatter:
     def format_heap_dump(parsed_logs):
         headers = ["State", "Current Free Memory", "Largest Free Block", "Min. Ever Free Size"]
         return tabulate(parsed_logs, headers=headers, tablefmt="grid")
+
+    @staticmethod
+    def update_cert_test_results_section(description, markdown_content, chunk_id=None):
+        # Use chunk-specific markers
+        marker_id = f" {chunk_id}" if chunk_id else ""
+        marker_start = f"<!-- START: Cert Test Results{marker_id} -->"
+        marker_end = f"<!-- END: Cert Test Results{marker_id} -->"
+        cert_section = f"{marker_start}\n{markdown_content}\n{marker_end}"
+
+        if marker_start in description and marker_end in description:
+            updated_description = re.sub(
+                rf"{re.escape(marker_start)}.*?{re.escape(marker_end)}",
+                cert_section,
+                description,
+                flags=re.DOTALL,
+            )
+        else:
+            updated_description = description.strip() + "\n\n" + cert_section
+        return updated_description