From 365268ae94f8640cd7a3de4e74444d7ce41e306f Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Tue, 24 Mar 2026 11:26:36 +0100 Subject: [PATCH 1/3] ci: move dev_password and dev_user alongside the test scripts --- conftest.py | 5 +++-- examples/network/bridge/pytest_example_bridge.py | 7 +++++++ tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/conftest.py b/conftest.py index 91cd649514..b733ae6383 100644 --- a/conftest.py +++ b/conftest.py @@ -42,6 +42,7 @@ from idf_pytest.plugin import IDF_LOCAL_PLUGIN_KEY from idf_pytest.plugin import IdfLocalPlugin from idf_pytest.plugin import requires_elf_or_map from idf_pytest.utils import format_case_id +from pytest_embedded.plugin import _request_param_or_config_option_or_default from pytest_embedded.plugin import multi_dut_fixture from pytest_embedded.utils import to_bytes from pytest_embedded.utils import to_str @@ -456,12 +457,12 @@ def log_minimum_free_heap_size(dut: IdfDut, config: str, idf_path: str) -> t.Cal @pytest.fixture(scope='session') def dev_password(request: FixtureRequest) -> str: - return request.config.getoption('dev_passwd') or '' + return _request_param_or_config_option_or_default(request, 'dev_password', '') # type: ignore @pytest.fixture(scope='session') def dev_user(request: FixtureRequest) -> str: - return request.config.getoption('dev_user') or '' + return _request_param_or_config_option_or_default(request, 'dev_user', '') # type: ignore ################## diff --git a/examples/network/bridge/pytest_example_bridge.py b/examples/network/bridge/pytest_example_bridge.py index 50f5d16e29..eddcf6f447 100644 --- a/examples/network/bridge/pytest_example_bridge.py +++ b/examples/network/bridge/pytest_example_bridge.py @@ -25,6 +25,8 @@ from pytest_embedded_idf.utils import idf_parametrize # Testbed configuration ETHVM_ENDNODE_USER = 'ci.ethvm' +ETHERNET_TEST_USER = os.getenv('ETHERNET_TEST_USER') +ETHERNET_TEST_PASSWORD = os.getenv('ETHERNET_TEST_PASSWORD') BR_PORTS_NUM = 2 IPERF_BW_LIM = 6 @@ -632,5 +634,10 @@ def setup_test_environment() -> Generator[None, None, None]: indirect=True, ) @idf_parametrize('target', ['esp32'], indirect=['target']) +@pytest.mark.parametrize( + 'dev_user, dev_password', + [(ETHERNET_TEST_USER, ETHERNET_TEST_PASSWORD)], + indirect=True, +) def test_esp_eth_bridge(dut: Dut, dev_user: str, dev_password: str) -> None: eth_bridge_test(dut, dev_user, dev_password) diff --git a/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml index 7db95ac674..ed15bfea1a 100644 --- a/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml +++ b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml @@ -55,7 +55,7 @@ # set while generating the pipeline nodes: "" INSTALL_EXTRA_TOOLS: "xtensa-esp-elf-gdb riscv32-esp-elf-gdb openocd-esp32 esp-rom-elfs" - PYTEST_EXTRA_FLAGS: "--dev-passwd ${ETHERNET_TEST_PASSWORD} --dev-user ${ETHERNET_TEST_USER} --capture=fd --verbosity=0 --unity-test-report-mode merge" + PYTEST_EXTRA_FLAGS: "--capture=fd --verbosity=0 --unity-test-report-mode merge" needs: - pipeline: $PARENT_PIPELINE_ID job: pipeline_variables From ddc7e0cdf7977fc9948f2be1e6747f999efc6f5c Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Tue, 24 Mar 2026 11:22:20 +0100 Subject: [PATCH 2/3] ci: move qemu test cli args alongside test scripts remove redundant host_test marker --- .gitlab/ci/host-test.yml | 1 - .../pytest_bootloader_support.py | 2 - .../test_apps/console/pytest_console.py | 165 ++++++++---------- components/efuse/test_apps/pytest_efuse.py | 1 - .../esp_event/test_apps/pytest_esp_event.py | 1 - .../test_apps/pytest_esp_ringbuf.py | 1 - .../test_apps/rom_tests/pytest_esp_rom.py | 1 - .../heap/test_apps/heap_tests/pytest_heap.py | 1 - .../pytest_pthread_unity_tests.py | 1 - .../contribute/esp-idf-tests-with-pytest.rst | 61 +++++++ .../contribute/esp-idf-tests-with-pytest.rst | 61 +++++++ .../cmake/import_lib/pytest_import_lib.py | 1 - .../hello_world/pytest_hello_world.py | 3 +- .../pytest_https_x509_bundle.py | 15 +- .../efuse/pytest_system_efuse_example.py | 1 - .../advanced_https_ota/pytest_advanced_ota.py | 2 - tools/ci/idf_pytest/plugin.py | 37 +++- .../secure_boot/pytest_secure_boot.py | 15 +- .../pytest_signed_app_no_secure_boot.py | 1 - .../std_filesystem/pytest_std_filesystem.py | 1 - .../esp_intr_dump/pytest_esp_intr_dump.py | 6 +- 21 files changed, 252 insertions(+), 126 deletions(-) diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index f9ef743d9d..77da25bb07 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -372,7 +372,6 @@ test_pytest_qemu: - run_cmd pytest --target $IDF_TARGET -m qemu - --embedded-services idf,qemu --junitxml=XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} --qemu-extra-args \"-global driver=timer.$IDF_TARGET.timg,property=wdt_disable,value=true\" diff --git a/components/bootloader_support/test_apps/bootloader_support/pytest_bootloader_support.py b/components/bootloader_support/test_apps/bootloader_support/pytest_bootloader_support.py index 059c3c5c9f..790a430d6b 100644 --- a/components/bootloader_support/test_apps/bootloader_support/pytest_bootloader_support.py +++ b/components/bootloader_support/test_apps/bootloader_support/pytest_bootloader_support.py @@ -11,14 +11,12 @@ def test_bootloader_support(dut: Dut) -> None: dut.run_all_single_board_cases() -@pytest.mark.host_test @pytest.mark.qemu @idf_parametrize('target', ['esp32'], indirect=['target']) def test_bootloader_support_qemu_esp32(dut: Dut) -> None: dut.run_all_single_board_cases() -@pytest.mark.host_test @pytest.mark.qemu @idf_parametrize('target', ['esp32c3'], indirect=['target']) def test_bootloader_support_qemu_esp32c3(dut: Dut) -> None: diff --git a/components/console/test_apps/console/pytest_console.py b/components/console/test_apps/console/pytest_console.py index 45ffbd5441..8c15097164 100644 --- a/components/console/test_apps/console/pytest_console.py +++ b/components/console/test_apps/console/pytest_console.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: CC0-1.0 import pytest from pytest_embedded import Dut @@ -50,63 +50,59 @@ def do_test_help_quit(dut: Dut) -> None: dut.expect(r'quit\s+Quit REPL environment\s+esp>') -@idf_parametrize('config', ['defaults'], indirect=['config']) @idf_parametrize( - 'target,test_on,markers', + 'target,config,embedded_services,markers', [ - ('linux', 'host', (pytest.mark.host_test,)), - ('esp32', 'target', (pytest.mark.generic,)), - ('esp32c3', 'target', (pytest.mark.generic,)), - ('esp32', 'qemu', (pytest.mark.host_test, pytest.mark.qemu)), + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), ], - indirect=['target'], + indirect=['target', 'config', 'embedded_services'], ) -def test_console(dut: Dut, test_on: str) -> None: +def test_console(dut: Dut) -> None: dut.run_all_single_board_cases(group='!ignore', timeout=120) -@idf_parametrize('config', ['defaults'], indirect=['config']) @idf_parametrize( - 'target,test_on,markers', + 'target,config,embedded_services,markers', [ - ('linux', 'host', (pytest.mark.host_test,)), - ('esp32', 'target', (pytest.mark.generic,)), - ('esp32c3', 'target', (pytest.mark.generic,)), - ('esp32', 'qemu', (pytest.mark.host_test, pytest.mark.qemu)), + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), ], - indirect=['target'], + indirect=['target', 'config', 'embedded_services'], ) -def test_console_repl(dut: Dut, test_on: str) -> None: +def test_console_repl(dut: Dut) -> None: do_test_quit(dut) -@idf_parametrize('config', ['defaults'], indirect=['config']) @idf_parametrize( - 'target,test_on,markers', + 'target,config,embedded_services,markers', [ - ('linux', 'host', (pytest.mark.host_test,)), - ('esp32', 'target', (pytest.mark.generic,)), - ('esp32c3', 'target', (pytest.mark.generic,)), - ('esp32', 'qemu', (pytest.mark.host_test, pytest.mark.qemu)), + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), ], - indirect=['target'], + indirect=['target', 'config', 'embedded_services'], ) -def test_console_help_sorted_registration(dut: Dut, test_on: str) -> None: +def test_console_help_sorted_registration(dut: Dut) -> None: do_test_help_generic(dut, 'sorted') -@idf_parametrize('config', ['defaults'], indirect=['config']) @idf_parametrize( - 'target,test_on,markers', + 'target,config,embedded_services,markers', [ - ('linux', 'host', (pytest.mark.host_test,)), - ('esp32', 'target', (pytest.mark.generic,)), - ('esp32c3', 'target', (pytest.mark.generic,)), - ('esp32', 'qemu', (pytest.mark.host_test, pytest.mark.qemu)), + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), ], - indirect=['target'], + indirect=['target', 'config', 'embedded_services'], ) -def test_console_help_reverse_registration(dut: Dut, test_on: str) -> None: +def test_console_help_reverse_registration(dut: Dut) -> None: dut.expect_exact('Press ENTER to see the list of tests') dut.confirm_write('"esp console help command - reverse registration"', expect_str='esp>') @@ -143,33 +139,31 @@ def test_console_sorted_help_reverse_registration(dut: Dut, test_on: str) -> Non do_test_help_generic(dut, 'reverse') -@idf_parametrize('config', ['defaults'], indirect=['config']) @idf_parametrize( - 'target,test_on,markers', + 'target,config,embedded_services,markers', [ - ('linux', 'host', (pytest.mark.host_test,)), - ('esp32', 'target', (pytest.mark.generic,)), - ('esp32c3', 'target', (pytest.mark.generic,)), - ('esp32', 'qemu', (pytest.mark.host_test, pytest.mark.qemu)), + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), ], - indirect=['target'], + indirect=['target', 'config', 'embedded_services'], ) -def test_console_help_quit(dut: Dut, test_on: str) -> None: +def test_console_help_quit(dut: Dut) -> None: do_test_help_quit(dut) -@idf_parametrize('config', ['defaults'], indirect=['config']) @idf_parametrize( - 'target,test_on,markers', + 'target,config,embedded_services,markers', [ - ('linux', 'host', (pytest.mark.host_test,)), - ('esp32', 'target', (pytest.mark.generic,)), - ('esp32c3', 'target', (pytest.mark.generic,)), - ('esp32', 'qemu', (pytest.mark.host_test, pytest.mark.qemu)), + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), ], - indirect=['target'], + indirect=['target', 'config', 'embedded_services'], ) -def test_console_help_verbose_level_0(dut: Dut, test_on: str) -> None: +def test_console_help_verbose_level_0(dut: Dut) -> None: help_verbose_info = 'Print the summary of all registered commands if no arguments are given,' dut.expect_exact('Press ENTER to see the list of tests') dut.confirm_write('"esp console help command - set verbose level = 0"', expect_str='esp>') @@ -180,18 +174,17 @@ def test_console_help_verbose_level_0(dut: Dut, test_on: str) -> None: dut.expect_exact('help', not_matching=help_verbose_info) -@idf_parametrize('config', ['defaults'], indirect=['config']) @idf_parametrize( - 'target,test_on,markers', + 'target,config,embedded_services,markers', [ - ('linux', 'host', (pytest.mark.host_test,)), - ('esp32', 'target', (pytest.mark.generic,)), - ('esp32c3', 'target', (pytest.mark.generic,)), - ('esp32', 'qemu', (pytest.mark.host_test, pytest.mark.qemu)), + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), ], - indirect=['target'], + indirect=['target', 'config', 'embedded_services'], ) -def test_console_help_verbose_level_1(dut: Dut, test_on: str) -> None: +def test_console_help_verbose_level_1(dut: Dut) -> None: help_verbose_info = 'Print the summary of all registered commands if no arguments are given,' dut.expect_exact('Press ENTER to see the list of tests') dut.confirm_write('"esp console help command - set verbose level = 1"', expect_str='esp>') @@ -201,18 +194,17 @@ def test_console_help_verbose_level_1(dut: Dut, test_on: str) -> None: dut.expect_exact(help_verbose_info) -@idf_parametrize('config', ['defaults'], indirect=['config']) @idf_parametrize( - 'target,test_on,markers', + 'target,config,embedded_services,markers', [ - ('linux', 'host', (pytest.mark.host_test,)), - ('esp32', 'target', (pytest.mark.generic,)), - ('esp32c3', 'target', (pytest.mark.generic,)), - ('esp32', 'qemu', (pytest.mark.host_test, pytest.mark.qemu)), + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), ], - indirect=['target'], + indirect=['target', 'config', 'embedded_services'], ) -def test_console_help_verbose_subcommand(dut: Dut, test_on: str) -> None: +def test_console_help_verbose_subcommand(dut: Dut) -> None: help_verbose_info = 'Print the summary of all registered commands if no arguments are given,' dut.expect_exact('Press ENTER to see the list of tests') dut.confirm_write('"esp console help command - --verbose sub command"', expect_str='esp>') @@ -227,18 +219,17 @@ def test_console_help_verbose_subcommand(dut: Dut, test_on: str) -> None: dut.expect_exact(help_verbose_info) -@idf_parametrize('config', ['defaults'], indirect=['config']) @idf_parametrize( - 'target,test_on,markers', + 'target,config,embedded_services,markers', [ - ('linux', 'host', (pytest.mark.host_test,)), - ('esp32', 'target', (pytest.mark.generic,)), - ('esp32c3', 'target', (pytest.mark.generic,)), - ('esp32', 'qemu', (pytest.mark.host_test, pytest.mark.qemu)), + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), ], - indirect=['target'], + indirect=['target', 'config', 'embedded_services'], ) -def test_console_help_deregister(dut: Dut, test_on: str) -> None: +def test_console_help_deregister(dut: Dut) -> None: dut.expect_exact('Press ENTER to see the list of tests') dut.confirm_write('"esp console deregister commands"', expect_str='esp>') @@ -250,18 +241,17 @@ def test_console_help_deregister(dut: Dut, test_on: str) -> None: dut.expect_exact(cmd_z_description, not_matching=cmd_a_description) -@idf_parametrize('config', ['defaults'], indirect=['config']) @idf_parametrize( - 'target,test_on,markers', + 'target,config,embedded_services,markers', [ - ('linux', 'host', (pytest.mark.host_test,)), - ('esp32', 'target', (pytest.mark.generic,)), - ('esp32c3', 'target', (pytest.mark.generic,)), - ('esp32', 'qemu', (pytest.mark.host_test, pytest.mark.qemu)), + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), ], - indirect=['target'], + indirect=['target', 'config', 'embedded_services'], ) -def test_console_help_re_register(dut: Dut, test_on: str) -> None: +def test_console_help_re_register(dut: Dut) -> None: dut.expect_exact('Press ENTER to see the list of tests') dut.confirm_write('"esp console re-register commands"', expect_str='esp>') @@ -271,17 +261,16 @@ def test_console_help_re_register(dut: Dut, test_on: str) -> None: dut.expect_exact('should appear first in help') -@idf_parametrize('config', ['defaults'], indirect=['config']) @idf_parametrize( - 'target,test_on,markers', + 'target,config,embedded_services,markers', [ - ('esp32', 'target', (pytest.mark.generic,)), - ('esp32c3', 'target', (pytest.mark.generic,)), - ('esp32', 'qemu', (pytest.mark.host_test, pytest.mark.qemu)), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), ], - indirect=['target'], + indirect=['target', 'config', 'embedded_services'], ) -def test_console_custom_uart_repl(dut: Dut, test_on: str) -> None: +def test_console_custom_uart_repl(dut: Dut) -> None: dut.expect_exact('Press ENTER to see the list of tests') dut.confirm_write('"esp console repl custom_uart test"', expect_str='Running repl on UART1') diff --git a/components/efuse/test_apps/pytest_efuse.py b/components/efuse/test_apps/pytest_efuse.py index 6d823e62cb..cc9b564ffc 100644 --- a/components/efuse/test_apps/pytest_efuse.py +++ b/components/efuse/test_apps/pytest_efuse.py @@ -15,7 +15,6 @@ def test_efuse(dut: Dut) -> None: @pytest.mark.qemu -@pytest.mark.host_test @idf_parametrize('target', ['esp32'], indirect=['target']) def test_efuse_qemu(dut: Dut) -> None: dut.run_all_single_board_cases() diff --git a/components/esp_event/test_apps/pytest_esp_event.py b/components/esp_event/test_apps/pytest_esp_event.py index 90af0229a4..b3ee9f5e8c 100644 --- a/components/esp_event/test_apps/pytest_esp_event.py +++ b/components/esp_event/test_apps/pytest_esp_event.py @@ -26,7 +26,6 @@ def test_esp_event_ext_ram(dut: Dut) -> None: dut.run_all_single_board_cases() -@pytest.mark.host_test @pytest.mark.qemu @pytest.mark.xfail('config.getvalue("target") == "esp32c3"', reason='Unstable on QEMU, needs investigation') @pytest.mark.parametrize( diff --git a/components/esp_ringbuf/test_apps/pytest_esp_ringbuf.py b/components/esp_ringbuf/test_apps/pytest_esp_ringbuf.py index ca44dcfa16..f27087d9c9 100644 --- a/components/esp_ringbuf/test_apps/pytest_esp_ringbuf.py +++ b/components/esp_ringbuf/test_apps/pytest_esp_ringbuf.py @@ -24,7 +24,6 @@ def test_esp_ringbuf(dut: Dut) -> None: dut.run_all_single_board_cases() -@pytest.mark.host_test @pytest.mark.qemu @pytest.mark.parametrize( 'target', diff --git a/components/esp_rom/test_apps/rom_tests/pytest_esp_rom.py b/components/esp_rom/test_apps/rom_tests/pytest_esp_rom.py index 0bc7bfef1a..b3e41e01af 100644 --- a/components/esp_rom/test_apps/rom_tests/pytest_esp_rom.py +++ b/components/esp_rom/test_apps/rom_tests/pytest_esp_rom.py @@ -15,7 +15,6 @@ def test_esp_rom(dut: Dut) -> None: dut.run_all_single_board_cases() -@pytest.mark.host_test @pytest.mark.qemu @idf_parametrize('target', ['esp32c3'], indirect=['target']) def test_esp_rom_qemu(dut: Dut) -> None: diff --git a/components/heap/test_apps/heap_tests/pytest_heap.py b/components/heap/test_apps/heap_tests/pytest_heap.py index 37093d0eac..8eb33d469a 100644 --- a/components/heap/test_apps/heap_tests/pytest_heap.py +++ b/components/heap/test_apps/heap_tests/pytest_heap.py @@ -21,7 +21,6 @@ def test_heap_poisoning(dut: Dut) -> None: dut.run_all_single_board_cases() -@pytest.mark.host_test @pytest.mark.qemu @pytest.mark.parametrize( 'config, embedded_services', diff --git a/components/pthread/test_apps/pthread_unity_tests/pytest_pthread_unity_tests.py b/components/pthread/test_apps/pthread_unity_tests/pytest_pthread_unity_tests.py index 2cf9732abd..6b81396727 100644 --- a/components/pthread/test_apps/pthread_unity_tests/pytest_pthread_unity_tests.py +++ b/components/pthread/test_apps/pthread_unity_tests/pytest_pthread_unity_tests.py @@ -47,7 +47,6 @@ def test_pthread_single_core_tls(dut: Dut) -> None: dut.run_all_single_board_cases(group='thread-specific', timeout=300) -@pytest.mark.host_test @pytest.mark.qemu @idf_parametrize('target', ['esp32'], indirect=['target']) def test_pthread_qemu(dut: Dut) -> None: diff --git a/docs/en/contribute/esp-idf-tests-with-pytest.rst b/docs/en/contribute/esp-idf-tests-with-pytest.rst index c1e6a19eaa..bf1fe59bf9 100644 --- a/docs/en/contribute/esp-idf-tests-with-pytest.rst +++ b/docs/en/contribute/esp-idf-tests-with-pytest.rst @@ -120,6 +120,26 @@ Next is the environment marker. The ``@pytest.mark.generic`` marker indicates th Finally, we have the test function. With a ``dut`` fixture. In single-dut test cases, the ``dut`` fixture is an instance of ``IdfDut`` class, for multi-dut test cases, it is a tuple of ``IdfDut`` instances. For more details regarding the ``IdfDut`` class, please refer to `pytest-embedded IdfDut API reference `__. +Running Tests in QEMU +^^^^^^^^^^^^^^^^^^^^^ + +To execute a pytest case in QEMU, add the ``@pytest.mark.qemu`` marker to the test function. + +.. code-block:: python + + @pytest.mark.qemu + @idf_parametrize('target', ['esp32', 'esp32c3'], indirect=['target']) + def test_hello_world_qemu(dut) -> None: + dut.expect('Hello world!') + +This is the simplest way to run the same test flow in QEMU instead of on physical hardware. + +For a simple QEMU-only test, adding ``pytest.mark.qemu`` is enough and the ``idf,qemu`` embedded services will be selected automatically. + +For a mixed environment matrix, specify ``embedded_services`` manually for each case. See the later section :ref:`same-app-with-different-running-environments` for a more complex example. + +For QEMU installation and setup, refer to :doc:`../api-guides/tools/qemu`. + Same App With Different sdkconfig Files ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -203,6 +223,47 @@ Now this test function would be replicated to 2 test cases (represented as test * ``esp32.foo.test_foo_bar`` * ``esp32s2.bar.test_foo_bar`` +Same App With Different Running Environments +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Sometimes the same app should be validated in different running environments, for example on the host with the Linux target, on real hardware, or in QEMU. If a single ``@pytest.mark.qemu`` test is not enough, combine ``target``, ``config``, and ``embedded_services`` in a single ``idf_parametrize`` decorator, and attach the required marker for each case. + +The following example is adapted from :idf_file:`components/console/test_apps/console/pytest_console.py`: + +.. code-block:: python + + @idf_parametrize( + 'target,config,embedded_services,markers', + [ + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), + ], + indirect=['target', 'config', 'embedded_services'], + ) + def test_console_repl(dut) -> None: + dut.expect_exact('Press ENTER to see the list of tests') + +This creates four test cases for the same app: + +* Linux host execution with the ``idf`` service +* ESP32 hardware execution with the ``esp,idf`` services +* ESP32-C3 hardware execution with the ``esp,idf`` services +* ESP32 execution in QEMU with the ``idf,qemu`` services + +When running locally, you can select only the environment you want: + +.. code-block:: shell + + $ pytest --target linux + $ pytest -m qemu + $ pytest -m qemu --target esp32 + +``pytest --target linux`` selects Linux target cases only. ``pytest -m qemu`` selects all QEMU-marked cases. ``pytest -m qemu --target esp32`` further limits the selection to QEMU cases for the ESP32 target. + +Use this pattern when the test logic is the same but the execution environment changes. + Testing Serial Output (Expecting) --------------------------------- diff --git a/docs/zh_CN/contribute/esp-idf-tests-with-pytest.rst b/docs/zh_CN/contribute/esp-idf-tests-with-pytest.rst index fc7fd44932..0b19324321 100644 --- a/docs/zh_CN/contribute/esp-idf-tests-with-pytest.rst +++ b/docs/zh_CN/contribute/esp-idf-tests-with-pytest.rst @@ -120,6 +120,26 @@ ESP-IDF 在主机端使用 pytest 框架（以及一些 pytest 插件）来自 关于测试函数，使用了一个 ``dut`` fixture。在单一 DUT 测试用例中，``dut`` fixture 是 ``IdfDut`` 类的一个实例，对于多个 DUT 测试用例，它是 ``IdfDut`` 实例的一个元组。有关 ``IdfDut`` 类的更多详细信息，请参阅 `pytest-embedded IdfDut API 参考 `__。 +在 QEMU 中运行测试 +^^^^^^^^^^^^^^^^^^^^^ + +要在 QEMU 中执行 pytest 测试用例，请将 ``@pytest.mark.qemu`` 添加到测试函数上。 + +.. code-block:: python + + @pytest.mark.qemu + @idf_parametrize('target', ['esp32', 'esp32c3'], indirect=['target']) + def test_hello_world_qemu(dut) -> None: + dut.expect('Hello world!') + +这是在 QEMU 中运行与物理硬件相同测试流程的最简单方式。 + +对于简单的纯 QEMU 测试，只需添加 ``pytest.mark.qemu``，系统会自动选择 ``idf,qemu`` 对应的 embedded services。 + +对于混合运行环境矩阵，则需要为每个用例手动指定 ``embedded_services``。更复杂的示例请参阅本指南后面的 :ref:`same-app-with-different-running-environments` 小节。 + +有关 QEMU 的安装和配置，请参阅 :doc:`../api-guides/tools/qemu`。 + 使用不同的 sdkconfig 文件运行相同的应用程序 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -203,6 +223,47 @@ ESP-IDF 在主机端使用 pytest 框架（以及一些 pytest 插件）来自 * ``esp32.foo.test_foo_bar`` * ``esp32s2.bar.test_foo_bar`` +在不同运行环境中运行相同的应用程序 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +有时，同一个应用程序需要在不同的运行环境中进行验证，例如在 Linux target 的主机上、真实硬件上，或在 QEMU 中运行。如果单独使用 ``@pytest.mark.qemu`` 测试还不够，可以在一个 ``idf_parametrize`` 装饰器中组合 ``target``、``config`` 和 ``embedded_services``，并为每种情况附加所需的 marker。 + +下面的示例改编自 :idf_file:`components/console/test_apps/console/pytest_console.py`： + +.. code-block:: python + + @idf_parametrize( + 'target,config,embedded_services,markers', + [ + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), + ], + indirect=['target', 'config', 'embedded_services'], + ) + def test_console_repl(dut) -> None: + dut.expect_exact('Press ENTER to see the list of tests') + +这会为同一个应用程序生成 4 个测试用例： + +* 在 Linux 主机上使用 ``idf`` service 运行 +* 在 ESP32 硬件上使用 ``esp,idf`` services 运行 +* 在 ESP32-C3 硬件上使用 ``esp,idf`` services 运行 +* 在 QEMU 中以 ESP32 为目标，使用 ``idf,qemu`` services 运行 + +在本地运行时，可以按需只选择某一种运行环境： + +.. code-block:: shell + + $ pytest --target linux + $ pytest -m qemu + $ pytest -m qemu --target esp32 + +``pytest --target linux`` 只选择 Linux target 的测试用例。``pytest -m qemu`` 选择所有带有 QEMU marker 的测试用例。``pytest -m qemu --target esp32`` 会进一步把范围限制为目标芯片为 ESP32 的 QEMU 测试用例。 + +当测试逻辑相同，但执行环境不同的时候，可使用此模式。 + 测试串行输出 ^^^^^^^^^^^^ diff --git a/examples/build_system/cmake/import_lib/pytest_import_lib.py b/examples/build_system/cmake/import_lib/pytest_import_lib.py index 95f86b0807..4bf5203c62 100644 --- a/examples/build_system/cmake/import_lib/pytest_import_lib.py +++ b/examples/build_system/cmake/import_lib/pytest_import_lib.py @@ -5,7 +5,6 @@ from pytest_embedded_idf.utils import idf_parametrize from pytest_embedded_qemu.dut import QemuDut -@pytest.mark.host_test @pytest.mark.qemu @idf_parametrize('target', ['esp32', 'esp32c3'], indirect=['target']) def test_pytest_host(dut: QemuDut) -> None: diff --git a/examples/get-started/hello_world/pytest_hello_world.py b/examples/get-started/hello_world/pytest_hello_world.py index 64f6bb8f5f..e3ae95b654 100644 --- a/examples/get-started/hello_world/pytest_hello_world.py +++ b/examples/get-started/hello_world/pytest_hello_world.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: CC0-1.0 import hashlib import logging -from typing import Callable +from collections.abc import Callable import pytest from pytest_embedded_idf.dut import IdfDut @@ -45,7 +45,6 @@ def verify_elf_sha256_embedding(app: QemuApp, sha256_reported: str) -> None: raise ValueError('ELF file SHA256 mismatch') -@pytest.mark.host_test @pytest.mark.qemu @idf_parametrize('target', ['esp32', 'esp32c3'], indirect=['target']) def test_hello_world_host(app: QemuApp, dut: QemuDut) -> None: diff --git a/examples/protocols/https_x509_bundle/pytest_https_x509_bundle.py b/examples/protocols/https_x509_bundle/pytest_https_x509_bundle.py index 52e48ce783..0c787a8097 100644 --- a/examples/protocols/https_x509_bundle/pytest_https_x509_bundle.py +++ b/examples/protocols/https_x509_bundle/pytest_https_x509_bundle.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Unlicense OR CC0-1.0 import logging import os @@ -20,13 +20,13 @@ def test_examples_protocol_https_x509_bundle(dut: Dut) -> None: # check and log bin size binary_file = os.path.join(dut.app.binary_path, 'https_x509_bundle.bin') bin_size = os.path.getsize(binary_file) - logging.info('https_x509_bundle_bin_size : {}KB'.format(bin_size // 1024)) + logging.info(f'https_x509_bundle_bin_size : {bin_size // 1024}KB') dut.expect(r'IPv4 address: (\d+\.\d+\.\d+\.\d+)[^\d]', timeout=30) # start test num_URLS = int(dut.expect(r'Connecting to (\d+) URLs', timeout=30)[1].decode()) for _ in range(num_URLS): dut.expect(r'Connection established to ([\s\S]*)', timeout=30) - dut.expect('Completed {} connections'.format(num_URLS), timeout=60) + dut.expect(f'Completed {num_URLS} connections', timeout=60) @pytest.mark.ethernet @@ -43,16 +43,15 @@ def test_examples_protocol_https_x509_bundle_dynamic_buffer(dut: Dut) -> None: # check and log bin size binary_file = os.path.join(dut.app.binary_path, 'https_x509_bundle.bin') bin_size = os.path.getsize(binary_file) - logging.info('https_x509_bundle_bin_size : {}KB'.format(bin_size // 1024)) + logging.info(f'https_x509_bundle_bin_size : {bin_size // 1024}KB') dut.expect(r'IPv4 address: (\d+\.\d+\.\d+\.\d+)[^\d]', timeout=30) # start test num_URLS = int(dut.expect(r'Connecting to (\d+) URLs', timeout=30)[1].decode()) dut.expect(r'Connection established to ([\s\S]*)', timeout=30) - dut.expect('Completed {} connections'.format(num_URLS), timeout=60) + dut.expect(f'Completed {num_URLS} connections', timeout=60) @pytest.mark.qemu -@pytest.mark.host_test @pytest.mark.parametrize( 'config', [ @@ -65,10 +64,10 @@ def test_examples_protocol_https_x509_bundle_default_crt_bundle_stress_test(dut: # check and log bin size binary_file = os.path.join(dut.app.binary_path, 'https_x509_bundle.bin') bin_size = os.path.getsize(binary_file) - logging.info('https_x509_bundle_bin_size : {}KB'.format(bin_size // 1024)) + logging.info(f'https_x509_bundle_bin_size : {bin_size // 1024}KB') dut.expect(r'IPv4 address: (\d+\.\d+\.\d+\.\d+)[^\d]', timeout=30) # start test num_URLS = int(dut.expect(r'Connecting to (\d+) URLs', timeout=30)[1].decode()) for _ in range(num_URLS): dut.expect(r'Connection established to ([\s\S]*)', timeout=30) - dut.expect('Completed {} connections'.format(num_URLS), timeout=60) + dut.expect(f'Completed {num_URLS} connections', timeout=60) diff --git a/examples/system/efuse/pytest_system_efuse_example.py b/examples/system/efuse/pytest_system_efuse_example.py index ab281d2d58..60d327ac1d 100644 --- a/examples/system/efuse/pytest_system_efuse_example.py +++ b/examples/system/efuse/pytest_system_efuse_example.py @@ -911,7 +911,6 @@ def test_examples_efuse_with_virt_sb_v2_and_fe(dut: Dut) -> None: dut.expect('example: Done') -@pytest.mark.host_test @pytest.mark.qemu @pytest.mark.parametrize( 'config', diff --git a/examples/system/ota/advanced_https_ota/pytest_advanced_ota.py b/examples/system/ota/advanced_https_ota/pytest_advanced_ota.py index 7a3d678235..a16327b102 100644 --- a/examples/system/ota/advanced_https_ota/pytest_advanced_ota.py +++ b/examples/system/ota/advanced_https_ota/pytest_advanced_ota.py @@ -1096,7 +1096,6 @@ def test_examples_protocol_advanced_https_ota_example_openssl_aligned_bin(dut: D @pytest.mark.qemu @pytest.mark.nightly_run -@pytest.mark.host_test @pytest.mark.parametrize( 'qemu_extra_args', [ @@ -1152,7 +1151,6 @@ def test_examples_protocol_advanced_https_ota_example_verify_min_chip_revision(d @pytest.mark.qemu @pytest.mark.nightly_run -@pytest.mark.host_test @pytest.mark.parametrize( 'qemu_extra_args', [ diff --git a/tools/ci/idf_pytest/plugin.py b/tools/ci/idf_pytest/plugin.py index 2d4bd17c88..b7e11541a6 100644 --- a/tools/ci/idf_pytest/plugin.py +++ b/tools/ci/idf_pytest/plugin.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2023-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import os import typing as t @@ -9,6 +9,7 @@ import pytest import yaml from _pytest.config import Config from _pytest.python import Function +from _pytest.python import Metafunc from _pytest.runner import CallInfo from dynamic_pipelines.constants import KNOWN_GENERATE_TEST_CHILD_PIPELINE_WARNINGS_FILEPATH from idf_ci import IdfPytestPlugin @@ -114,6 +115,40 @@ class IdfLocalPlugin: return item.callspec.params.get(key, default) or default + @staticmethod + def _has_parametrized_arg(metafunc: Metafunc, arg_name: str) -> bool: + for marker in metafunc.definition.iter_markers(name='parametrize'): + if not marker.args: + continue + + argnames = marker.args[0] + if isinstance(argnames, str): + names = [name.strip() for name in argnames.split(',')] + else: + names = list(argnames) + + if arg_name in names: + return True + + for callspec in getattr(metafunc, '_calls', []): + if arg_name in callspec.params: + return True + + return False + + @pytest.hookimpl(trylast=True) + def pytest_generate_tests(self, metafunc: Metafunc) -> None: + if 'embedded_services' not in metafunc.fixturenames: + return + + if metafunc.definition.get_closest_marker('qemu') is None: + return + + if self._has_parametrized_arg(metafunc, 'embedded_services'): + return + + metafunc.parametrize('embedded_services', ['idf,qemu'], indirect=True) + @pytest.hookimpl(wrapper=True) def pytest_collection_modifyitems(self, config: Config, items: list[Function]) -> t.Generator[None, None, None]: yield # throw it back to idf-ci diff --git a/tools/test_apps/security/secure_boot/pytest_secure_boot.py b/tools/test_apps/security/secure_boot/pytest_secure_boot.py index 36b5ce17f9..51f8ef1ace 100644 --- a/tools/test_apps/security/secure_boot/pytest_secure_boot.py +++ b/tools/test_apps/security/secure_boot/pytest_secure_boot.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Unlicense OR CC0-1.0 import itertools import os @@ -105,7 +105,7 @@ def corrupt_sig_block(sig_block, seed=0, corrupt_sig=True, corrupt_crc=False, si data = sig_block[:149] new_sig = sig = sig_block[149:245] else: - raise ValueError('Invalid signature type: {}'.format(signature_type)) + raise ValueError(f'Invalid signature type: {signature_type}') crc = sig_block[1196:1200] padding = sig_block[1200:1216] @@ -165,7 +165,6 @@ def test_examples_security_secure_boot_ecdsa(dut: Dut) -> None: # Test secure boot flow. # Correctly signed bootloader + correctly signed app should work -@pytest.mark.host_test @pytest.mark.qemu @pytest.mark.parametrize( 'qemu_extra_args', @@ -242,7 +241,7 @@ def _examples_security_secure_boot_key_revoke(dut: Dut) -> None: dut.serial.reset_efuses() dut.burn_wafer_version() dut.secure_boot_burn_en_bit() - dut.serial.burn_efuse('SECURE_BOOT_KEY_REVOKE%d' % index, 1) + dut.serial.burn_efuse(f'SECURE_BOOT_KEY_REVOKE{index}', 1) dut.secure_boot_burn_digest(secure_boot_key, index, 0) dut.expect('secure boot verification failed', timeout=5) dut.serial.reset_efuses() @@ -277,9 +276,9 @@ def get_signature_type_size(dut: Dut, signature_type: int) -> int: elif dut.app.sdkconfig.get('CONFIG_SECURE_BOOT_ECDSA_KEY_LEN_384_BITS'): signature_type_size = SIGNATURE_TYPE_ECDSA_P384_SIZE else: - raise ValueError('Invalid signature type: {}'.format(signature_type)) + raise ValueError(f'Invalid signature type: {signature_type}') else: - raise ValueError('Invalid signature type: {}'.format(signature_type)) + raise ValueError(f'Invalid signature type: {signature_type}') return signature_type_size @@ -297,7 +296,7 @@ def _examples_security_secure_boot_corrupt_bl_sig(dut: Dut, signature_type: int) secure_boot_key = dut.app.sdkconfig.get('SECURE_BOOT_SIGNING_KEY') for seed in seeds: - print('Case %d / %d' % (seed, max_seed)) + print(f'Case {seed} / {max_seed}') corrupt_bl = corrupt_signature(signed_bl, seed=seed) with open('corrupt_bl.bin', 'wb') as corrupt_file: corrupt_file.write(corrupt_bl) @@ -349,7 +348,7 @@ def _examples_security_secure_boot_corrupt_app_sig(dut: Dut, signature_type: int max_seed = max(seeds) for seed in seeds: - print('Case %d / %d' % (seed, max_seed)) + print(f'Case {seed} / {max_seed}') corrupt_app = corrupt_signature(signed_app, seed=seed) with open('corrupt_app.bin', 'wb') as corrupt_file: corrupt_file.write(corrupt_app) diff --git a/tools/test_apps/security/signed_app_no_secure_boot/pytest_signed_app_no_secure_boot.py b/tools/test_apps/security/signed_app_no_secure_boot/pytest_signed_app_no_secure_boot.py index 9dde2e391a..101f971a65 100644 --- a/tools/test_apps/security/signed_app_no_secure_boot/pytest_signed_app_no_secure_boot.py +++ b/tools/test_apps/security/signed_app_no_secure_boot/pytest_signed_app_no_secure_boot.py @@ -6,7 +6,6 @@ from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -@pytest.mark.host_test @pytest.mark.qemu @pytest.mark.parametrize('config', ['secure_update_with_fe'], indirect=True) @idf_parametrize('target', ['esp32c3'], indirect=['target']) diff --git a/tools/test_apps/storage/std_filesystem/pytest_std_filesystem.py b/tools/test_apps/storage/std_filesystem/pytest_std_filesystem.py index 7567611604..86031bc508 100644 --- a/tools/test_apps/storage/std_filesystem/pytest_std_filesystem.py +++ b/tools/test_apps/storage/std_filesystem/pytest_std_filesystem.py @@ -6,7 +6,6 @@ from pytest_embedded_idf.utils import idf_parametrize @pytest.mark.qemu -@pytest.mark.host_test @idf_parametrize('target', ['esp32', 'esp32c3'], indirect=['target']) def test_std_filesystem(dut: Dut) -> None: dut.expect_exact('All tests passed', timeout=200) diff --git a/tools/test_apps/system/esp_intr_dump/pytest_esp_intr_dump.py b/tools/test_apps/system/esp_intr_dump/pytest_esp_intr_dump.py index cccfdbedeb..e03da57cc2 100644 --- a/tools/test_apps/system/esp_intr_dump/pytest_esp_intr_dump.py +++ b/tools/test_apps/system/esp_intr_dump/pytest_esp_intr_dump.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2023-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: CC0-1.0 import os @@ -10,7 +10,6 @@ PROMPT = 'test_intr_dump>' @pytest.mark.qemu -@pytest.mark.host_test @idf_parametrize('target', ['esp32'], indirect=['target']) def test_esp_intr_dump_nonshared(dut: Dut) -> None: dut.expect_exact(PROMPT, timeout=30) @@ -24,7 +23,6 @@ def test_esp_intr_dump_nonshared(dut: Dut) -> None: @pytest.mark.qemu -@pytest.mark.host_test @idf_parametrize('target', ['esp32'], indirect=['target']) def test_esp_intr_dump_shared(dut: Dut) -> None: dut.expect_exact(PROMPT, timeout=30) @@ -54,5 +52,5 @@ def test_esp_intr_dump_expected_output(dut: Dut) -> None: dut.expect_exact(PROMPT, timeout=30) dut.write('intr_dump\n') exp_out_file = os.path.join(os.path.dirname(__file__), 'expected_output', f'{dut.target}.txt') - for line in open(exp_out_file, 'r').readlines(): + for line in open(exp_out_file).readlines(): dut.expect_exact(line.strip()) From 036bb8ec773278fcc167a4c117cf11c65ef424db Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Thu, 26 Mar 2026 10:40:40 +0100 Subject: [PATCH 3/3] ci: move linux test cli args alongside the test scripts --- .gitlab/ci/host-test.yml | 2 +- .../test_apps/console/pytest_console.py | 6 +-- .../contribute/esp-idf-tests-with-pytest.rst | 48 +++++++++++++++---- .../contribute/esp-idf-tests-with-pytest.rst | 48 +++++++++++++++---- tools/ci/idf_pytest/plugin.py | 23 +++++++-- 5 files changed, 98 insertions(+), 29 deletions(-) diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index 77da25bb07..fc4d2a1f1f 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -396,7 +396,7 @@ test_pytest_linux: - run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME} - run_cmd pytest --target linux - --embedded-services idf + -m \"not macos\" --junitxml=XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} diff --git a/components/console/test_apps/console/pytest_console.py b/components/console/test_apps/console/pytest_console.py index 8c15097164..084772e388 100644 --- a/components/console/test_apps/console/pytest_console.py +++ b/components/console/test_apps/console/pytest_console.py @@ -127,15 +127,13 @@ def test_console_help_reverse_registration(dut: Dut) -> None: @idf_parametrize('config', ['sorted'], indirect=['config']) @idf_parametrize('target', ['linux'], indirect=['target']) -@idf_parametrize('test_on,markers', [('host', (pytest.mark.host_test,))]) -def test_console_sorted_help_sorted_registration(dut: Dut, test_on: str) -> None: +def test_console_sorted_help_sorted_registration(dut: Dut) -> None: do_test_help_generic(dut, 'sorted') @idf_parametrize('config', ['sorted'], indirect=['config']) @idf_parametrize('target', ['linux'], indirect=['target']) -@idf_parametrize('test_on,markers', [('host', (pytest.mark.host_test,))]) -def test_console_sorted_help_reverse_registration(dut: Dut, test_on: str) -> None: +def test_console_sorted_help_reverse_registration(dut: Dut) -> None: do_test_help_generic(dut, 'reverse') diff --git a/docs/en/contribute/esp-idf-tests-with-pytest.rst b/docs/en/contribute/esp-idf-tests-with-pytest.rst index bf1fe59bf9..315920a5bd 100644 --- a/docs/en/contribute/esp-idf-tests-with-pytest.rst +++ b/docs/en/contribute/esp-idf-tests-with-pytest.rst @@ -120,25 +120,51 @@ Next is the environment marker. The ``@pytest.mark.generic`` marker indicates th Finally, we have the test function. With a ``dut`` fixture. In single-dut test cases, the ``dut`` fixture is an instance of ``IdfDut`` class, for multi-dut test cases, it is a tuple of ``IdfDut`` instances. For more details regarding the ``IdfDut`` class, please refer to `pytest-embedded IdfDut API reference `__. -Running Tests in QEMU -^^^^^^^^^^^^^^^^^^^^^ +Running Tests on Linux +^^^^^^^^^^^^^^^^^^^^^^ -To execute a pytest case in QEMU, add the ``@pytest.mark.qemu`` marker to the test function. +To execute a pytest case on the Linux host, set ``target`` to ``linux``. .. code-block:: python - @pytest.mark.qemu - @idf_parametrize('target', ['esp32', 'esp32c3'], indirect=['target']) - def test_hello_world_qemu(dut) -> None: + @idf_parametrize('target', ['linux'], indirect=['target']) + def test_hello_world_linux(dut) -> None: dut.expect('Hello world!') -This is the simplest way to run the same test flow in QEMU instead of on physical hardware. +This is the simplest way to run the same test flow on the Linux host instead of on physical hardware. -For a simple QEMU-only test, adding ``pytest.mark.qemu`` is enough and the ``idf,qemu`` embedded services will be selected automatically. +For a simple Linux-only test, setting ``target`` to ``linux`` is enough and the ``idf`` embedded services will be selected automatically. The ``pytest.mark.host_test`` marker is not required. -For a mixed environment matrix, specify ``embedded_services`` manually for each case. See the later section :ref:`same-app-with-different-running-environments` for a more complex example. +For a mixed environment matrix, specify ``embedded_services`` manually for each case. See :ref:`Same App With Different Running Environments ` for a more complex example. -For QEMU installation and setup, refer to :doc:`../api-guides/tools/qemu`. +.. only:: TARGET_SUPPORT_QEMU + + Running Tests in QEMU + ^^^^^^^^^^^^^^^^^^^^^ + + To execute a pytest case in QEMU, add the ``@pytest.mark.qemu`` marker to the test function. + + .. code-block:: python + + @pytest.mark.qemu + @idf_parametrize('target', ['esp32', 'esp32c3'], indirect=['target']) + def test_hello_world_qemu(dut) -> None: + dut.expect('Hello world!') + + This is the simplest way to run the same test flow in QEMU instead of on physical hardware. + + For a simple QEMU-only test, adding ``pytest.mark.qemu`` is enough and the ``idf,qemu`` embedded services will be selected automatically. + + For a mixed environment matrix, specify ``embedded_services`` manually for each case. See the later section in this guide for a more complex example. + + For QEMU installation and setup, refer to page :doc:`/api-guides/tools/qemu`. + +Deprecation of ``pytest.mark.host_test`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``pytest.mark.host_test`` is no longer needed and should not be added to new test cases. + +For Linux target test cases and QEMU test cases, the required behavior is handled dynamically by the test framework. In particular, the embedded services are selected automatically for simple Linux-only and QEMU-only cases. Same App With Different sdkconfig Files ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -223,6 +249,8 @@ Now this test function would be replicated to 2 test cases (represented as test * ``esp32.foo.test_foo_bar`` * ``esp32s2.bar.test_foo_bar`` +.. _pytest-same-app-different-running-environments: + Same App With Different Running Environments ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/zh_CN/contribute/esp-idf-tests-with-pytest.rst b/docs/zh_CN/contribute/esp-idf-tests-with-pytest.rst index 0b19324321..2767c116fe 100644 --- a/docs/zh_CN/contribute/esp-idf-tests-with-pytest.rst +++ b/docs/zh_CN/contribute/esp-idf-tests-with-pytest.rst @@ -120,25 +120,51 @@ ESP-IDF 在主机端使用 pytest 框架（以及一些 pytest 插件）来自 关于测试函数，使用了一个 ``dut`` fixture。在单一 DUT 测试用例中，``dut`` fixture 是 ``IdfDut`` 类的一个实例，对于多个 DUT 测试用例，它是 ``IdfDut`` 实例的一个元组。有关 ``IdfDut`` 类的更多详细信息，请参阅 `pytest-embedded IdfDut API 参考 `__。 -在 QEMU 中运行测试 -^^^^^^^^^^^^^^^^^^^^^ +在 Linux 上运行测试 +^^^^^^^^^^^^^^^^^^^^ -要在 QEMU 中执行 pytest 测试用例，请将 ``@pytest.mark.qemu`` 添加到测试函数上。 +要在 Linux 主机上执行 pytest 测试用例，请将 ``target`` 设置为 ``linux``。 .. code-block:: python - @pytest.mark.qemu - @idf_parametrize('target', ['esp32', 'esp32c3'], indirect=['target']) - def test_hello_world_qemu(dut) -> None: + @idf_parametrize('target', ['linux'], indirect=['target']) + def test_hello_world_linux(dut) -> None: dut.expect('Hello world!') -这是在 QEMU 中运行与物理硬件相同测试流程的最简单方式。 +这是在 Linux 主机上运行与物理硬件相同测试流程的最简单方式。 -对于简单的纯 QEMU 测试，只需添加 ``pytest.mark.qemu``，系统会自动选择 ``idf,qemu`` 对应的 embedded services。 +对于简单的纯 Linux 测试，只需将 ``target`` 设置为 ``linux``，系统会自动选择 ``idf`` 对应的 embedded services。``pytest.mark.host_test`` marker 不再需要。 -对于混合运行环境矩阵，则需要为每个用例手动指定 ``embedded_services``。更复杂的示例请参阅本指南后面的 :ref:`same-app-with-different-running-environments` 小节。 +对于混合运行环境矩阵，则需要为每个用例手动指定 ``embedded_services``。更复杂的示例请参阅 :ref:`在不同运行环境中运行相同的应用程序 ` 小节。 -有关 QEMU 的安装和配置，请参阅 :doc:`../api-guides/tools/qemu`。 +.. only:: TARGET_SUPPORT_QEMU + + 在 QEMU 中运行测试 + ^^^^^^^^^^^^^^^^^^^^^ + + 要在 QEMU 中执行 pytest 测试用例，请将 ``@pytest.mark.qemu`` 添加到测试函数上。 + + .. code-block:: python + + @pytest.mark.qemu + @idf_parametrize('target', ['esp32', 'esp32c3'], indirect=['target']) + def test_hello_world_qemu(dut) -> None: + dut.expect('Hello world!') + + 这是在 QEMU 中运行与物理硬件相同测试流程的最简单方式。 + + 对于简单的纯 QEMU 测试，只需添加 ``pytest.mark.qemu``，系统会自动选择 ``idf,qemu`` 对应的 embedded services。 + + 对于混合运行环境矩阵，则需要为每个用例手动指定 ``embedded_services``。更复杂的示例请参阅本指南后面的对应小节。 + + 有关 QEMU 的安装和配置，请参阅页面 :doc:`../api-guides/tools/qemu`。 + +``pytest.mark.host_test`` 的弃用说明 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``pytest.mark.host_test`` 已不再需要，也不应再添加到新的测试用例中。 + +对于 Linux target 测试用例和 QEMU 测试用例，相关行为会由测试框架动态处理。尤其是在简单的纯 Linux 或纯 QEMU 场景下，embedded services 会被自动选择。 使用不同的 sdkconfig 文件运行相同的应用程序 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -223,6 +249,8 @@ ESP-IDF 在主机端使用 pytest 框架（以及一些 pytest 插件）来自 * ``esp32.foo.test_foo_bar`` * ``esp32s2.bar.test_foo_bar`` +.. _pytest-same-app-different-running-environments: + 在不同运行环境中运行相同的应用程序 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/tools/ci/idf_pytest/plugin.py b/tools/ci/idf_pytest/plugin.py index b7e11541a6..2d30aa7341 100644 --- a/tools/ci/idf_pytest/plugin.py +++ b/tools/ci/idf_pytest/plugin.py @@ -136,18 +136,33 @@ class IdfLocalPlugin: return False + @staticmethod + def _is_linux_target_run(config: Config) -> bool: + target = config.getoption('target') + if not target: + return False + + if isinstance(target, str): + targets = [_t.strip() for _t in target.split(',')] + else: + targets = [str(_t).strip() for _t in target] + + return 'linux' in targets + @pytest.hookimpl(trylast=True) def pytest_generate_tests(self, metafunc: Metafunc) -> None: if 'embedded_services' not in metafunc.fixturenames: return - if metafunc.definition.get_closest_marker('qemu') is None: - return - if self._has_parametrized_arg(metafunc, 'embedded_services'): return - metafunc.parametrize('embedded_services', ['idf,qemu'], indirect=True) + if metafunc.definition.get_closest_marker('qemu') is not None: + metafunc.parametrize('embedded_services', ['idf,qemu'], indirect=True) + return + + if self._is_linux_target_run(metafunc.config): + metafunc.parametrize('embedded_services', ['idf'], indirect=True) @pytest.hookimpl(wrapper=True) def pytest_collection_modifyitems(self, config: Config, items: list[Function]) -> t.Generator[None, None, None]: