diff --git a/.gitlab/ci/build.yml b/.gitlab/ci/build.yml index 87a47e0b87..147b421a58 100644 --- a/.gitlab/ci/build.yml +++ b/.gitlab/ci/build.yml @@ -370,6 +370,6 @@ generate_disabled_apps_report: expire_in: 1 week when: always script: - - pip install dominate idf-build-apps>=2.12.0 + - pip install dominate idf-build-apps - run_cmd python tools/ci/gen_disabled_report.py --output disabled_report.html --verbose --enable-preview-targets - echo "Report generated at https://${CI_PAGES_HOSTNAME}:${CI_SERVER_PORT}/-/esp-idf/-/jobs/${CI_JOB_ID}/artifacts/disabled_report.html" diff --git a/.gitlab/ci/common.yml b/.gitlab/ci/common.yml index 0b3717a95a..8f54ebad30 100644 --- a/.gitlab/ci/common.yml +++ b/.gitlab/ci/common.yml @@ -97,6 +97,7 @@ variables: CCACHE_MAXSIZE: "50G" FF_USE_NEW_BASH_EVAL_STRATEGY: "true" + FORCE_COLOR: "1" # rich print with color ################################################ # `before_script` and `after_script` Templates # diff --git a/.gitlab/ci/default-build-test-rules.yml b/.gitlab/ci/default-build-test-rules.yml index 11a291dd3c..62b2bf09d0 100644 --- a/.gitlab/ci/default-build-test-rules.yml +++ b/.gitlab/ci/default-build-test-rules.yml @@ -2,15 +2,12 @@ # - extra_default_build_targets: # besides of the SUPPORTED_TARGETS in IDF, # enable build for the specified targets by default as well. +# !!! DEPRECATED: use `additional_build_targets` in .idf_build_apps.toml instead +# # - bypass_check_test_targets: # suppress the check_build_test_rules check-test-script warnings for the specified targets # # This file should ONLY be used during bringup. Should be reset to empty after the bringup process -extra_default_build_targets: - - esp32c5 - - esp32c61 - - esp32h21 - - esp32h4 bypass_check_test_targets: - esp32h21 diff --git a/.idf_build_apps.toml b/.idf_build_apps.toml index 07ea248d34..cbe3170bf0 100644 --- a/.idf_build_apps.toml +++ b/.idf_build_apps.toml @@ -25,6 +25,11 @@ size_json_filename = "size_${CI_JOB_ID}.json" verbose = 1 # INFO +additional_build_targets = [ + 'esp32h21', + 'esp32h4', +] + # collect collect_app_info_filename = "app_info_${CI_JOB_NAME_SLUG}.txt" junitxml = "build_summary_${CI_JOB_NAME_SLUG}.xml" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bab9afc3ed..15201b2b05 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -144,7 +144,7 @@ repos: require_serial: true additional_dependencies: - PyYAML == 5.3.1 - - idf-build-apps~=2.12 + - idf-build-apps~=2.13 - id: sort-yaml-files name: sort yaml files entry: tools/ci/sort_yaml.py diff --git a/conftest.py b/conftest.py index aa19094c2a..13a04ddda0 100644 --- a/conftest.py +++ b/conftest.py @@ -91,11 +91,11 @@ def test_case_name(request: FixtureRequest, target: str, config: str) -> str: @pytest.fixture(scope='session') -def pipeline_id(request: FixtureRequest) -> t.Optional[str]: +def pipeline_id(request: FixtureRequest) -> str | None: return request.config.getoption('pipeline_id', None) or os.getenv('PARENT_PIPELINE_ID', None) # type: ignore -def get_pipeline_commit_sha_by_pipeline_id(pipeline_id: str) -> t.Optional[str]: +def get_pipeline_commit_sha_by_pipeline_id(pipeline_id: str) -> str | None: gl = gitlab_api.Gitlab(os.getenv('CI_PROJECT_ID', 'espressif/esp-idf')) pipeline = gl.project.pipelines.get(pipeline_id) if not pipeline: @@ -120,12 +120,12 @@ class AppDownloader: def __init__( self, commit_sha: str, - pipeline_id: t.Optional[str] = None, + pipeline_id: str | None = None, ) -> None: self.commit_sha = commit_sha self.pipeline_id = pipeline_id - def download_app(self, app_build_path: str, artifact_type: t.Optional[str] = None) -> None: + def download_app(self, app_build_path: str, artifact_type: str | None = None) -> None: args = [ 'idf-ci', 'gitlab', @@ -155,9 +155,9 @@ class OpenOCD: self.RETRY_DELAY = 1 self.TELNET_PORT = 4444 self.dut = dut - self.telnet: t.Optional[Telnet] = None + self.telnet: Telnet | None = None self.log_file = os.path.join(self.dut.logdir, 'ocd.txt') - self.proc: t.Optional[pexpect.spawn] = None + self.proc: pexpect.spawn | None = None def __enter__(self) -> 'OpenOCD': return self @@ -169,7 +169,7 @@ class OpenOCD: desc_path = os.path.join(self.dut.app.binary_path, 'project_description.json') try: - with open(desc_path, 'r') as f: + with open(desc_path) as f: project_desc = json.load(f) except FileNotFoundError: logging.error('Project description file not found at %s', desc_path) @@ -200,7 +200,7 @@ class OpenOCD: if self.proc and self.proc.isalive(): self.proc.expect_exact('Info : Listening on port 3333 for gdb connections', timeout=5) self.connect_telnet() - self.write('log_output {}'.format(self.log_file)) + self.write(f'log_output {self.log_file}') return self except (pexpect.exceptions.EOF, pexpect.exceptions.TIMEOUT, ConnectionRefusedError) as e: logging.error('Error running OpenOCD: %s', str(e)) @@ -258,8 +258,8 @@ def openocd_dut(dut: IdfDut) -> OpenOCD: @pytest.fixture(scope='session') def app_downloader( - pipeline_id: t.Optional[str], -) -> t.Optional[AppDownloader]: + pipeline_id: str | None, +) -> AppDownloader | None: if commit_sha := os.getenv('PIPELINE_COMMIT_SHA'): logging.debug('pipeline commit sha from CI env is %s', commit_sha) return AppDownloader(commit_sha, None) @@ -283,9 +283,9 @@ def app_downloader( def build_dir( request: FixtureRequest, app_path: str, - target: t.Optional[str], - config: t.Optional[str], - app_downloader: t.Optional[AppDownloader], + target: str | None, + config: str | None, + app_downloader: AppDownloader | None, ) -> str: """ Check local build dir with the following priority: @@ -561,10 +561,10 @@ def pytest_runtest_makereport(item, call): # type: ignore if isinstance(_dut, list): logs_files.extend([template.format(get_path(d.logfile)) for d in _dut]) - dut_artifacts_url.append('{}:'.format(_dut[0].test_case_name)) + dut_artifacts_url.append(f'{_dut[0].test_case_name}:') else: logs_files.append(template.format(get_path(_dut.logfile))) - dut_artifacts_url.append('{}:'.format(_dut.test_case_name)) + dut_artifacts_url.append(f'{_dut.test_case_name}:') for file in logs_files: dut_artifacts_url.append(' - {}'.format(quote(file, safe=':/'))) diff --git a/tools/ci/check_build_test_rules.py b/tools/ci/check_build_test_rules.py index d9fe670007..3ef3b0ddbc 100755 --- a/tools/ci/check_build_test_rules.py +++ b/tools/ci/check_build_test_rules.py @@ -124,14 +124,12 @@ def parse_existing_table(app_dir: str) -> tuple[str | None, list[str]]: def get_grouped_apps( paths: list[str], exclude_dirs: list[str] | None = None, - extra_targets: list[str] | None = None, ) -> dict[str, list[App]]: apps = sorted( find_apps( paths, 'all', exclude_list=exclude_dirs or [], - default_build_targets=SUPPORTED_TARGETS + (extra_targets or []), ) ) @@ -145,10 +143,8 @@ def get_grouped_apps( def check_readme( paths: list[str], exclude_dirs: list[str] | None = None, - *, - extra_targets: list[str] | None = None, ) -> None: - grouped_apps = get_grouped_apps(paths, exclude_dirs, extra_targets) + grouped_apps = get_grouped_apps(paths, exclude_dirs) exit_code = 0 for app_dir, apps in grouped_apps.items(): @@ -243,11 +239,10 @@ def check_test_scripts( paths: list[str], exclude_dirs: list[str] | None = None, *, - extra_targets: list[str] | None = None, bypass_targets: list[str] | None = None, ) -> None: # takes long time, run only in CI - grouped_apps = get_grouped_apps(paths, exclude_dirs, extra_targets) + grouped_apps = get_grouped_apps(paths, exclude_dirs) grouped_cases = get_grouped_cases(paths) exit_code = 0 @@ -347,14 +342,12 @@ if __name__ == '__main__': else: _exclude_dirs = [os.path.join(IDF_PATH, 'tools', 'templates', 'sample_project')] - _extra_targets: list[str] = [] _bypass_targets: list[str] = [] if arg.config: with open(arg.config) as fr: configs = yaml.safe_load(fr) if configs: - _extra_targets = configs.get('extra_default_build_targets') or [] _bypass_targets = configs.get('bypass_check_test_targets') or [] os.environ.update( @@ -369,12 +362,10 @@ if __name__ == '__main__': check_readme( list(check_dirs), _exclude_dirs, - extra_targets=_extra_targets, ) elif arg.action == 'check-test-scripts': check_test_scripts( list(check_dirs), _exclude_dirs, - extra_targets=_extra_targets, bypass_targets=_bypass_targets, ) diff --git a/tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py b/tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py index 3da7ab709c..580d45ecce 100644 --- a/tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py +++ b/tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py @@ -7,8 +7,6 @@ import logging import os import __init__ # noqa: F401 # inject the system path -import yaml -from idf_build_apps.manifest import DEFAULT_BUILD_TARGETS from idf_build_apps.utils import semicolon_separated_str_to_list from idf_ci.idf_gitlab import build_child_pipeline from idf_ci.utils import setup_logging @@ -35,18 +33,6 @@ def _separate_str_to_list(s: str) -> list[str]: def main(arguments: argparse.Namespace) -> None: - # load from default build test rules config file - extra_default_build_targets: list[str] = [] - if arguments.default_build_test_rules: - with open(arguments.default_build_test_rules) as fr: - configs = yaml.safe_load(fr) - - if configs: - extra_default_build_targets = configs.get('extra_default_build_targets') or [] - - if extra_default_build_targets: - DEFAULT_BUILD_TARGETS.set(list(set(DEFAULT_BUILD_TARGETS.get()).union(set(extra_default_build_targets)))) - setup_logging(logging.DEBUG) build_child_pipeline( paths=args.paths, @@ -75,11 +61,6 @@ if __name__ == '__main__': default=TEST_PATHS, help='Paths to the apps to build.', ) - parser.add_argument( - '--default-build-test-rules', - default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml'), - help='default build test rules config file', - ) parser.add_argument( '--compare-manifest-sha-filepath', default=os.path.join(IDF_PATH, '.manifest_sha'),