diff --git a/tools/idf_py_actions/core_ext.py b/tools/idf_py_actions/core_ext.py index 3068130d64..80b13db1ea 100644 --- a/tools/idf_py_actions/core_ext.py +++ b/tools/idf_py_actions/core_ext.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import fnmatch import json @@ -8,18 +8,32 @@ import re import shutil import subprocess import sys -from typing import Any, Dict, List, Optional +from typing import Any +from typing import Dict +from typing import List +from typing import Optional from urllib.error import URLError -from urllib.request import Request, urlopen +from urllib.request import Request +from urllib.request import urlopen from webbrowser import open_new_tab import click from click.core import Context -from idf_py_actions.constants import GENERATORS, PREVIEW_TARGETS, SUPPORTED_TARGETS, URL_TO_DOC +from idf_py_actions.constants import GENERATORS +from idf_py_actions.constants import PREVIEW_TARGETS +from idf_py_actions.constants import SUPPORTED_TARGETS +from idf_py_actions.constants import URL_TO_DOC from idf_py_actions.errors import FatalError from idf_py_actions.global_options import global_options -from idf_py_actions.tools import (PropertyDict, TargetChoice, ensure_build_directory, generate_hints, get_target, - idf_version, merge_action_lists, run_target, yellow_print) +from idf_py_actions.tools import ensure_build_directory +from idf_py_actions.tools import generate_hints +from idf_py_actions.tools import get_target +from idf_py_actions.tools import idf_version +from idf_py_actions.tools import merge_action_lists +from idf_py_actions.tools import PropertyDict +from idf_py_actions.tools import run_target +from idf_py_actions.tools import TargetChoice +from idf_py_actions.tools import yellow_print def action_extensions(base_actions: Dict, project_path: str) -> Any: @@ -33,7 +47,39 @@ def action_extensions(base_actions: Dict, project_path: str) -> Any: ensure_build_directory(args, ctx.info_name) run_target(target_name, args, force_progression=GENERATORS[args.generator].get('force_progression', False)) - def size_target(target_name: str, ctx: Context, args: PropertyDict, output_format: str, output_file: str) -> None: + def confserver_target(target_name: str, ctx: Context, args: PropertyDict, buffer_size: int) -> None: + """ + Execute the idf.py confserver command with the specified buffer size. + """ + ensure_build_directory(args, ctx.info_name) + if buffer_size < 2048: + yellow_print( + f'WARNING: The specified buffer size {buffer_size} KB is less than the ' + 'recommended minimum of 2048 KB for idf.py confserver. Consider increasing it to at least 2048 KB ' + 'by setting environment variable IDF_CONFSERVER_BUFFER_SIZE= or by calling ' + 'idf.py confserver --buffer-size .' + ) + try: + run_target( + target_name, + args, + force_progression=GENERATORS[args.generator].get('force_progression', False), + buffer_size=buffer_size, + ) + except ValueError as e: + if str(e) == 'Separator is not found, and chunk exceed the limit': + # Buffer size too small/one-line output of the command too long + raise FatalError( + f'ERROR: Command failed with an error message "{e}". ' + 'Try increasing the buffer size to 2048 (or higher) by setting environment variable ' + 'IDF_CONFSERVER_BUFFER_SIZE= or by calling ' + 'idf.py confserver --buffer-size .' + ) + else: + raise + + def size_target(target_name: str, ctx: Context, args: PropertyDict, output_format: str, + output_file: str, legacy: bool) -> None: """ Builds the app and then executes a size-related target passed in 'target_name'. `tool_error_handler` handler is used to suppress errors during the build, @@ -390,9 +436,22 @@ def action_extensions(base_actions: Dict, project_path: str) -> Any: ], }, 'confserver': { - 'callback': build_target, + 'callback': confserver_target, 'help': 'Run JSON configuration server.', - 'options': global_options, + 'options': global_options + + [ + { + 'names': ['--buffer-size'], + 'help': ( + 'Set the buffer size (in KB) in order to accommodate initial confserver response.' + 'Default value and recommended minimum is 2048 (KB), but it might need to be ' + 'increased for very large projects.' + ), + 'type': int, + 'default': 2048, + 'envvar': 'IDF_CONFSERVER_BUFFER_SIZE', + } + ], }, 'size': { 'callback': size_target, diff --git a/tools/idf_py_actions/tools.py b/tools/idf_py_actions/tools.py index 22c4d83ea7..a7aaf8f275 100644 --- a/tools/idf_py_actions/tools.py +++ b/tools/idf_py_actions/tools.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import asyncio import importlib @@ -278,9 +278,20 @@ def fit_text_in_terminal(out: str) -> str: class RunTool: - def __init__(self, tool_name: str, args: List, cwd: str, env: Optional[Dict]=None, custom_error_handler: Optional[FunctionType]=None, - build_dir: Optional[str]=None, hints: bool=True, force_progression: bool=False, interactive: bool=False, convert_output: bool=False - ) -> None: + def __init__( + self, + tool_name: str, + args: list, + cwd: str, + env: Optional[Dict] = None, + custom_error_handler: Optional[FunctionType] = None, + build_dir: Optional[str] = None, + hints: bool = True, + force_progression: bool = False, + interactive: bool = False, + convert_output: bool = False, + buffer_size: Optional[int] = None, + ) -> None: self.tool_name = tool_name self.args = args self.cwd = cwd @@ -292,6 +303,7 @@ class RunTool: self.force_progression = force_progression self.interactive = interactive self.convert_output = convert_output + self.buffer_size = buffer_size or 256 def __call__(self) -> None: def quote_arg(arg: str) -> str: @@ -342,8 +354,14 @@ class RunTool: # Note: we explicitly pass in os.environ here, as we may have set IDF_PATH there during startup # limit was added for avoiding error in idf.py confserver try: - p = await asyncio.create_subprocess_exec(*cmd, env=env_copy, limit=1024 * 256, cwd=self.cwd, stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE) + p = await asyncio.create_subprocess_exec( + *cmd, + env=env_copy, + limit=1024 * self.buffer_size, + cwd=self.cwd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) except NotImplementedError: message = f'ERROR: {sys.executable} doesn\'t support asyncio. The issue can be worked around by re-running idf.py with the "--no-hints" argument.' if sys.platform == 'win32': @@ -461,8 +479,15 @@ def run_tool(*args: Any, **kwargs: Any) -> None: return RunTool(*args, **kwargs)() -def run_target(target_name: str, args: 'PropertyDict', env: Optional[Dict]=None, - custom_error_handler: Optional[FunctionType]=None, force_progression: bool=False, interactive: bool=False) -> None: +def run_target( + target_name: str, + args: 'PropertyDict', + env: Optional[Dict] = None, + custom_error_handler: Optional[FunctionType] = None, + force_progression: bool = False, + interactive: bool = False, + buffer_size: Optional[int] = None, +) -> None: """Run target in build directory.""" if env is None: env = {} @@ -479,8 +504,17 @@ def run_target(target_name: str, args: 'PropertyDict', env: Optional[Dict]=None, if 'CLICOLOR_FORCE' not in env: env['CLICOLOR_FORCE'] = '1' - RunTool(generator_cmd[0], generator_cmd + [target_name], args.build_dir, env, custom_error_handler, hints=not args.no_hints, - force_progression=force_progression, interactive=interactive)() + RunTool( + generator_cmd[0], + generator_cmd + [target_name], + args.build_dir, + env, + custom_error_handler, + hints=not args.no_hints, + force_progression=force_progression, + interactive=interactive, + buffer_size=buffer_size, + )() def _strip_quotes(value: str, regexp: re.Pattern=re.compile(r"^\"(.*)\"$|^'(.*)'$|^(.*)$")) -> Optional[str]: