espressif/esp-idf
1
0
Fork 0
mirror of https://github.com/espressif/esp-idf.git synced 2026-04-27 19:13:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: set sufficient buffer limit for idf.py confserver

Browse Source
This commit is contained in:
Jan Beran
2025-10-03 10:37:12 +02:00
parent a247a4dbe1
commit 05e3c24322
2 changed files with 112 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/idf_py_actions/core_ext.py
+68 -9
View File
@@ -1,4 +1,4 @@
# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
import fnmatch
import json
@@ -8,18 +8,32 @@ import re
import shutil
import subprocess
import sys
from typing import Any, Dict, List, Optional
from typing import Any
from typing import Dict
from typing import List
from typing import Optional
from urllib.error import URLError
from urllib.request import Request, urlopen
from urllib.request import Request
from urllib.request import urlopen
from webbrowser import open_new_tab
import click
from click.core import Context
from idf_py_actions.constants import GENERATORS, PREVIEW_TARGETS, SUPPORTED_TARGETS, URL_TO_DOC
from idf_py_actions.constants import GENERATORS
from idf_py_actions.constants import PREVIEW_TARGETS
from idf_py_actions.constants import SUPPORTED_TARGETS
from idf_py_actions.constants import URL_TO_DOC
from idf_py_actions.errors import FatalError
from idf_py_actions.global_options import global_options
from idf_py_actions.tools import (PropertyDict, TargetChoice, ensure_build_directory, generate_hints, get_target,
idf_version, merge_action_lists, run_target, yellow_print)
from idf_py_actions.tools import ensure_build_directory
from idf_py_actions.tools import generate_hints
from idf_py_actions.tools import get_target
from idf_py_actions.tools import idf_version
from idf_py_actions.tools import merge_action_lists
from idf_py_actions.tools import PropertyDict
from idf_py_actions.tools import run_target
from idf_py_actions.tools import TargetChoice
from idf_py_actions.tools import yellow_print
def action_extensions(base_actions: Dict, project_path: str) -> Any:
@@ -33,7 +47,39 @@ def action_extensions(base_actions: Dict, project_path: str) -> Any:
ensure_build_directory(args, ctx.info_name)
run_target(target_name, args, force_progression=GENERATORS[args.generator].get('force_progression', False))
def size_target(target_name: str, ctx: Context, args: PropertyDict, output_format: str, output_file: str) -> None:
def confserver_target(target_name: str, ctx: Context, args: PropertyDict, buffer_size: int) -> None:
"""
Execute the idf.py confserver command with the specified buffer size.
"""
ensure_build_directory(args, ctx.info_name)
if buffer_size < 2048:
yellow_print(
f'WARNING: The specified buffer size {buffer_size} KB is less than the '
'recommended minimum of 2048 KB for idf.py confserver. Consider increasing it to at least 2048 KB '
'by setting environment variable IDF_CONFSERVER_BUFFER_SIZE=<buffer size in KB> or by calling '
'idf.py confserver --buffer-size <buffer size in KB>.'
)
try:
run_target(
target_name,
args,
force_progression=GENERATORS[args.generator].get('force_progression', False),
buffer_size=buffer_size,
)
except ValueError as e:
if str(e) == 'Separator is not found, and chunk exceed the limit':
# Buffer size too small/one-line output of the command too long
raise FatalError(
f'ERROR: Command failed with an error message "{e}". '
'Try increasing the buffer size to 2048 (or higher) by setting environment variable '
'IDF_CONFSERVER_BUFFER_SIZE=<buffer size in KB> or by calling '
'idf.py confserver --buffer-size <buffer size in KB>.'
)
else:
raise
def size_target(target_name: str, ctx: Context, args: PropertyDict, output_format: str,
output_file: str, legacy: bool) -> None:
"""
Builds the app and then executes a size-related target passed in 'target_name'.
`tool_error_handler` handler is used to suppress errors during the build,
@@ -390,9 +436,22 @@ def action_extensions(base_actions: Dict, project_path: str) -> Any:
],
},
'confserver': {
'callback': build_target,
'callback': confserver_target,
'help': 'Run JSON configuration server.',
'options': global_options,
'options': global_options
+ [
{
'names': ['--buffer-size'],
'help': (
'Set the buffer size (in KB) in order to accommodate initial confserver response.'
'Default value and recommended minimum is 2048 (KB), but it might need to be '
'increased for very large projects.'
),
'type': int,
'default': 2048,
'envvar': 'IDF_CONFSERVER_BUFFER_SIZE',
}
],
},
'size': {
'callback': size_target,
tools/idf_py_actions/tools.py
+44 -10
View File
@@ -1,4 +1,4 @@
# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD
# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
import asyncio
import importlib
@@ -278,9 +278,20 @@ def fit_text_in_terminal(out: str) -> str:
class RunTool:
def __init__(self, tool_name: str, args: List, cwd: str, env: Optional[Dict]=None, custom_error_handler: Optional[FunctionType]=None,
build_dir: Optional[str]=None, hints: bool=True, force_progression: bool=False, interactive: bool=False, convert_output: bool=False
) -> None:
def __init__(
self,
tool_name: str,
args: list,
cwd: str,
env: Optional[Dict] = None,
custom_error_handler: Optional[FunctionType] = None,
build_dir: Optional[str] = None,
hints: bool = True,
force_progression: bool = False,
interactive: bool = False,
convert_output: bool = False,
buffer_size: Optional[int] = None,
) -> None:
self.tool_name = tool_name
self.args = args
self.cwd = cwd
@@ -292,6 +303,7 @@ class RunTool:
self.force_progression = force_progression
self.interactive = interactive
self.convert_output = convert_output
self.buffer_size = buffer_size or 256
def __call__(self) -> None:
def quote_arg(arg: str) -> str:
@@ -342,8 +354,14 @@ class RunTool:
# Note: we explicitly pass in os.environ here, as we may have set IDF_PATH there during startup
# limit was added for avoiding error in idf.py confserver
try:
p = await asyncio.create_subprocess_exec(*cmd, env=env_copy, limit=1024 * 256, cwd=self.cwd, stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE)
p = await asyncio.create_subprocess_exec(
*cmd,
env=env_copy,
limit=1024 * self.buffer_size,
cwd=self.cwd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
except NotImplementedError:
message = f'ERROR: {sys.executable} doesn\'t support asyncio. The issue can be worked around by re-running idf.py with the "--no-hints" argument.'
if sys.platform == 'win32':
@@ -461,8 +479,15 @@ def run_tool(*args: Any, **kwargs: Any) -> None:
return RunTool(*args, **kwargs)()
def run_target(target_name: str, args: 'PropertyDict', env: Optional[Dict]=None,
custom_error_handler: Optional[FunctionType]=None, force_progression: bool=False, interactive: bool=False) -> None:
def run_target(
target_name: str,
args: 'PropertyDict',
env: Optional[Dict] = None,
custom_error_handler: Optional[FunctionType] = None,
force_progression: bool = False,
interactive: bool = False,
buffer_size: Optional[int] = None,
) -> None:
"""Run target in build directory."""
if env is None:
env = {}
@@ -479,8 +504,17 @@ def run_target(target_name: str, args: 'PropertyDict', env: Optional[Dict]=None,
if 'CLICOLOR_FORCE' not in env:
env['CLICOLOR_FORCE'] = '1'
RunTool(generator_cmd[0], generator_cmd + [target_name], args.build_dir, env, custom_error_handler, hints=not args.no_hints,
force_progression=force_progression, interactive=interactive)()
RunTool(
generator_cmd[0],
generator_cmd + [target_name],
args.build_dir,
env,
custom_error_handler,
hints=not args.no_hints,
force_progression=force_progression,
interactive=interactive,
buffer_size=buffer_size,
)()
def _strip_quotes(value: str, regexp: re.Pattern=re.compile(r"^\"(.*)\"$|^'(.*)'$|^(.*)$")) -> Optional[str]: