fix: set sufficient buffer limit for idf.py confserver

tools/idf_py_actions/core_ext.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 import fnmatch
 import glob
@@ -49,6 +49,37 @@ def action_extensions(base_actions: Dict, project_path: str) -> Any:
         ensure_build_directory(args, ctx.info_name)
         run_target(target_name, args, force_progression=GENERATORS[args.generator].get('force_progression', False))
 
+    def confserver_target(target_name: str, ctx: Context, args: PropertyDict, buffer_size: int) -> None:
+        """
+        Execute the idf.py confserver command with the specified buffer size.
+        """
+        ensure_build_directory(args, ctx.info_name)
+        if buffer_size < 2048:
+            yellow_print(
+                f'WARNING: The specified buffer size {buffer_size} KB is less than the '
+                'recommended minimum of 2048 KB for idf.py confserver. Consider increasing it to at least 2048 KB '
+                'by setting environment variable IDF_CONFSERVER_BUFFER_SIZE=<buffer size in KB> or by calling '
+                'idf.py confserver --buffer-size <buffer size in KB>.'
+            )
+        try:
+            run_target(
+                target_name,
+                args,
+                force_progression=GENERATORS[args.generator].get('force_progression', False),
+                buffer_size=buffer_size,
+            )
+        except ValueError as e:
+            if str(e) == 'Separator is not found, and chunk exceed the limit':
+                # Buffer size too small/one-line output of the command too long
+                raise FatalError(
+                    f'ERROR: Command failed with an error message "{e}". '
+                    'Try increasing the buffer size to 2048 (or higher) by setting environment variable '
+                    'IDF_CONFSERVER_BUFFER_SIZE=<buffer size in KB> or by calling '
+                    'idf.py confserver --buffer-size <buffer size in KB>.'
+                )
+            else:
+                raise
+
     def size_target(target_name: str, ctx: Context, args: PropertyDict, output_format: str,
                     output_file: str, diff_map_file: str, legacy: bool) -> None:
         """
@@ -467,9 +498,22 @@ def action_extensions(base_actions: Dict, project_path: str) -> Any:
                 ],
             },
             'confserver': {
-                'callback': build_target,
+                'callback': confserver_target,
                 'help': 'Run JSON configuration server.',
-                'options': global_options,
+                'options': global_options
+                + [
+                    {
+                        'names': ['--buffer-size'],
+                        'help': (
+                            'Set the buffer size (in KB) in order to accommodate initial confserver response.'
+                            'Default value and recommended minimum is 2048 (KB), but it might need to be '
+                            'increased for very large projects.'
+                        ),
+                        'type': int,
+                        'default': 2048,
+                        'envvar': 'IDF_CONFSERVER_BUFFER_SIZE',
+                    }
+                ],
             },
             'size': {
                 'callback': size_target,

tools/idf_py_actions/tools.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 import asyncio
 import importlib
@@ -284,9 +284,20 @@ def fit_text_in_terminal(out: str) -> str:
 
 
 class RunTool:
-    def __init__(self, tool_name: str, args: List, cwd: str, env: Optional[Dict]=None, custom_error_handler: Optional[FunctionType]=None,
-                 build_dir: Optional[str]=None, hints: bool=True, force_progression: bool=False, interactive: bool=False, convert_output: bool=False
-                 ) -> None:
+    def __init__(
+        self,
+        tool_name: str,
+        args: list,
+        cwd: str,
+        env: Optional[Dict] = None,
+        custom_error_handler: Optional[FunctionType] = None,
+        build_dir: Optional[str] = None,
+        hints: bool = True,
+        force_progression: bool = False,
+        interactive: bool = False,
+        convert_output: bool = False,
+        buffer_size: Optional[int] = None,
+    ) -> None:
         self.tool_name = tool_name
         self.args = args
         self.cwd = cwd
@@ -298,6 +309,7 @@ class RunTool:
         self.force_progression = force_progression
         self.interactive = interactive
         self.convert_output = convert_output
+        self.buffer_size = buffer_size or 256
 
     def __call__(self) -> None:
         def quote_arg(arg: str) -> str:
@@ -348,8 +360,14 @@ class RunTool:
         # Note: we explicitly pass in os.environ here, as we may have set IDF_PATH there during startup
         # limit was added for avoiding error in idf.py confserver
         try:
-            p = await asyncio.create_subprocess_exec(*cmd, env=env_copy, limit=1024 * 256, cwd=self.cwd, stdout=asyncio.subprocess.PIPE,
-                                                     stderr=asyncio.subprocess.PIPE)
+            p = await asyncio.create_subprocess_exec(
+                *cmd,
+                env=env_copy,
+                limit=1024 * self.buffer_size,
+                cwd=self.cwd,
+                stdout=asyncio.subprocess.PIPE,
+                stderr=asyncio.subprocess.PIPE,
+            )
         except NotImplementedError:
             message = f'ERROR: {sys.executable} doesn\'t support asyncio. The issue can be worked around by re-running idf.py with the "--no-hints" argument.'
             if sys.platform == 'win32':
@@ -478,8 +496,15 @@ def run_tool(*args: Any, **kwargs: Any) -> None:
     return RunTool(*args, **kwargs)()
 
 
-def run_target(target_name: str, args: 'PropertyDict', env: Optional[Dict]=None,
-               custom_error_handler: Optional[FunctionType]=None, force_progression: bool=False, interactive: bool=False) -> None:
+def run_target(
+    target_name: str,
+    args: 'PropertyDict',
+    env: Optional[Dict] = None,
+    custom_error_handler: Optional[FunctionType] = None,
+    force_progression: bool = False,
+    interactive: bool = False,
+    buffer_size: Optional[int] = None,
+) -> None:
     """Run target in build directory."""
     if env is None:
         env = {}
@@ -496,8 +521,17 @@ def run_target(target_name: str, args: 'PropertyDict', env: Optional[Dict]=None,
         if 'CLICOLOR_FORCE' not in env:
             env['CLICOLOR_FORCE'] = '1'
 
-    RunTool(generator_cmd[0], generator_cmd + [target_name], args.build_dir, env, custom_error_handler, hints=not args.no_hints,
-            force_progression=force_progression, interactive=interactive)()
+    RunTool(
+        generator_cmd[0],
+        generator_cmd + [target_name],
+        args.build_dir,
+        env,
+        custom_error_handler,
+        hints=not args.no_hints,
+        force_progression=force_progression,
+        interactive=interactive,
+        buffer_size=buffer_size,
+    )()
 
 
 def _strip_quotes(value: str, regexp: re.Pattern=re.compile(r"^\"(.*)\"$|^'(.*)'$|^(.*)$")) -> Optional[str]: