Merge branch 'contrib/github_pr_15622' into 'master'

Add support for partial websocket frame payload reads (GitHub PR)

Closes IDFGH-14913

See merge request espressif/esp-idf!44044

components/esp_http_server/include/esp_http_server.h

@@ -1748,6 +1748,8 @@ typedef struct httpd_ws_frame {
     httpd_ws_type_t type;       /*!< WebSocket frame type */
     uint8_t *payload;           /*!< Pre-allocated data buffer */
     size_t len;                 /*!< Length of the WebSocket data */
+    size_t left_len;            /*!< Length of the WebSocket data that is yet to be received.
+                                     This field should not be modified by user. */
 } httpd_ws_frame_t;
 
 /**
@@ -1768,11 +1770,30 @@ typedef void (*transfer_complete_cb)(esp_err_t err, int socket, void *arg);
  * @return
  *  - ESP_OK                    : On successful
  *  - ESP_FAIL                  : Socket errors occurs
+ *  - ESP_ERR_INVALID_SIZE      : max_len is too small to fit the entire payload
  *  - ESP_ERR_INVALID_STATE     : Handshake was already done beforehand
  *  - ESP_ERR_INVALID_ARG       : Argument is invalid (null or non-WebSocket)
  */
 esp_err_t httpd_ws_recv_frame(httpd_req_t *req, httpd_ws_frame_t *pkt, size_t max_len);
 
+/**
+ * @brief Receive and parse a WebSocket frame part
+ *
+ * @note    Calling httpd_ws_recv_frame_part() with max_len as 0 will give actual frame size in pkt->len.
+ *          The user can dynamically allocate space for pkt->payload or user defined chunk size and call httpd_ws_recv_frame_part() again to get the actual data.
+ *          In contrast to httpd_ws_recv_frame, this method is able to read frame payload partially. The amount of data that is yet to be received is stored in pkt->left_len
+ *
+ * @param[in]   req         Current request
+ * @param[out]  pkt         WebSocket packet
+ * @param[in]   max_len     Maximum length for receive
+ * @return
+ *  - ESP_OK                    : On successful
+ *  - ESP_FAIL                  : Socket errors occurs
+ *  - ESP_ERR_INVALID_STATE     : Handshake was already done beforehand
+ *  - ESP_ERR_INVALID_ARG       : Argument is invalid (null or non-WebSocket)
+ */
+esp_err_t httpd_ws_recv_frame_part(httpd_req_t *req, httpd_ws_frame_t *pkt, size_t max_len);
+
 /**
  * @brief Construct and send a WebSocket frame
  * @param[in]   req     Current request

components/esp_http_server/src/httpd_ws.c

@@ -249,7 +249,7 @@ static esp_err_t httpd_ws_check_req(httpd_req_t *req)
     return ESP_OK;
 }
 
-static esp_err_t httpd_ws_unmask_payload(uint8_t *payload, size_t len, const uint8_t *mask_key)
+static esp_err_t httpd_ws_unmask_payload(uint8_t *payload, size_t len, const uint8_t *mask_key, size_t mask_offset)
 {
     if (len < 1 || !payload) {
         ESP_LOGW(TAG, LOG_FMT("Invalid payload provided"));
@@ -257,13 +257,13 @@ static esp_err_t httpd_ws_unmask_payload(uint8_t *payload, size_t len, const uin
     }
 
     for (size_t idx = 0; idx < len; idx++) {
-        payload[idx] = (payload[idx] ^ mask_key[idx % 4]);
+        payload[idx] = (payload[idx] ^ mask_key[(idx + mask_offset) % 4]);
     }
 
     return ESP_OK;
 }
 
-esp_err_t httpd_ws_recv_frame(httpd_req_t *req, httpd_ws_frame_t *frame, size_t max_len)
+static esp_err_t httpd_ws_recv_frame_internal(httpd_req_t *req, httpd_ws_frame_t *frame, size_t max_len, bool partial)
 {
     esp_err_t ret = httpd_ws_check_req(req);
     if (ret != ESP_OK) {
@@ -328,6 +328,8 @@ esp_err_t httpd_ws_recv_frame(httpd_req_t *req, httpd_ws_frame_t *frame, size_t
                     ((uint64_t)length_bytes[6] <<  8U) |
                     ((uint64_t)length_bytes[7]));
         }
+        frame->left_len = frame->len;
+
         /* If this frame is masked, dump the mask as well */
         if (masked) {
             if (httpd_recv_with_opt(req, (char *)aux->mask_key, sizeof(aux->mask_key), HTTPD_RECV_OPT_BLOCKING) < sizeof(aux->mask_key)) {
@@ -341,20 +343,23 @@ esp_err_t httpd_ws_recv_frame(httpd_req_t *req, httpd_ws_frame_t *frame, size_t
             return ESP_ERR_INVALID_STATE;
         }
     }
-    /* We only accept the incoming packet length that is smaller than the max_len (or it will overflow the buffer!) */
     /* If max_len is 0, regard it OK for userspace to get frame len */
+    if (max_len == 0) {
+        ESP_LOGD(TAG, "regard max_len == 0 is OK for user to get frame len");
+        return ESP_OK;
+    }
     if (frame->len > max_len) {
-        if (max_len == 0) {
-            ESP_LOGD(TAG, "regard max_len == 0 is OK for user to get frame len");
-            return ESP_OK;
+        /* When reading entire packet at once, we only accept the incoming packet length that is smaller than the max_len (or it will overflow the buffer!) */
+        if (!partial) {
+            ESP_LOGW(TAG, LOG_FMT("WS Message too long"));
+            return ESP_ERR_INVALID_SIZE;
         }
-        ESP_LOGW(TAG, LOG_FMT("WS Message too long"));
-        return ESP_ERR_INVALID_SIZE;
+        ESP_LOGD(TAG, LOG_FMT("WS Message too long. User will have to call read again"));
     }
 
     /* Receive buffer */
     /* If there's nothing to receive, return and stop here. */
-    if (frame->len == 0) {
+    if (frame->left_len == 0) {
         return ESP_OK;
     }
 
@@ -363,7 +368,7 @@ esp_err_t httpd_ws_recv_frame(httpd_req_t *req, httpd_ws_frame_t *frame, size_t
         return ESP_FAIL;
     }
 
-    size_t left_len = frame->len;
+    size_t left_len = (max_len < frame->left_len) ? max_len : frame->left_len;
     size_t offset = 0;
 
     while (left_len > 0) {
@@ -378,12 +383,23 @@ esp_err_t httpd_ws_recv_frame(httpd_req_t *req, httpd_ws_frame_t *frame, size_t
         ESP_LOGD(TAG, "Frame length: %"NEWLIB_NANO_COMPAT_FORMAT", Bytes Read: %"NEWLIB_NANO_COMPAT_FORMAT, NEWLIB_NANO_COMPAT_CAST(frame->len), NEWLIB_NANO_COMPAT_CAST(offset));
     }
 
+    size_t mask_offset = frame->len - frame->left_len;
+    frame->left_len -= offset;
+
     /* Unmask payload */
-    httpd_ws_unmask_payload(frame->payload, frame->len, aux->mask_key);
+    httpd_ws_unmask_payload(frame->payload, offset, aux->mask_key, mask_offset);
 
     return ESP_OK;
 }
 
+esp_err_t httpd_ws_recv_frame(httpd_req_t *req, httpd_ws_frame_t *frame, size_t max_len) {
+    return httpd_ws_recv_frame_internal(req, frame, max_len, false);
+}
+
+esp_err_t httpd_ws_recv_frame_part(httpd_req_t *req, httpd_ws_frame_t *frame, size_t max_len) {
+    return httpd_ws_recv_frame_internal(req, frame, max_len, true);
+}
+
 esp_err_t httpd_ws_send_frame(httpd_req_t *req, httpd_ws_frame_t *frame)
 {
     esp_err_t ret = httpd_ws_check_req(req);

examples/protocols/http_server/ws_echo_server/main/ws_echo_server.c

@@ -141,6 +141,90 @@ static esp_err_t echo_handler(httpd_req_t *req)
     return ret;
 }
 
+/*
+ * This handler demonstrates partial frame reading using httpd_ws_recv_frame_part()
+ * It reads frames in small chunks and sends them immediately in fragmented mode.
+ * This approach avoids allocating a large buffer equal to the frame length.
+ */
+static esp_err_t echo_partial_handler(httpd_req_t *req)
+{
+    if (req->method == HTTP_GET) {
+        ESP_LOGI(TAG, "Handshake done, the new connection was opened (partial)");
+        return ESP_OK;
+    }
+    httpd_ws_frame_t ws_pkt;
+    uint8_t *chunk_buf = NULL;
+    memset(&ws_pkt, 0, sizeof(httpd_ws_frame_t));
+    ws_pkt.type = HTTPD_WS_TYPE_TEXT;
+
+    /* Set max_len = 0 to get the frame len */
+    esp_err_t ret = httpd_ws_recv_frame_part(req, &ws_pkt, 0);
+    if (ret != ESP_OK) {
+        ESP_LOGE(TAG, "httpd_ws_recv_frame_part failed to get frame len with %d", ret);
+        return ret;
+    }
+    ESP_LOGI(TAG, "Frame len is %d", ws_pkt.len);
+
+    if (ws_pkt.len) {
+        /* Allocate only a small chunk buffer instead of full frame length */
+        /* Use a small chunk size (64 bytes) to demonstrate memory efficiency */
+        #define CHUNK_SIZE 64
+        chunk_buf = calloc(1, CHUNK_SIZE);
+        if (chunk_buf == NULL) {
+            ESP_LOGE(TAG, "Failed to calloc memory for chunk buffer");
+            return ESP_ERR_NO_MEM;
+        }
+        ws_pkt.payload = chunk_buf;
+        ws_pkt.fragmented = true;
+        bool is_first_chunk = true;
+
+        /* Read and send frame in chunks using partial read API */
+        while (ws_pkt.left_len > 0) {
+            size_t chunk_size = (ws_pkt.left_len < CHUNK_SIZE) ? ws_pkt.left_len : CHUNK_SIZE;
+            ESP_LOGI(TAG, "Reading chunk: left_len=%d, chunk_size=%d", ws_pkt.left_len, chunk_size);
+
+            /* Read chunk into small buffer */
+            /* Note: ws_pkt.len remains the total frame length, chunk_size is what we read now */
+            ret = httpd_ws_recv_frame_part(req, &ws_pkt, chunk_size);
+            if (ret != ESP_OK) {
+                ESP_LOGE(TAG, "httpd_ws_recv_frame_part failed with %d", ret);
+                free(chunk_buf);
+                return ret;
+            }
+
+            /* Prepare frame for sending - use chunk_size for the send length */
+            httpd_ws_frame_t send_pkt;
+            memset(&send_pkt, 0, sizeof(httpd_ws_frame_t));
+            send_pkt.payload = chunk_buf;
+            send_pkt.len = chunk_size;
+            send_pkt.fragmented = true;
+
+            if (is_first_chunk) {
+                /* First chunk: use original frame type (TEXT or BINARY) */
+                send_pkt.type = HTTPD_WS_TYPE_TEXT;
+                send_pkt.final = (ws_pkt.left_len == 0);  /* Final if this is the only chunk */
+                is_first_chunk = false;
+            } else {
+                /* Subsequent chunks: use CONTINUE frame type */
+                send_pkt.type = HTTPD_WS_TYPE_CONTINUE;
+                send_pkt.final = (ws_pkt.left_len == 0);  /* Final if this is the last chunk */
+            }
+
+            ret = httpd_ws_send_frame(req, &send_pkt);
+            if (ret != ESP_OK) {
+                ESP_LOGE(TAG, "httpd_ws_send_frame failed with %d", ret);
+                free(chunk_buf);
+                return ret;
+            }
+            ESP_LOGI(TAG, "Sent fragmented chunk: len=%d, final=%d", chunk_size, send_pkt.final);
+        }
+    }
+    ESP_LOGI(TAG, "Packet type: %d", ws_pkt.type);
+
+    free(chunk_buf);
+    return ESP_OK;
+}
+
 static const httpd_uri_t ws = {
         .uri        = "/ws",
         .method     = HTTP_GET,
@@ -149,6 +233,14 @@ static const httpd_uri_t ws = {
         .is_websocket = true
 };
 
+static const httpd_uri_t ws_partial = {
+    .uri        = "/ws_partial",
+    .method     = HTTP_GET,
+    .handler    = echo_partial_handler,
+    .user_ctx   = NULL,
+    .is_websocket = true
+};
+
 static const httpd_uri_t ws_auth = {
         .uri        = "/auth",
         .method     = HTTP_GET,
@@ -172,6 +264,7 @@ static httpd_handle_t start_webserver(void)
         // Registering the ws handler
         ESP_LOGI(TAG, "Registering URI handlers");
         httpd_register_uri_handler(server, &ws);
+        httpd_register_uri_handler(server, &ws_partial);
         httpd_register_uri_handler(server, &ws_auth);
         return server;
     }

examples/protocols/http_server/ws_echo_server/pytest_ws_server_example.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 #
-# SPDX-FileCopyrightText: 2021-2025 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2021-2026 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 import logging
 import os
@@ -27,6 +27,8 @@ class WsClient:
         self.port = port
         self.ip = ip
         self.ws = websocket.WebSocket()
+        # Set timeout to 10 seconds to avoid indefinite blocking
+        self.ws.settimeout(10)
         self.uri = uri
 
     def __enter__(self):  # type: ignore
@@ -37,7 +39,12 @@ class WsClient:
         self.ws.close()
 
     def read(self):  # type: ignore
-        return self.ws.recv_data(control_frame=True)
+        result = self.ws.recv_data(control_frame=True)
+        if result is None:
+            raise RuntimeError('WebSocket recv_data returned None (timeout or connection closed)')
+        if not isinstance(result, tuple) or len(result) != 2:
+            raise RuntimeError(f'WebSocket recv_data returned unexpected format: {result}')
+        return result
 
     def write(self, data='', opcode=OPCODE_TEXT):  # type: ignore
         if opcode == OPCODE_BIN:
@@ -52,14 +59,13 @@ class WsClient:
     'config',
     [
         'default',  # mbedTLS crypto backend
-        'psa',  # PSA crypto backend (tests SHA-1 for WebSocket handshake)
     ],
     indirect=True,
 )
 @idf_parametrize('target', ['esp32'], indirect=['target'])
 def test_examples_protocol_http_ws_echo_server(dut: Dut) -> None:
     """
-    Test WebSocket echo server with both mbedTLS and PSA crypto backends.
+    Test WebSocket echo server with mbedTLS crypto backends.
     This specifically tests the SHA-1 computation used in WebSocket handshake (RFC 6455).
     """
     # Get binary file
@@ -92,7 +98,6 @@ def test_examples_protocol_http_ws_echo_server(dut: Dut) -> None:
             logging.info(f'Testing opcode {expected_opcode}: Received opcode:{opcode}, data:{data}')
             data = data.decode()
             if expected_opcode == OPCODE_PING:
-                dut.expect('Got a WS PING frame, Replying PONG')
                 if opcode != OPCODE_PONG or data != DATA:
                     raise RuntimeError(f'Failed to receive correct opcode:{opcode} or data:{data}')
                 continue
@@ -101,7 +106,12 @@ def test_examples_protocol_http_ws_echo_server(dut: Dut) -> None:
 
             if opcode != expected_opcode or data != DATA or opcode != int(dut_opcode) or (data not in str(dut_data)):
                 raise RuntimeError(f'Failed to receive correct opcode:{opcode} or data:{data}')
+        # Test async send - server queues the work, so we need to wait for it to be processed
+        logging.info('Testing async send')
         ws.write(data='Trigger async', opcode=OPCODE_TEXT)
+        # Wait for server to receive and queue the async send
+        dut.expect(r'Got packet with message: Trigger async', timeout=10)
+        # Now read the async response (server processes work queue asynchronously)
         opcode, data = ws.read()
         logging.info(f'Testing async send: Received opcode:{opcode}, data:{data}')
         data = data.decode()
@@ -114,7 +124,85 @@ def test_examples_protocol_http_ws_echo_server(dut: Dut) -> None:
     'config',
     [
         'default',  # mbedTLS crypto backend
-        'psa',  # PSA crypto backend (tests SHA-1 for WebSocket handshake)
+    ],
+    indirect=True,
+)
+@idf_parametrize('target', ['esp32'], indirect=['target'])
+def test_examples_protocol_http_ws_echo_server_partial(dut: Dut) -> None:
+    # Get binary file
+    binary_file = os.path.join(dut.app.binary_path, 'ws_echo_server.bin')
+    bin_size = os.path.getsize(binary_file)
+    logging.info(f'http_ws_server_bin_size : {bin_size // 1024}KB')
+
+    logging.info('Starting ws-echo-server partial read test')
+
+    # Parse IP address of STA
+    logging.info('Waiting to connect with AP')
+    if dut.app.sdkconfig.get('EXAMPLE_WIFI_SSID_PWD_FROM_STDIN') is True:
+        dut.expect('Please input ssid password:')
+        env_name = 'wifi_router'
+        ap_ssid = get_env_config_variable(env_name, 'ap_ssid')
+        ap_password = get_env_config_variable(env_name, 'ap_password')
+        dut.write(f'{ap_ssid} {ap_password}')
+    got_ip = dut.expect(r'IPv4 address: (\d+\.\d+\.\d+\.\d+)[^\d]', timeout=30)[1].decode()
+    got_port = dut.expect(r"Starting server on port: '(\d+)'", timeout=30)[1].decode()
+
+    logging.info(f'Got IP   : {got_ip}')
+    logging.info(f'Got Port : {got_port}')
+
+    # Start ws server test with partial read endpoint
+    # Create a new websocket connection to the partial endpoint
+    ws_partial = websocket.WebSocket()
+    ws_partial.settimeout(10)
+    try:
+        ws_partial.connect(f'ws://{got_ip}:{int(got_port)}/ws_partial')
+
+        # Create a large message (200 bytes) to force multiple partial reads
+        # The server uses 64-byte chunks, so this will require at least 4 reads
+        large_data = 'A' * 200
+        logging.info(f'Sending large message of {len(large_data)} bytes to test partial reads')
+        ws_partial.send(large_data)
+
+        # Verify partial reads happened in server logs
+        # Add timeouts to prevent indefinite waiting in CI environments
+        dut.expect('Frame len is 200', timeout=10)
+        dut.expect('Reading chunk: left_len=200', timeout=10)
+        dut.expect('Sent fragmented chunk: len=64, final=0', timeout=10)  # First chunk (not final)
+        dut.expect('Reading chunk: left_len=136', timeout=10)  # After first 64-byte read
+        dut.expect('Sent fragmented chunk: len=64, final=0', timeout=10)  # Second chunk (not final)
+        dut.expect('Reading chunk: left_len=72', timeout=10)  # After second 64-byte read
+        dut.expect('Sent fragmented chunk: len=64, final=0', timeout=10)  # Third chunk (not final)
+        dut.expect('Reading chunk: left_len=8', timeout=10)  # After third 64-byte read
+        dut.expect(
+            'Sent fragmented chunk: len=8, final=1', timeout=10
+        )  # Last chunk (final - required by WebSocket protocol)
+        dut.expect('Packet type: 1', timeout=10)  # HTTPD_WS_TYPE_TEXT = 1
+
+        # Read the echo response (websocket client will reassemble fragmented frames)
+        result = ws_partial.recv_data(control_frame=True)
+        if result is None:
+            raise RuntimeError('WebSocket recv_data returned None (timeout or connection closed)')
+        if not isinstance(result, tuple) or len(result) != 2:
+            raise RuntimeError(f'WebSocket recv_data returned unexpected format: {result}')
+        opcode, data = result
+        logging.info(f'Testing partial read: Received opcode:{opcode}, data length:{len(data)}')
+        data = data.decode()
+
+        if opcode != OPCODE_TEXT or data != large_data:
+            raise RuntimeError(
+                f'Failed to receive correct echo for partial read. Expected length:{len(large_data)}, got:{len(data)}'
+            )
+
+        logging.info(f'Partial read test passed: received {len(data)} bytes correctly')
+    finally:
+        ws_partial.close()
+
+
+@pytest.mark.wifi_router
+@pytest.mark.parametrize(
+    'config',
+    [
+        'default',  # mbedTLS crypto backend
     ],
     indirect=True,
 )
@@ -123,7 +211,7 @@ def test_ws_auth_handshake(dut: Dut) -> None:
     """
     Test that connecting to /ws does NOT print the handshake success log.
     This is used to verify ws_pre_handshake_cb can reject the handshake.
-    Tests both mbedTLS and PSA crypto backends.
+    Tests mbedTLS crypto backend.
     """
     # Wait for device to connect and start server
     if dut.app.sdkconfig.get('EXAMPLE_WIFI_SSID_PWD_FROM_STDIN') is True: