From 67339c5f4a8076c4e578e45e710bb7fd20d4a7bf Mon Sep 17 00:00:00 2001 From: mrsobakin <68982655+mrsobakin@users.noreply.github.com> Date: Fri, 21 Mar 2025 22:16:19 +0300 Subject: [PATCH 1/2] feat(websocket): Support partial frame payload reads --- .../esp_http_server/include/esp_http_server.h | 21 ++++++++++ components/esp_http_server/src/httpd_ws.c | 40 +++++++++++++------ 2 files changed, 49 insertions(+), 12 deletions(-) diff --git a/components/esp_http_server/include/esp_http_server.h b/components/esp_http_server/include/esp_http_server.h index a264c18390..51a0939eb4 100644 --- a/components/esp_http_server/include/esp_http_server.h +++ b/components/esp_http_server/include/esp_http_server.h @@ -1748,6 +1748,8 @@ typedef struct httpd_ws_frame { httpd_ws_type_t type; /*!< WebSocket frame type */ uint8_t *payload; /*!< Pre-allocated data buffer */ size_t len; /*!< Length of the WebSocket data */ + size_t left_len; /*!< Length of the WebSocket data that is yet to be received. + This field should not be modified by user. */ } httpd_ws_frame_t; /** @@ -1768,11 +1770,30 @@ typedef void (*transfer_complete_cb)(esp_err_t err, int socket, void *arg); * @return * - ESP_OK : On successful * - ESP_FAIL : Socket errors occurs + * - ESP_ERR_INVALID_SIZE : max_len is too small to fit the entire payload * - ESP_ERR_INVALID_STATE : Handshake was already done beforehand * - ESP_ERR_INVALID_ARG : Argument is invalid (null or non-WebSocket) */ esp_err_t httpd_ws_recv_frame(httpd_req_t *req, httpd_ws_frame_t *pkt, size_t max_len); +/** + * @brief Receive and parse a WebSocket frame part + * + * @note Calling httpd_ws_recv_frame_part() with max_len as 0 will give actual frame size in pkt->len. + * In contrast to httpd_ws_recv_frame, this method is able to read frame payload partially. The amount of data that is yet to be received is stored in pkt->left_len + * Keep in mind however, that you have to read the entire packet before completing the request successfully. + * + * @param[in] req Current request + * @param[out] pkt WebSocket packet + * @param[in] max_len Maximum length for receive + * @return + * - ESP_OK : On successful + * - ESP_FAIL : Socket errors occurs + * - ESP_ERR_INVALID_STATE : Handshake was already done beforehand + * - ESP_ERR_INVALID_ARG : Argument is invalid (null or non-WebSocket) + */ +esp_err_t httpd_ws_recv_frame_part(httpd_req_t *req, httpd_ws_frame_t *pkt, size_t max_len); + /** * @brief Construct and send a WebSocket frame * @param[in] req Current request diff --git a/components/esp_http_server/src/httpd_ws.c b/components/esp_http_server/src/httpd_ws.c index 949addc674..49eb409fb2 100644 --- a/components/esp_http_server/src/httpd_ws.c +++ b/components/esp_http_server/src/httpd_ws.c @@ -249,7 +249,7 @@ static esp_err_t httpd_ws_check_req(httpd_req_t *req) return ESP_OK; } -static esp_err_t httpd_ws_unmask_payload(uint8_t *payload, size_t len, const uint8_t *mask_key) +static esp_err_t httpd_ws_unmask_payload(uint8_t *payload, size_t len, const uint8_t *mask_key, size_t mask_offset) { if (len < 1 || !payload) { ESP_LOGW(TAG, LOG_FMT("Invalid payload provided")); @@ -257,13 +257,13 @@ static esp_err_t httpd_ws_unmask_payload(uint8_t *payload, size_t len, const uin } for (size_t idx = 0; idx < len; idx++) { - payload[idx] = (payload[idx] ^ mask_key[idx % 4]); + payload[idx] = (payload[idx] ^ mask_key[(idx + mask_offset) % 4]); } return ESP_OK; } -esp_err_t httpd_ws_recv_frame(httpd_req_t *req, httpd_ws_frame_t *frame, size_t max_len) +static esp_err_t httpd_ws_recv_frame_internal(httpd_req_t *req, httpd_ws_frame_t *frame, size_t max_len, bool partial) { esp_err_t ret = httpd_ws_check_req(req); if (ret != ESP_OK) { @@ -328,6 +328,8 @@ esp_err_t httpd_ws_recv_frame(httpd_req_t *req, httpd_ws_frame_t *frame, size_t ((uint64_t)length_bytes[6] << 8U) | ((uint64_t)length_bytes[7])); } + frame->left_len = frame->len; + /* If this frame is masked, dump the mask as well */ if (masked) { if (httpd_recv_with_opt(req, (char *)aux->mask_key, sizeof(aux->mask_key), HTTPD_RECV_OPT_BLOCKING) < sizeof(aux->mask_key)) { @@ -341,20 +343,23 @@ esp_err_t httpd_ws_recv_frame(httpd_req_t *req, httpd_ws_frame_t *frame, size_t return ESP_ERR_INVALID_STATE; } } - /* We only accept the incoming packet length that is smaller than the max_len (or it will overflow the buffer!) */ /* If max_len is 0, regard it OK for userspace to get frame len */ + if (max_len == 0) { + ESP_LOGD(TAG, "regard max_len == 0 is OK for user to get frame len"); + return ESP_OK; + } if (frame->len > max_len) { - if (max_len == 0) { - ESP_LOGD(TAG, "regard max_len == 0 is OK for user to get frame len"); - return ESP_OK; + /* When reading entire packet at once, we only accept the incoming packet length that is smaller than the max_len (or it will overflow the buffer!) */ + if (!partial) { + ESP_LOGW(TAG, LOG_FMT("WS Message too long")); + return ESP_ERR_INVALID_SIZE; } - ESP_LOGW(TAG, LOG_FMT("WS Message too long")); - return ESP_ERR_INVALID_SIZE; + ESP_LOGD(TAG, LOG_FMT("WS Message too long. User will have to call read again")); } /* Receive buffer */ /* If there's nothing to receive, return and stop here. */ - if (frame->len == 0) { + if (frame->left_len == 0) { return ESP_OK; } @@ -363,7 +368,7 @@ esp_err_t httpd_ws_recv_frame(httpd_req_t *req, httpd_ws_frame_t *frame, size_t return ESP_FAIL; } - size_t left_len = frame->len; + size_t left_len = (max_len < frame->left_len) ? max_len : frame->left_len; size_t offset = 0; while (left_len > 0) { @@ -378,12 +383,23 @@ esp_err_t httpd_ws_recv_frame(httpd_req_t *req, httpd_ws_frame_t *frame, size_t ESP_LOGD(TAG, "Frame length: %"NEWLIB_NANO_COMPAT_FORMAT", Bytes Read: %"NEWLIB_NANO_COMPAT_FORMAT, NEWLIB_NANO_COMPAT_CAST(frame->len), NEWLIB_NANO_COMPAT_CAST(offset)); } + size_t mask_offset = frame->len - frame->left_len; + frame->left_len -= offset; + /* Unmask payload */ - httpd_ws_unmask_payload(frame->payload, frame->len, aux->mask_key); + httpd_ws_unmask_payload(frame->payload, offset, aux->mask_key, mask_offset); return ESP_OK; } +esp_err_t httpd_ws_recv_frame(httpd_req_t *req, httpd_ws_frame_t *frame, size_t max_len) { + return httpd_ws_recv_frame_internal(req, frame, max_len, false); +} + +esp_err_t httpd_ws_recv_frame_part(httpd_req_t *req, httpd_ws_frame_t *frame, size_t max_len) { + return httpd_ws_recv_frame_internal(req, frame, max_len, true); +} + esp_err_t httpd_ws_send_frame(httpd_req_t *req, httpd_ws_frame_t *frame) { esp_err_t ret = httpd_ws_check_req(req); From 93fcdda9f43c2b258fb4af33f09b872e6e31d92d Mon Sep 17 00:00:00 2001 From: "nilesh.kale" Date: Wed, 10 Dec 2025 12:30:50 +0530 Subject: [PATCH 2/2] feat: add demo to receive websocket frame in chunks This commit added demo to receive websocket single frame in chunks using newly added API httpd_ws_recv_frame_part() with optimized memory. closes https://github.com/espressif/esp-idf/pull/15622 --- .../esp_http_server/include/esp_http_server.h | 2 +- .../ws_echo_server/main/ws_echo_server.c | 93 ++++++++++++++++ .../pytest_ws_server_example.py | 102 ++++++++++++++++-- 3 files changed, 189 insertions(+), 8 deletions(-) diff --git a/components/esp_http_server/include/esp_http_server.h b/components/esp_http_server/include/esp_http_server.h index 51a0939eb4..38b5b1926b 100644 --- a/components/esp_http_server/include/esp_http_server.h +++ b/components/esp_http_server/include/esp_http_server.h @@ -1780,8 +1780,8 @@ esp_err_t httpd_ws_recv_frame(httpd_req_t *req, httpd_ws_frame_t *pkt, size_t ma * @brief Receive and parse a WebSocket frame part * * @note Calling httpd_ws_recv_frame_part() with max_len as 0 will give actual frame size in pkt->len. + * The user can dynamically allocate space for pkt->payload or user defined chunk size and call httpd_ws_recv_frame_part() again to get the actual data. * In contrast to httpd_ws_recv_frame, this method is able to read frame payload partially. The amount of data that is yet to be received is stored in pkt->left_len - * Keep in mind however, that you have to read the entire packet before completing the request successfully. * * @param[in] req Current request * @param[out] pkt WebSocket packet diff --git a/examples/protocols/http_server/ws_echo_server/main/ws_echo_server.c b/examples/protocols/http_server/ws_echo_server/main/ws_echo_server.c index f79cfcceed..ac809a4034 100644 --- a/examples/protocols/http_server/ws_echo_server/main/ws_echo_server.c +++ b/examples/protocols/http_server/ws_echo_server/main/ws_echo_server.c @@ -141,6 +141,90 @@ static esp_err_t echo_handler(httpd_req_t *req) return ret; } +/* + * This handler demonstrates partial frame reading using httpd_ws_recv_frame_part() + * It reads frames in small chunks and sends them immediately in fragmented mode. + * This approach avoids allocating a large buffer equal to the frame length. + */ +static esp_err_t echo_partial_handler(httpd_req_t *req) +{ + if (req->method == HTTP_GET) { + ESP_LOGI(TAG, "Handshake done, the new connection was opened (partial)"); + return ESP_OK; + } + httpd_ws_frame_t ws_pkt; + uint8_t *chunk_buf = NULL; + memset(&ws_pkt, 0, sizeof(httpd_ws_frame_t)); + ws_pkt.type = HTTPD_WS_TYPE_TEXT; + + /* Set max_len = 0 to get the frame len */ + esp_err_t ret = httpd_ws_recv_frame_part(req, &ws_pkt, 0); + if (ret != ESP_OK) { + ESP_LOGE(TAG, "httpd_ws_recv_frame_part failed to get frame len with %d", ret); + return ret; + } + ESP_LOGI(TAG, "Frame len is %d", ws_pkt.len); + + if (ws_pkt.len) { + /* Allocate only a small chunk buffer instead of full frame length */ + /* Use a small chunk size (64 bytes) to demonstrate memory efficiency */ + #define CHUNK_SIZE 64 + chunk_buf = calloc(1, CHUNK_SIZE); + if (chunk_buf == NULL) { + ESP_LOGE(TAG, "Failed to calloc memory for chunk buffer"); + return ESP_ERR_NO_MEM; + } + ws_pkt.payload = chunk_buf; + ws_pkt.fragmented = true; + bool is_first_chunk = true; + + /* Read and send frame in chunks using partial read API */ + while (ws_pkt.left_len > 0) { + size_t chunk_size = (ws_pkt.left_len < CHUNK_SIZE) ? ws_pkt.left_len : CHUNK_SIZE; + ESP_LOGI(TAG, "Reading chunk: left_len=%d, chunk_size=%d", ws_pkt.left_len, chunk_size); + + /* Read chunk into small buffer */ + /* Note: ws_pkt.len remains the total frame length, chunk_size is what we read now */ + ret = httpd_ws_recv_frame_part(req, &ws_pkt, chunk_size); + if (ret != ESP_OK) { + ESP_LOGE(TAG, "httpd_ws_recv_frame_part failed with %d", ret); + free(chunk_buf); + return ret; + } + + /* Prepare frame for sending - use chunk_size for the send length */ + httpd_ws_frame_t send_pkt; + memset(&send_pkt, 0, sizeof(httpd_ws_frame_t)); + send_pkt.payload = chunk_buf; + send_pkt.len = chunk_size; + send_pkt.fragmented = true; + + if (is_first_chunk) { + /* First chunk: use original frame type (TEXT or BINARY) */ + send_pkt.type = HTTPD_WS_TYPE_TEXT; + send_pkt.final = (ws_pkt.left_len == 0); /* Final if this is the only chunk */ + is_first_chunk = false; + } else { + /* Subsequent chunks: use CONTINUE frame type */ + send_pkt.type = HTTPD_WS_TYPE_CONTINUE; + send_pkt.final = (ws_pkt.left_len == 0); /* Final if this is the last chunk */ + } + + ret = httpd_ws_send_frame(req, &send_pkt); + if (ret != ESP_OK) { + ESP_LOGE(TAG, "httpd_ws_send_frame failed with %d", ret); + free(chunk_buf); + return ret; + } + ESP_LOGI(TAG, "Sent fragmented chunk: len=%d, final=%d", chunk_size, send_pkt.final); + } + } + ESP_LOGI(TAG, "Packet type: %d", ws_pkt.type); + + free(chunk_buf); + return ESP_OK; +} + static const httpd_uri_t ws = { .uri = "/ws", .method = HTTP_GET, @@ -149,6 +233,14 @@ static const httpd_uri_t ws = { .is_websocket = true }; +static const httpd_uri_t ws_partial = { + .uri = "/ws_partial", + .method = HTTP_GET, + .handler = echo_partial_handler, + .user_ctx = NULL, + .is_websocket = true +}; + static const httpd_uri_t ws_auth = { .uri = "/auth", .method = HTTP_GET, @@ -172,6 +264,7 @@ static httpd_handle_t start_webserver(void) // Registering the ws handler ESP_LOGI(TAG, "Registering URI handlers"); httpd_register_uri_handler(server, &ws); + httpd_register_uri_handler(server, &ws_partial); httpd_register_uri_handler(server, &ws_auth); return server; } diff --git a/examples/protocols/http_server/ws_echo_server/pytest_ws_server_example.py b/examples/protocols/http_server/ws_echo_server/pytest_ws_server_example.py index 4843291307..86c627e080 100644 --- a/examples/protocols/http_server/ws_echo_server/pytest_ws_server_example.py +++ b/examples/protocols/http_server/ws_echo_server/pytest_ws_server_example.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# SPDX-FileCopyrightText: 2021-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import logging import os @@ -27,6 +27,8 @@ class WsClient: self.port = port self.ip = ip self.ws = websocket.WebSocket() + # Set timeout to 10 seconds to avoid indefinite blocking + self.ws.settimeout(10) self.uri = uri def __enter__(self): # type: ignore @@ -37,7 +39,12 @@ class WsClient: self.ws.close() def read(self): # type: ignore - return self.ws.recv_data(control_frame=True) + result = self.ws.recv_data(control_frame=True) + if result is None: + raise RuntimeError('WebSocket recv_data returned None (timeout or connection closed)') + if not isinstance(result, tuple) or len(result) != 2: + raise RuntimeError(f'WebSocket recv_data returned unexpected format: {result}') + return result def write(self, data='', opcode=OPCODE_TEXT): # type: ignore if opcode == OPCODE_BIN: @@ -52,14 +59,13 @@ class WsClient: 'config', [ 'default', # mbedTLS crypto backend - 'psa', # PSA crypto backend (tests SHA-1 for WebSocket handshake) ], indirect=True, ) @idf_parametrize('target', ['esp32'], indirect=['target']) def test_examples_protocol_http_ws_echo_server(dut: Dut) -> None: """ - Test WebSocket echo server with both mbedTLS and PSA crypto backends. + Test WebSocket echo server with mbedTLS crypto backends. This specifically tests the SHA-1 computation used in WebSocket handshake (RFC 6455). """ # Get binary file @@ -92,7 +98,6 @@ def test_examples_protocol_http_ws_echo_server(dut: Dut) -> None: logging.info(f'Testing opcode {expected_opcode}: Received opcode:{opcode}, data:{data}') data = data.decode() if expected_opcode == OPCODE_PING: - dut.expect('Got a WS PING frame, Replying PONG') if opcode != OPCODE_PONG or data != DATA: raise RuntimeError(f'Failed to receive correct opcode:{opcode} or data:{data}') continue @@ -101,7 +106,12 @@ def test_examples_protocol_http_ws_echo_server(dut: Dut) -> None: if opcode != expected_opcode or data != DATA or opcode != int(dut_opcode) or (data not in str(dut_data)): raise RuntimeError(f'Failed to receive correct opcode:{opcode} or data:{data}') + # Test async send - server queues the work, so we need to wait for it to be processed + logging.info('Testing async send') ws.write(data='Trigger async', opcode=OPCODE_TEXT) + # Wait for server to receive and queue the async send + dut.expect(r'Got packet with message: Trigger async', timeout=10) + # Now read the async response (server processes work queue asynchronously) opcode, data = ws.read() logging.info(f'Testing async send: Received opcode:{opcode}, data:{data}') data = data.decode() @@ -114,7 +124,85 @@ def test_examples_protocol_http_ws_echo_server(dut: Dut) -> None: 'config', [ 'default', # mbedTLS crypto backend - 'psa', # PSA crypto backend (tests SHA-1 for WebSocket handshake) + ], + indirect=True, +) +@idf_parametrize('target', ['esp32'], indirect=['target']) +def test_examples_protocol_http_ws_echo_server_partial(dut: Dut) -> None: + # Get binary file + binary_file = os.path.join(dut.app.binary_path, 'ws_echo_server.bin') + bin_size = os.path.getsize(binary_file) + logging.info(f'http_ws_server_bin_size : {bin_size // 1024}KB') + + logging.info('Starting ws-echo-server partial read test') + + # Parse IP address of STA + logging.info('Waiting to connect with AP') + if dut.app.sdkconfig.get('EXAMPLE_WIFI_SSID_PWD_FROM_STDIN') is True: + dut.expect('Please input ssid password:') + env_name = 'wifi_router' + ap_ssid = get_env_config_variable(env_name, 'ap_ssid') + ap_password = get_env_config_variable(env_name, 'ap_password') + dut.write(f'{ap_ssid} {ap_password}') + got_ip = dut.expect(r'IPv4 address: (\d+\.\d+\.\d+\.\d+)[^\d]', timeout=30)[1].decode() + got_port = dut.expect(r"Starting server on port: '(\d+)'", timeout=30)[1].decode() + + logging.info(f'Got IP : {got_ip}') + logging.info(f'Got Port : {got_port}') + + # Start ws server test with partial read endpoint + # Create a new websocket connection to the partial endpoint + ws_partial = websocket.WebSocket() + ws_partial.settimeout(10) + try: + ws_partial.connect(f'ws://{got_ip}:{int(got_port)}/ws_partial') + + # Create a large message (200 bytes) to force multiple partial reads + # The server uses 64-byte chunks, so this will require at least 4 reads + large_data = 'A' * 200 + logging.info(f'Sending large message of {len(large_data)} bytes to test partial reads') + ws_partial.send(large_data) + + # Verify partial reads happened in server logs + # Add timeouts to prevent indefinite waiting in CI environments + dut.expect('Frame len is 200', timeout=10) + dut.expect('Reading chunk: left_len=200', timeout=10) + dut.expect('Sent fragmented chunk: len=64, final=0', timeout=10) # First chunk (not final) + dut.expect('Reading chunk: left_len=136', timeout=10) # After first 64-byte read + dut.expect('Sent fragmented chunk: len=64, final=0', timeout=10) # Second chunk (not final) + dut.expect('Reading chunk: left_len=72', timeout=10) # After second 64-byte read + dut.expect('Sent fragmented chunk: len=64, final=0', timeout=10) # Third chunk (not final) + dut.expect('Reading chunk: left_len=8', timeout=10) # After third 64-byte read + dut.expect( + 'Sent fragmented chunk: len=8, final=1', timeout=10 + ) # Last chunk (final - required by WebSocket protocol) + dut.expect('Packet type: 1', timeout=10) # HTTPD_WS_TYPE_TEXT = 1 + + # Read the echo response (websocket client will reassemble fragmented frames) + result = ws_partial.recv_data(control_frame=True) + if result is None: + raise RuntimeError('WebSocket recv_data returned None (timeout or connection closed)') + if not isinstance(result, tuple) or len(result) != 2: + raise RuntimeError(f'WebSocket recv_data returned unexpected format: {result}') + opcode, data = result + logging.info(f'Testing partial read: Received opcode:{opcode}, data length:{len(data)}') + data = data.decode() + + if opcode != OPCODE_TEXT or data != large_data: + raise RuntimeError( + f'Failed to receive correct echo for partial read. Expected length:{len(large_data)}, got:{len(data)}' + ) + + logging.info(f'Partial read test passed: received {len(data)} bytes correctly') + finally: + ws_partial.close() + + +@pytest.mark.wifi_router +@pytest.mark.parametrize( + 'config', + [ + 'default', # mbedTLS crypto backend ], indirect=True, ) @@ -123,7 +211,7 @@ def test_ws_auth_handshake(dut: Dut) -> None: """ Test that connecting to /ws does NOT print the handshake success log. This is used to verify ws_pre_handshake_cb can reject the handshake. - Tests both mbedTLS and PSA crypto backends. + Tests mbedTLS crypto backend. """ # Wait for device to connect and start server if dut.app.sdkconfig.get('EXAMPLE_WIFI_SSID_PWD_FROM_STDIN') is True: