11import asyncio
22import json
33import re
4+ import time
45from pathlib import Path
6+ from typing import NamedTuple
57
68import pytest
79
@@ -103,6 +105,67 @@ def test_keeps_a_bom_after_the_body_start():
103105 assert result == ['{"id": 1}' , '\ufeff {"id": 2}' ]
104106
105107
108+ LONG_RECORD_CHUNK_SIZE = 8 * 1024
109+ LONG_RECORD_SIZE = 16 * 1024 * 1024
110+ # The linear splitter needs a few milliseconds for a 16MB record, while re-splitting the
111+ # whole buffer on every chunk needed seconds, so this budget is generous on a slow
112+ # machine and still fails on a return to quadratic behaviour.
113+ LONG_RECORD_BUDGET_SECONDS = 2
114+
115+
116+ @pytest .fixture (scope = "module" )
117+ def long_record ():
118+ # Built on demand rather than at import, so runs that do not select the tests below
119+ # do not pay for the 16MB body during collection.
120+ return json .dumps ({"note" : "a" * LONG_RECORD_SIZE })
121+
122+
123+ def chunked (body , size = LONG_RECORD_CHUNK_SIZE ):
124+ offsets = range (0 , len (body ), size )
125+ return [body [offset : offset + size ] for offset in offsets ]
126+
127+
128+ class TimedLines (NamedTuple ):
129+ lines : list [str ]
130+ elapsed : float
131+
132+
133+ def timed_lines (chunks ):
134+ started_at = time .perf_counter ()
135+ lines = list (iter_jsonl_lines (chunks ))
136+ return TimedLines (lines , time .perf_counter () - started_at )
137+
138+
139+ async def collected_lines (chunks ):
140+ return [line async for line in aiter_jsonl_lines (chunks )]
141+
142+
143+ async def timed_async_lines (chunks ):
144+ started_at = time .perf_counter ()
145+ lines = await collected_lines (chunks )
146+ return TimedLines (lines , time .perf_counter () - started_at )
147+
148+
149+ def test_long_record_across_chunks_is_linear (long_record ):
150+ chunks = chunked (f"{ long_record } \n " )
151+
152+ result = timed_lines (chunks )
153+
154+ assert result .lines == [long_record ]
155+ assert result .elapsed < LONG_RECORD_BUDGET_SECONDS
156+
157+
158+ def test_unterminated_long_record_is_linear (long_record ):
159+ # The body confirm_stream_format tolerates without a content type: a single long
160+ # line, unterminated, so every chunk stays buffered and nothing is flushed early.
161+ chunks = chunked (long_record )
162+
163+ result = timed_lines (chunks )
164+
165+ assert result .lines == [long_record ]
166+ assert result .elapsed < LONG_RECORD_BUDGET_SECONDS
167+
168+
106169def test_decode_record_line_returns_the_object ():
107170 result = decode_record_line ('{"id": "ID-1"}' )
108171
@@ -156,6 +219,15 @@ async def test_async_preserves_standalone_trailing_cr():
156219 assert result == ['{"id": 1}\r ' ]
157220
158221
222+ async def test_async_long_record_is_linear (long_record ):
223+ chunks = async_chunks (chunked (f"{ long_record } \n " ))
224+
225+ result = await timed_async_lines (chunks )
226+
227+ assert result .lines == [long_record ]
228+ assert result .elapsed < LONG_RECORD_BUDGET_SECONDS
229+
230+
159231async def test_async_drops_a_bom_opening_the_body ():
160232 chunks = async_chunks (['\ufeff {"id": 1}\n {"id": 2}\n ' ])
161233
@@ -164,6 +236,14 @@ async def test_async_drops_a_bom_opening_the_body():
164236 assert result == ['{"id": 1}' , '{"id": 2}' ]
165237
166238
239+ async def test_async_drops_bom_after_empty_chunk ():
240+ chunks = async_chunks (["" , '\ufeff {"id": 1}\n ' ])
241+
242+ result = await collected_lines (chunks )
243+
244+ assert result == ['{"id": 1}' ]
245+
246+
167247async def test_async_drops_the_bom_in_its_own_chunk ():
168248 chunks = async_chunks (["\ufeff " , '{"id": 1}\n ' ])
169249
0 commit comments