From b7aac9b0406e4f587dd02e80fcf5dfe242f1d019 Mon Sep 17 00:00:00 2001 From: Lee Yunjin Date: Sun, 11 Oct 2026 00:36:12 +0900 Subject: [PATCH] test(https): pool queue growth regression test and RSS/throughput A/B tests/test_https_pool_queue.c holds the only pool thread while 5000 tasks queue behind it, so the ring wraps and then grows several times; every task must run exactly once, in submission order. docs/performance/https-pool-dynamic-queue.md records the A/B of the static 2097152-slot ring against the growable one (8d5a5958): pool init RSS +98484 KiB vs +176 KiB, idle RSS of a 4-worker server 422 MB vs 29 MB, unchanged keep-alive/churn/burst results, and a 6000-connection burst on one worker that grew the ring to 4096 slots with no failures. --- .gitignore | 1 + Makefile | 5 ++ docs/performance/https-pool-dynamic-queue.md | 94 ++++++++++++++++++++ tests/test_https_pool_queue.c | 94 ++++++++++++++++++++ 4 files changed, 194 insertions(+) create mode 100644 docs/performance/https-pool-dynamic-queue.md create mode 100644 tests/test_https_pool_queue.c diff --git a/.gitignore b/.gitignore index d97c513d8..ea6d0158d 100644 --- a/.gitignore +++ b/.gitignore @@ -137,6 +137,7 @@ fly.board/ test_https test_https_park test_https_park_full_gc +test_https_pool_queue benchmarks/web-frameworks/cwist_bench dist/ cwist.pc diff --git a/Makefile b/Makefile index 35816f0f2..e7ff3bfb4 100644 --- a/Makefile +++ b/Makefile @@ -907,6 +907,7 @@ TEST_TARGETS = test_worker_affinity \ test_https \ test_https_park \ test_https_park_full_gc \ + test_https_pool_queue \ test_https_metrics \ test_http2 \ test_http2_prebuffer \ @@ -1154,6 +1155,10 @@ test_https_park: $(LIB_NAME) tests/test_https_park.c $(CC) $(CFLAGS) -o test_https_park tests/test_https_park.c $(LIB_NAME) $(LIBS) ./test_https_park +test_https_pool_queue: $(LIB_NAME) tests/test_https_pool_queue.c + $(CC) $(CFLAGS) -o test_https_pool_queue tests/test_https_pool_queue.c $(LIB_NAME) $(LIBS) + ./test_https_pool_queue + test_https_park_full_gc: $(LIB_NAME) tests/test_https_park_full_gc.c tests/test_https_park.c $(CC) $(CFLAGS) -o test_https_park_full_gc tests/test_https_park_full_gc.c $(LIB_NAME) $(LIBS) ./test_https_park_full_gc diff --git a/docs/performance/https-pool-dynamic-queue.md b/docs/performance/https-pool-dynamic-queue.md new file mode 100644 index 000000000..29892de38 --- /dev/null +++ b/docs/performance/https-pool-dynamic-queue.md @@ -0,0 +1,94 @@ +# HTTPS pool task queue: static ring vs growable ring + +Experiments behind commit `8d5a5958` ("grow the pool task ring on demand +instead of a 96 MiB static array"), main `e7568313`. + +## What changed + +`https_thread_pool_t.queue` was a static `https_pool_task_t[2097152]`. Each +task is 48 bytes, so the array was about 96 MiB. `https_pool_init()` memsets +the whole pool struct, so that 96 MiB became resident in every process that +starts the pool, before the first connection. With `CWIST_C1M_MODE=1` that is +every worker process plus the master. + +The queue is now `queue` / `queue_len`: a ring that starts empty, takes +`HTTPS_TASK_QUEUE_INITIAL` (1024) slots on first use, and doubles through +`resize_queue()` when full, up to the old limit `HTTPS_TASK_QUEUE_MAX` +(2097152). At that limit, or if an allocation fails, submitters block on +`cond_not_full` as before. The ring is not shrunk. + +## Setup + +- AMD Ryzen 5 5600X (6 cores / 12 threads), Linux; client and server on loopback. +- Two non-ASan builds from the same tree, differing only in + `src/net/http/https.c` (`5f779476` vs `8d5a5958`). +- Server: a minimal `cwist_app` with HTTPS (`GET /` returns `ok`), started + with `CWIST_C1M_MODE=1`. +- RSS is the sum of `VmRSS` over the master and its workers. + +## 1. RSS of `https_pool_init()` alone + +| build | VmRSS before init | after init | delta | +|---|---|---|---| +| static ring | 9 876 KiB | 108 360 KiB | **+98 484 KiB** | +| growable ring | 10 104 KiB | 10 280 KiB | **+176 KiB** (thread stacks) | + +`size(1)`: the server binary's `.bss` drops from 100 861 061 to 197 765 bytes. + +## 2. 4 workers, steady and churn load (3 alternating rounds) + +`CWIST_WORKERS=4`. Load in order: `wrk -t4 -c64 -d10s` (keep-alive), +`ab -n 5000 -c 64` (a new TLS connection per request), then +`h2load --h1 -n 3000 -c 3000` (burst). + +| round | build | idle RSS | peak RSS | keep-alive req/s | churn req/s | 3000-conn burst | +|---|---|---|---|---|---|---| +| 1 | static | 422 380 KiB | 459 060 KiB | 159 541 | 840 | 3000/3000, 3.62 s | +| 1 | growable | 29 308 KiB | 67 196 KiB | 162 408 | 841 | 3000/3000, 3.73 s | +| 2 | static | 422 324 KiB | 463 564 KiB | 155 651 | 798 | 3000/3000, 4.01 s | +| 2 | growable | 28 892 KiB | 63 920 KiB | 138 338 | 766 | 3000/3000, 3.91 s | +| 3 | static | 420 748 KiB | 460 768 KiB | 146 037 | 857 | 3000/3000, 3.35 s | +| 3 | growable | 29 368 KiB | 65 616 KiB | 153 601 | 842 | 3000/3000, 3.49 s | + +- Idle RSS drops by about 393 MiB, which is 5 processes × ~96 MiB less a + little. Peak RSS under load drops by the same amount. +- Throughput, churn rate, and burst completion are within run-to-run noise. + Round 2's keep-alive gap goes the other way in rounds 1 and 3. + +## 3. Forcing the ring to grow: 1 worker, 2 pool threads, 6000 simultaneous connections + +`CWIST_WORKERS=1 CWIST_WORKER_THREADS=2`, then +`h2load --h1 -n 6000 -c 6000 -t 8`. + +| round | build | idle RSS | peak RSS | result | finished in | +|---|---|---|---|---|---| +| 1 | static | 112 552 KiB | 161 612 KiB | 6000/6000, 0 failed | 6.61 s | +| 1 | growable | 14 440 KiB | 61 832 KiB | 6000/6000, 0 failed | 6.60 s | +| 2 | static | 112 788 KiB | 189 924 KiB | 6000/6000, 0 failed | 6.49 s | +| 2 | growable | 14 388 KiB | 62 244 KiB | 6000/6000, 0 failed | 6.46 s | + +Afterwards, gdb on the worker of the growable build showed +`g_https_pool.queue_len = 4096` and `count = 0`. So the burst did grow the +ring twice (1024 → 2048 → 4096), every task was served, and the queue +drained. + +## 4. Order and loss under growth: `tests/test_https_pool_queue.c` + +One pool thread is held in a handler while 5000 tasks queue behind it. The +ring first wraps at its initial size, then grows several times while +wrapped. Every task carries a sequence number and must run exactly once, in +submission order. It passes under ASan + UBSan. `test_https`, +`test_https_park`, `test_https_park_full_gc`, `test_https_full_gc`, +`test_http2`, and `test_shutdown` also pass under ASan with the change. + +## Not done / open + +- **Shrinking.** A process keeps the capacity of its largest burst: 48 bytes + per slot, so 4096 slots are 192 KiB and the full 2097152 slots are back to + 96 MiB. Shrinking when the ring drains below a quarter would bound this, + but needs a policy against resize thrash, so it is not part of the minimal + change. +- **Lock-free queue.** Not explored, and pool-mutex contention was not + measured here, so these runs do not show whether the lock matters. The + growable ring keeps the existing mutex + condvar design, so swapping in a + different queue later stays a separate, measurable change. diff --git a/tests/test_https_pool_queue.c b/tests/test_https_pool_queue.c new file mode 100644 index 000000000..f3bfed594 --- /dev/null +++ b/tests/test_https_pool_queue.c @@ -0,0 +1,94 @@ +/** + * @file test_https_pool_queue.c + * @brief The HTTPS pool's task ring grows on demand and keeps FIFO order. + * + * One pool thread is held inside a handler while thousands of tasks are + * queued behind it, forcing the ring through several doublings, including + * growth while the ring has wrapped. Tasks carry a sequence number in + * user_ctx; every one must run exactly once, in submission order. The + * connections are inert stand-ins (no fd, no SSL), so teardown is cheap. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define TASKS 5000 + +static pthread_mutex_t g_gate_mu = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t g_gate_cv = PTHREAD_COND_INITIALIZER; +static bool g_gate_open = false; +static atomic_bool g_blocker_running = false; +static long g_seen[TASKS]; +static atomic_long g_seen_n = 0; + +static cwist_https_connection *fake_conn(void) { + cwist_https_connection *c = cwist_alloc(sizeof(*c)); + assert(c != NULL); + memset(c, 0, sizeof(*c)); + c->fd = -1; + return c; +} + +static void blocker(cwist_https_connection *conn, void *ctx) { + (void)conn; + (void)ctx; + atomic_store(&g_blocker_running, true); + pthread_mutex_lock(&g_gate_mu); + while (!g_gate_open) pthread_cond_wait(&g_gate_cv, &g_gate_mu); + pthread_mutex_unlock(&g_gate_mu); +} + +static void record(cwist_https_connection *conn, void *ctx) { + (void)conn; + long n = atomic_fetch_add(&g_seen_n, 1); + assert(n < TASKS); + g_seen[n] = (long)(intptr_t)ctx; +} + +static void submit_round(long base, long count) { + for (long i = 0; i < count; i++) { + https_pool_submit_conn(fake_conn(), NULL, record, (void *)(intptr_t)(base + i)); + } +} + +int main(void) { + setenv("CWIST_WORKER_THREADS", "1", 1); + assert(https_pool_init() == 0); + + /* Wrap the ring at its initial size first: a few small batches that the + * single worker drains between submits move head/tail off slot 0. */ + for (int r = 0; r < 3; r++) { + long before = atomic_load(&g_seen_n); + submit_round(before, 700); + while (atomic_load(&g_seen_n) < before + 700) usleep(1000); + } + long base = atomic_load(&g_seen_n); + + /* Hold the only worker, then queue far past the initial capacity. */ + https_pool_submit_conn(fake_conn(), NULL, blocker, NULL); + while (!atomic_load(&g_blocker_running)) usleep(1000); + submit_round(base, TASKS - base); + + pthread_mutex_lock(&g_gate_mu); + g_gate_open = true; + pthread_cond_broadcast(&g_gate_cv); + pthread_mutex_unlock(&g_gate_mu); + while (atomic_load(&g_seen_n) < TASKS) usleep(1000); + + for (long i = 0; i < TASKS; i++) { + if (g_seen[i] != i) { + fprintf(stderr, "task %ld ran as #%ld\n", g_seen[i], i); + return 1; + } + } + https_pool_destroy(); + printf("test_https_pool_queue: %d tasks ran once each, in order\n", TASKS); + return 0; +}