From 1d791baf5fbeef6f1d66784d060b29c7b1eb573a Mon Sep 17 00:00:00 2001 From: Brandon Chatham Date: Wed, 7 Oct 2026 02:26:27 +0000 Subject: [PATCH 1/2] test(integration): give saturation followers a post-load catch-up window Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- test/integration/seiload_test.go | 42 +++++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/test/integration/seiload_test.go b/test/integration/seiload_test.go index ccd7b609..946252e0 100644 --- a/test/integration/seiload_test.go +++ b/test/integration/seiload_test.go @@ -219,6 +219,10 @@ const ( // followerMaxLag is how many blocks a follower may trail the validators' // head after load: read skew between two RPCs, not a stall. followerMaxLag = 10 + // followerCatchUpTimeout bounds how long a follower that fell behind under + // load has to return within followerMaxLag before it counts as stalled. + followerCatchUpTimeout = 5 * time.Minute + followerCatchUpPoll = 10 * time.Second // maxRevertRatio is the share of executed transactions that may revert // before a run stops describing its workload. maxRevertRatio = 0.01 @@ -227,9 +231,10 @@ const ( ) // assertChainLive fails unless the validators still produce blocks and every -// follower sits within followerMaxLag of their head. catching_up is a one-way -// latch that a follower stalled after its first catch-up never re-flips, so -// heights are compared directly. +// follower returns within followerMaxLag of their head inside +// followerCatchUpTimeout. catching_up is a one-way latch that a follower +// stalled after its first catch-up never re-flips, so heights are compared +// directly. func assertChainLive(ctx context.Context, t *testing.T, hc *http.Client, ch *chain) { t.Helper() validators := ch.network.TendermintRPC() @@ -239,12 +244,31 @@ func assertChainLive(ctx context.Context, t *testing.T, hc *http.Client, ch *cha t.Errorf("post-load validators halted: %v", err) return } - head := mustLatestHeight(ctx, t, hc, validators, "post-load validator") - for _, n := range ch.rpcNodes { - h := mustLatestHeight(ctx, t, hc, n.TendermintRPC(), "post-load "+n.Name()) - if lag := head - h; lag > followerMaxLag { - t.Errorf("post-load %s at height %d trails the validator head %d by %d blocks (> %d)", - n.Name(), h, head, lag, followerMaxLag) + deadline := time.Now().Add(followerCatchUpTimeout) + for { + head := mustLatestHeight(ctx, t, hc, validators, "post-load validator") + var lagging []string + for _, n := range ch.rpcNodes { + h := mustLatestHeight(ctx, t, hc, n.TendermintRPC(), "post-load "+n.Name()) + if lag := head - h; lag > followerMaxLag { + lagging = append(lagging, fmt.Sprintf("%s at height %d trails the validator head %d by %d blocks (> %d)", + n.Name(), h, head, lag, followerMaxLag)) + } + } + if len(lagging) == 0 { + return + } + if time.Now().After(deadline) { + for _, l := range lagging { + t.Errorf("post-load %s after %s", l, followerCatchUpTimeout) + } + return + } + select { + case <-ctx.Done(): + t.Errorf("post-load follower catch-up: %v", ctx.Err()) + return + case <-time.After(followerCatchUpPoll): } } } From 8a64e6cfa012db546db0809f3d1ef60bf8632c2f Mon Sep 17 00:00:00 2001 From: Brandon Chatham Date: Wed, 7 Oct 2026 02:29:50 +0000 Subject: [PATCH 2/2] test(integration): recheck validator liveness after follower catch-up; report ctx end directly Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- test/integration/seiload_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/integration/seiload_test.go b/test/integration/seiload_test.go index 946252e0..743e40d1 100644 --- a/test/integration/seiload_test.go +++ b/test/integration/seiload_test.go @@ -246,6 +246,10 @@ func assertChainLive(ctx context.Context, t *testing.T, hc *http.Client, ch *cha } deadline := time.Now().Add(followerCatchUpTimeout) for { + if err := ctx.Err(); err != nil { + t.Errorf("post-load follower catch-up: %v", err) + return + } head := mustLatestHeight(ctx, t, hc, validators, "post-load validator") var lagging []string for _, n := range ch.rpcNodes { @@ -256,6 +260,11 @@ func assertChainLive(ctx context.Context, t *testing.T, hc *http.Client, ch *cha } } if len(lagging) == 0 { + advanceCtx, cancel := context.WithTimeout(ctx, time.Minute) + defer cancel() + if err := sei.WaitHeightAdvances(advanceCtx, hc, validators, 2); err != nil { + t.Errorf("post-load validators halted during follower catch-up: %v", err) + } return } if time.Now().After(deadline) {