From 2f8573c3811ac6efb0265ee3239789670a893669 Mon Sep 17 00:00:00 2001 From: corneliusroemer-agent Date: Thu, 10 Sep 2026 10:10:45 +0000 Subject: [PATCH 1/2] fix(deployment): let the LAPIS readiness probe ride out a brief SILO timeout /sample/info calls SILO live with a hard-coded 100ms budget, so a moment of load on the node makes the probe fail. Three failures in a row evicted the pod from its Service, and with one replica the ingress then had no backend and answered 503 while SILO was perfectly healthy. Eviction latency is periodSeconds x failureThreshold: 30s before, which loses to the ~40s failure burst measured in run 33877810215. This makes it 60s, and the shorter period halves how long recovery takes. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Lk4J5vNvAC1X6KZ1JsWhar --- kubernetes/loculus/templates/lapis-deployment.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kubernetes/loculus/templates/lapis-deployment.yaml b/kubernetes/loculus/templates/lapis-deployment.yaml index 4119e39368..cc5201d606 100644 --- a/kubernetes/loculus/templates/lapis-deployment.yaml +++ b/kubernetes/loculus/templates/lapis-deployment.yaml @@ -56,8 +56,9 @@ spec: httpGet: path: /sample/info port: 8080 - periodSeconds: 10 - failureThreshold: 3 + # 60s of continuous failure before eviction; LAPIS gives its SILO call only 100ms + periodSeconds: 5 + failureThreshold: 12 timeoutSeconds: 5 livenessProbe: httpGet: From 7a58311307d2a4722b70b09e45ef2018ef554b12 Mon Sep 17 00:00:00 2001 From: Cornelius Roemer Date: Thu, 10 Sep 2026 12:20:10 +0200 Subject: [PATCH 2/2] Apply suggestion from @corneliusroemer --- kubernetes/loculus/templates/lapis-deployment.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/kubernetes/loculus/templates/lapis-deployment.yaml b/kubernetes/loculus/templates/lapis-deployment.yaml index cc5201d606..2e471bd0a4 100644 --- a/kubernetes/loculus/templates/lapis-deployment.yaml +++ b/kubernetes/loculus/templates/lapis-deployment.yaml @@ -56,7 +56,6 @@ spec: httpGet: path: /sample/info port: 8080 - # 60s of continuous failure before eviction; LAPIS gives its SILO call only 100ms periodSeconds: 5 failureThreshold: 12 timeoutSeconds: 5