fix: reach READY in headless kubernetes deployments

.github/workflows/k8s-integration.yml

@@ -0,0 +1,25 @@
+name: k8s integration
+
+# Runs HSDS headless (no head node) on a real k3s cluster and asserts the
+# service nodes actually reach READY and serve requests. Unit tests cannot see
+# this failure mode: it only appears when service nodes discover data nodes
+# through the Kubernetes API, and it presents as healthy pods that 503.
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+  workflow_dispatch:
+
+jobs:
+  headless-k8s:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install k3d
+        run: curl -sSL https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash
+
+      - name: Run headless k8s readiness test
+        run: tests/k8s/run.sh 3

admin/kubernetes/k8s_deployment_aws.yml

@@ -59,12 +59,22 @@ spec:
                 secretKeyRef:
                   name: aws-auth-keys
                   key: aws_secret_access_key
+          # asserts node_state, not merely that the process answers: /info is in
+          # INFO_METHODS (hsds_logger.request) and bypasses the node_state gate, so an
+          # httpGet against it returns 200 even while the node is wedged in WAITING and
+          # 503-ing every real request. hsds-node-state takes its port from NODE_TYPE.
           livenessProbe:
-            httpGet:
-              path: /info
-              port: 5101
-            initialDelaySeconds: 5
+            exec:
+              command: ["hsds-node-state"]
+            initialDelaySeconds: 30
             periodSeconds: 60
+            failureThreshold: 5
+          readinessProbe:
+            exec:
+              command: ["hsds-node-state"]
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            failureThreshold: 3
         - name: dn
           image: hdfgroup/hsds:0.7.0
           imagePullPolicy: IfNotPresent
@@ -102,12 +112,22 @@ spec:
                 secretKeyRef:
                   name: aws-auth-keys
                   key: aws_secret_access_key
+          # asserts node_state, not merely that the process answers: /info is in
+          # INFO_METHODS (hsds_logger.request) and bypasses the node_state gate, so an
+          # httpGet against it returns 200 even while the node is wedged in WAITING and
+          # 503-ing every real request. hsds-node-state takes its port from NODE_TYPE.
           livenessProbe:
-            httpGet:
-              path: /info
-              port: 6101
-            initialDelaySeconds: 5
+            exec:
+              command: ["hsds-node-state"]
+            initialDelaySeconds: 30
             periodSeconds: 60
+            failureThreshold: 5
+          readinessProbe:
+            exec:
+              command: ["hsds-node-state"]
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            failureThreshold: 3
       volumes:
         - name: accounts
           secret:

admin/kubernetes/k8s_deployment_azure.yml

@@ -57,12 +57,22 @@ spec:
             valueFrom:
               fieldRef:
                 fieldPath: status.podIP
+          # asserts node_state, not merely that the process answers: /info is in
+          # INFO_METHODS (hsds_logger.request) and bypasses the node_state gate, so an
+          # httpGet against it returns 200 even while the node is wedged in WAITING and
+          # 503-ing every real request. hsds-node-state takes its port from NODE_TYPE.
           livenessProbe:
-            httpGet:
-              path: /info
-              port: 5101
-            initialDelaySeconds: 5
+            exec:
+              command: ["hsds-node-state"]
+            initialDelaySeconds: 30
             periodSeconds: 60
+            failureThreshold: 5
+          readinessProbe:
+            exec:
+              command: ["hsds-node-state"]
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            failureThreshold: 3
         - name: dn
           image: hdfgroup/hsds:0.7.0
           imagePullPolicy: IfNotPresent 
@@ -96,12 +106,22 @@ spec:
               secretKeyRef:
                 name: azure-conn-str
                 key: az_conn_str
+          # asserts node_state, not merely that the process answers: /info is in
+          # INFO_METHODS (hsds_logger.request) and bypasses the node_state gate, so an
+          # httpGet against it returns 200 even while the node is wedged in WAITING and
+          # 503-ing every real request. hsds-node-state takes its port from NODE_TYPE.
           livenessProbe:
-            httpGet:
-              path: /info
-              port: 6101
-            initialDelaySeconds: 5
+            exec:
+              command: ["hsds-node-state"]
+            initialDelaySeconds: 30
             periodSeconds: 60
+            failureThreshold: 5
+          readinessProbe:
+            exec:
+              command: ["hsds-node-state"]
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            failureThreshold: 3
       volumes:
       - name: accounts
         secret:

admin/kubernetes/k8s_deployment_posix.yml

@@ -44,12 +44,22 @@ spec:
               value: sn
             - name: HEAD_PORT
               value: "0" # no head container
+          # asserts node_state, not merely that the process answers: /info is in
+          # INFO_METHODS (hsds_logger.request) and bypasses the node_state gate, so an
+          # httpGet against it returns 200 even while the node is wedged in WAITING and
+          # 503-ing every real request. hsds-node-state takes its port from NODE_TYPE.
           livenessProbe:
-            httpGet:
-              path: /info
-              port: 5101
-            initialDelaySeconds: 5
+            exec:
+              command: ["hsds-node-state"]
+            initialDelaySeconds: 30
             periodSeconds: 60
+            failureThreshold: 5
+          readinessProbe:
+            exec:
+              command: ["hsds-node-state"]
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            failureThreshold: 3
         - name: dn
           image: hdfgroup/hsds:0.7.0
           imagePullPolicy: IfNotPresent
@@ -74,12 +84,22 @@ spec:
               value: dn
             - name: HEAD_PORT
               value: "0" # no head container
+          # asserts node_state, not merely that the process answers: /info is in
+          # INFO_METHODS (hsds_logger.request) and bypasses the node_state gate, so an
+          # httpGet against it returns 200 even while the node is wedged in WAITING and
+          # 503-ing every real request. hsds-node-state takes its port from NODE_TYPE.
           livenessProbe:
-            httpGet:
-              path: /info
-              port: 6101
-            initialDelaySeconds: 5
+            exec:
+              command: ["hsds-node-state"]
+            initialDelaySeconds: 30
             periodSeconds: 60
+            failureThreshold: 5
+          readinessProbe:
+            exec:
+              command: ["hsds-node-state"]
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            failureThreshold: 3
       volumes:
         - name: data
           hostPath:

hsds/basenode.py

@@ -160,6 +160,18 @@ async def k8s_update_dn_info(app):
     elif app["dn_urls"] != dn_urls:
         log.info(f"pod ips have changed: {dn_urls}, fetch dn_ids")
         scale_update = True
+    elif app.get("cluster_state") != "READY":
+        # the roster was incomplete last pass - e.g. a dn had not yet assigned
+        # itself a node_number, so it reported -1. Re-fetch until it converges;
+        # keying only off dn_urls changes leaves the cluster wedged in WAITING
+        # forever once the pod set goes stable.
+        #
+        # Costs one /info request per dn per health check, so n^2 across the cluster,
+        # but only while not READY. A cluster that never converges therefore polls
+        # indefinitely where it previously checked once; add backoff here if that
+        # becomes a problem at larger node counts.
+        log.info("cluster_state is not READY, re-fetching dn_ids")
+        scale_update = True
     else:
         scale_update = False
 
@@ -210,14 +222,34 @@ async def k8s_update_dn_info(app):
         log.info(f"scaling - updating dn_ids to: {dn_ids}")
         app["dn_ids"] = dn_ids
 
+        # Any partial view sets WAITING, so a rescale briefly returns 503 until the
+        # roster reconverges - about one health check interval. That is the trade this
+        # gate makes: nodes holding different dn rosters compute different partitions
+        # for the same obj_id, so serving through the churn risks inconsistent reads
+        # rather than a short unavailability.
         if len(dn_ids) != new_count:
             log.warn(f"scaling - got {len(dn_ids)} dn_ids expected {new_count}")
+            app["cluster_state"] = "WAITING"
         elif len(dn_node_numbers) != len(dn_urls):
             log.warn(f"scaling - got {len(dn_node_numbers)} node numbers, expected {new_count}")
+            app["cluster_state"] = "WAITING"
         elif not consecutive:
             log.warn(f"scaling - node_numbers not consecutive - got: {dn_node_numbers}")
+            app["cluster_state"] = "WAITING"
+        elif min_node_count != len(dn_urls) or max_node_count != len(dn_urls):
+            msg = "scaling - dn node_counts have not converged, got range: "
+            msg += f"{min_node_count}-{max_node_count}, expected: {len(dn_urls)}"
+            log.warn(msg)
+            app["cluster_state"] = "WAITING"
         else:
             log.info("scaling - node numbers complete")
+            # No head node exists in this mode to report cluster_state, so derive it
+            # from the dn roster. This deliberately covers only the dn dimension of
+            # isClusterReady(): that function also compares the sn count against
+            # target_sn_count, which is not checked here. getObjPartition() partitions
+            # by dn count, so dn completeness is what decides whether nodes agree on
+            # partitioning - an sn that is not up is simply not serving.
+            app["cluster_state"] = "READY"
 
 
 async def docker_update_dn_info(app):

hsds/node_state.py

@@ -0,0 +1,59 @@
+##############################################################################
+# Copyright by The HDF Group.                                                #
+# All rights reserved.                                                       #
+#                                                                            #
+# This file is part of HSDS (HDF5 Scalable Data Service), Libraries and      #
+# Utilities.  The full HSDS copyright notice, including                      #
+# terms governing use, modification, and redistribution, is contained in     #
+# the file COPYING, which can be found at the root of the source code        #
+# distribution tree.  If you do not have access to this file, you may        #
+# request a copy from help@hdfgroup.org.                                     #
+##############################################################################
+
+"""Print this node's state and exit 0 only when it is READY.
+
+Intended as a container liveness/readiness probe. /info is listed in
+INFO_METHODS in hsds_logger.request and so bypasses the node_state gate, which
+means an httpGet probe against it returns 200 even while the node is stuck in
+WAITING and returning 503 to every real request. A probe that distinguishes a
+serving node from a wedged one therefore has to inspect node_state itself.
+
+The port is derived from NODE_TYPE, so the sn and dn containers can share one
+identical probe command rather than each hardcoding a port.
+
+Exit codes:
+    0  node reports READY
+    1  node reports any other state, or /info could not be reached
+    2  NODE_TYPE has no configured port (misconfiguration, not a wedged node)
+"""
+
+import json
+import os
+import sys
+import urllib.request
+
+from . import config
+
+
+def main():
+    node_type = os.environ.get("NODE_TYPE") or "sn"
+    port = config.get(f"{node_type}_port")
+    if not port:
+        print(f"no port configured for NODE_TYPE={node_type}", file=sys.stderr)
+        return 2
+
+    url = f"http://localhost:{port}/info"
+    try:
+        with urllib.request.urlopen(url, timeout=10) as rsp:
+            state = json.load(rsp)["node"]["state"]
+    except Exception as e:
+        print(f"{url}: {e}", file=sys.stderr)
+        return 1
+
+    # stdout is the state itself, so the same command is useful for diagnostics
+    print(state)
+    return 0 if state == "READY" else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

pyproject.toml

@@ -94,4 +94,5 @@ hsds-servicenode = "hsds.servicenode:main"
 hsds-headnode = "hsds.headnode:main"
 hsds-node = "hsds.node_runner:main"
 hsds-chunklocator = "hsds.chunklocator:main"
+hsds-node-state = "hsds.node_state:main"