feat(seinode): maintenance hold for nodeConfig nodes (spec 010, PLT-1390)

api/v1alpha1/seinode_types.go

@@ -97,6 +97,7 @@ import (
 // and it needs nodeConfig, because only that node's plans run the reset.
 // +kubebuilder:validation:XValidation:rule="!has(self.dataResetGeneration) || has(self.nodeConfig)",message="spec.dataResetGeneration needs spec.nodeConfig: only a node that reads its config from ConfigMaps runs the declarative data reset"
 // +kubebuilder:validation:XValidation:rule="!has(oldSelf.dataResetGeneration) || (has(self.dataResetGeneration) && self.dataResetGeneration >= oldSelf.dataResetGeneration)",message="spec.dataResetGeneration can only increase: lowering or removing it would rearm a data wipe; to undo a reset commit, revert the config and keep the counter"
+// +kubebuilder:validation:XValidation:rule="!has(self.maintenance) || !has(self.maintenance.hold) || has(self.nodeConfig)",message="spec.maintenance.hold needs spec.nodeConfig: only a node that reads its config from ConfigMaps runs the maintenance hold"
 type SeiNodeSpec struct {
 	// ChainID of the chain this node belongs to.
 	// Constrained to DNS-1123 label characters because the controller composes
@@ -286,6 +287,44 @@ type SeiNodeSpec struct {
 	// +kubebuilder:validation:Minimum=0
 	// +optional
 	DataResetGeneration int64 `json:"dataResetGeneration,omitempty"`
+
+	// Maintenance holds seid at the sidecar start gate with the pod alive and
+	// the data volume mounted, for work through kubectl exec. Requires
+	// spec.nodeConfig.
+	// +optional
+	Maintenance *MaintenanceSpec `json:"maintenance,omitempty"`
+}
+
+// MaintenanceHold is a maintenance hold mode.
+// +kubebuilder:validation:Enum=Immediate;AfterExit
+type MaintenanceHold string
+
+const (
+	// MaintenanceHoldImmediate closes the start gate and stops seid now.
+	MaintenanceHoldImmediate MaintenanceHold = "Immediate"
+	// MaintenanceHoldAfterExit closes the start gate and leaves seid running;
+	// when seid exits on its own, for example at halt-height, it parks. On a
+	// node parked by an Immediate hold it starts seid once first.
+	MaintenanceHoldAfterExit MaintenanceHold = "AfterExit"
+)
+
+// MaintenanceSpec is the maintenance request on a SeiNode.
+type MaintenanceSpec struct {
+	// Hold keeps seid from starting while it is set. Immediate stops seid now;
+	// AfterExit lets it run until it exits. Remove it to release seid. While
+	// held, the controller still applies the StatefulSet, so a template change
+	// rolls the pod and the new pod stays parked. A hold set on a new node parks
+	// it before seid first runs.
+	// +optional
+	Hold MaintenanceHold `json:"hold,omitempty"`
+}
+
+// HoldRequested returns the requested maintenance hold, or "" for none.
+func (s *SeiNodeSpec) HoldRequested() MaintenanceHold {
+	if s.Maintenance == nil {
+		return ""
+	}
+	return s.Maintenance.Hold
 }
 
 // Resources overrides the seid-container footprint in pod-resource shape.
@@ -754,6 +793,12 @@ const (
 	// a merge it can still describe the previous reset.
 	ConditionDataResetInProgress = "DataResetInProgress"
 
+	// ConditionMaintenanceInProgress reports the maintenance hold
+	// (spec.maintenance.hold). InProgress-style and always-present: False is
+	// the steady state. True/Held means seid is parked, so exec work on the data
+	// is safe.
+	ConditionMaintenanceInProgress = "MaintenanceInProgress"
+
 	// ConditionDataVolumeResizeInProgress reports whether the node's data volume
 	// is still growing toward spec.dataVolume.storage.resources.requests.storage.
 	// InProgress-style and always-present: True is the exception, False the
@@ -764,6 +809,23 @@ const (
 	ConditionDataVolumeResizeInProgress = "DataVolumeResizeInProgress"
 )
 
+// Reasons for the MaintenanceInProgress condition. Stable enum (public API for
+// alerting/runbooks per CLAUDE.md "Conditions").
+const (
+	// ReasonMaintenanceNotApplicable: the node has no spec.nodeConfig.
+	ReasonMaintenanceNotApplicable = "NotApplicable"
+	// ReasonNotHeld: no hold is requested or in effect.
+	ReasonNotHeld = "NotHeld"
+	// ReasonHoldPending: the requested hold differs from the hold in effect,
+	// and its plan has not finished.
+	ReasonHoldPending = "HoldPending"
+	// ReasonHeld: an Immediate hold is in effect; seid is parked.
+	ReasonHeld = "Held"
+	// ReasonArmed: an AfterExit hold is in effect; the gate is closed and seid
+	// parks at its next exit.
+	ReasonArmed = "Armed"
+)
+
 // Reasons for the DataResetInProgress condition. Stable enum (public API for
 // alerting/runbooks per CLAUDE.md "Conditions").
 const (
@@ -1031,6 +1093,14 @@ type SeiNodeStatus struct {
 	// the spec value, so creating a node never wipes it.
 	// +optional
 	DataResetGeneration int64 `json:"dataResetGeneration,omitempty"`
+
+	// MaintenanceHold is the hold in effect: Immediate means seid is parked by
+	// the hold, AfterExit means the start gate is closed and seid may still run,
+	// empty means no hold acts on the node. The controller writes it when a hold,
+	// release, or held reset plan completes, and compares it with
+	// spec.maintenance.hold to decide the next plan.
+	// +optional
+	MaintenanceHold MaintenanceHold `json:"maintenanceHold,omitempty"`
 }
 
 // NodeEndpointStatus carries the in-cluster URLs this SeiNode serves, derived

config/crd/sei.io_seinodes.yaml

@@ -591,6 +591,24 @@ spec:
                 maxLength: 512
                 minLength: 1
                 type: string
+              maintenance:
+                description: |-
+                  Maintenance holds seid at the sidecar start gate with the pod alive and
+                  the data volume mounted, for work through kubectl exec. Requires
+                  spec.nodeConfig.
+                properties:
+                  hold:
+                    description: |-
+                      Hold keeps seid from starting while it is set. Immediate stops seid now;
+                      AfterExit lets it run until it exits. Remove it to release seid. While
+                      held, the controller still applies the StatefulSet, so a template change
+                      rolls the pod and the new pod stays parked. A hold set on a new node parks
+                      it before seid first runs.
+                    enum:
+                    - Immediate
+                    - AfterExit
+                    type: string
+                type: object
               nodeConfig:
                 description: |-
                   NodeConfig supplies this node's seid config files from existing
@@ -1496,6 +1514,9 @@ spec:
                 and keep the counter'
               rule: '!has(oldSelf.dataResetGeneration) || (has(self.dataResetGeneration)
                 && self.dataResetGeneration >= oldSelf.dataResetGeneration)'
+            - message: 'spec.maintenance.hold needs spec.nodeConfig: only a node that
+                reads its config from ConfigMaps runs the maintenance hold'
+              rule: '!has(self.maintenance) || !has(self.maintenance.hold) || has(self.nodeConfig)'
           status:
             description: SeiNodeStatus defines the observed state of a SeiNode.
             properties:
@@ -1677,6 +1698,17 @@ spec:
                       leave the listener closed.
                     type: string
                 type: object
+              maintenanceHold:
+                description: |-
+                  MaintenanceHold is the hold in effect: Immediate means seid is parked by
+                  the hold, AfterExit means the start gate is closed and seid may still run,
+                  empty means no hold acts on the node. The controller writes it when a hold,
+                  release, or held reset plan completes, and compares it with
+                  spec.maintenance.hold to decide the next plan.
+                enum:
+                - Immediate
+                - AfterExit
+                type: string
               phase:
                 description: Phase is the high-level lifecycle state.
                 enum:

docs/specs/010-maintenance-hold/spec.md

@@ -228,7 +228,7 @@ reaches `Running` with seid parked.
 #### Acceptance Criteria
 
 1. WHEN the hold is removed from a held node and no reset is pending, THE controller SHALL build a plan that marks the sidecar ready.
-2. WHEN the hold is removed and a reset is pending, THE controller SHALL build the reset plan with its final `mark-ready`.
+2. WHEN the hold is removed and a reset is pending, THE controller SHALL build the reset plan with its `mark-ready`, followed by a step that clears `status.maintenanceHold`.
 3. WHEN the plan that marks the sidecar ready completes, THE controller SHALL clear `status.maintenanceHold`.
 
 ### Requirement 5: The controller reports the hold
@@ -240,8 +240,8 @@ reaches `Running` with seid parked.
 #### Acceptance Criteria
 
 1. The controller SHALL seed a `MaintenanceInProgress` condition on every SeiNode. `True` is the exception, `False` is the steady state.
-2. WHILE no hold is set, THE condition SHALL be `False` with reason `NotHeld`, or `NotApplicable` on a node without `spec.nodeConfig`.
-3. WHILE a hold is set and differs from `status.maintenanceHold`, WHILE a hold plan runs, or WHILE the sidecar reports the gate open under a hold, THE condition SHALL be `True` with reason `HoldPending`.
+2. WHILE no hold is set and none is in effect, THE condition SHALL be `False` with reason `NotHeld`, or `NotApplicable` on a node without `spec.nodeConfig`.
+3. WHILE the hold set differs from `status.maintenanceHold` (a removed hold included), WHILE a plan that changes the hold in effect runs, or WHILE the sidecar reports the gate open under a hold, THE condition SHALL be `True` with reason `HoldPending`.
 4. WHILE an `Immediate` hold is in effect, THE condition SHALL be `True` with reason `Held`.
 5. WHILE an `AfterExit` hold is in effect, THE condition SHALL be `True` with reason `Armed`.
 6. WHEN the hold takes effect or the node is released, THE controller SHALL record an event on the SeiNode.

internal/controller/node/controller.go

@@ -130,13 +130,8 @@ func (r *SeiNodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (re
 	observedPhase := node.Status.Phase
 	prevSidecar := apimeta.FindStatusCondition(node.Status.Conditions, seiv1alpha1.ConditionSidecarReady)
 	prevStateSync := apimeta.FindStatusCondition(node.Status.Conditions, seiv1alpha1.ConditionStateSyncReady)
-	// A copy, not the pointer FindStatusCondition returns: SetStatusCondition
-	// mutates the slice element in place, which would erase the transition.
-	var prevDataReset *metav1.Condition
-	if c := apimeta.FindStatusCondition(node.Status.Conditions, seiv1alpha1.ConditionDataResetInProgress); c != nil {
-		cp := *c
-		prevDataReset = &cp
-	}
+	prevDataReset := conditionSnapshot(node, seiv1alpha1.ConditionDataResetInProgress)
+	prevMaintenance := conditionSnapshot(node, seiv1alpha1.ConditionMaintenanceInProgress)
 
 	setNodePausedCondition(node)
 
@@ -253,6 +248,7 @@ func (r *SeiNodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (re
 	// it: spec-derived plus the persisted plan, so it rides the flush on every
 	// path, including Paused, where a pending reset waits.
 	planner.ResolveDataReset(node)
+	planner.ResolveMaintenance(node)
 
 	// Failed is terminal — flush any condition updates and exit.
 	if node.Status.Phase == seiv1alpha1.PhaseFailed {
@@ -344,6 +340,7 @@ func (r *SeiNodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (re
 	}
 
 	r.emitDataResetEvent(node, prevDataReset)
+	r.emitMaintenanceEvent(node, prevMaintenance)
 	r.observeCommittedHeight(ctx, node, suppressDrift)
 
 	if err := flushStatus(); err != nil {
@@ -654,3 +651,36 @@ func (r *SeiNodeReconciler) emitDataResetEvent(node *seiv1alpha1.SeiNode, prev *
 		r.Recorder.Event(node, corev1.EventTypeWarning, "DataResetFailed", cur.Message)
 	}
 }
+
+// conditionSnapshot copies a condition, or returns nil when it is absent. A
+// copy, not the pointer FindStatusCondition returns: SetStatusCondition mutates
+// the slice element in place, which would erase the transition.
+func conditionSnapshot(node *seiv1alpha1.SeiNode, condType string) *metav1.Condition {
+	c := apimeta.FindStatusCondition(node.Status.Conditions, condType)
+	if c == nil {
+		return nil
+	}
+	cp := *c
+	return &cp
+}
+
+// emitMaintenanceEvent records when a hold takes effect and when seid is
+// released (spec 010 Requirement 5).
+func (r *SeiNodeReconciler) emitMaintenanceEvent(node *seiv1alpha1.SeiNode, prev *metav1.Condition) {
+	cur := apimeta.FindStatusCondition(node.Status.Conditions, seiv1alpha1.ConditionMaintenanceInProgress)
+	if cur == nil || r.Recorder == nil || (prev != nil && prev.Reason == cur.Reason) {
+		return
+	}
+	switch cur.Reason {
+	case seiv1alpha1.ReasonHeld:
+		r.Recorder.Event(node, corev1.EventTypeNormal, "MaintenanceHeld", cur.Message)
+	case seiv1alpha1.ReasonArmed:
+		r.Recorder.Event(node, corev1.EventTypeNormal, "MaintenanceArmed", cur.Message)
+	case seiv1alpha1.ReasonNotHeld:
+		// A release runs a plan, so the condition passes through HoldPending
+		// on its way to NotHeld; any True state before NotHeld is a release.
+		if prev != nil && prev.Status == metav1.ConditionTrue {
+			r.Recorder.Event(node, corev1.EventTypeNormal, "MaintenanceReleased", cur.Message)
+		}
+	}
+}

internal/controller/node/datareset_events_test.go

@@ -46,3 +46,38 @@ func TestEmitDataResetEvent(t *testing.T) {
 		})
 	}
 }
+
+// 010 Req 5.6 (harbor e2e finding): a release reaches NotHeld from HoldPending,
+// because the release plan runs first, and still records MaintenanceReleased.
+func TestEmitMaintenanceEvent(t *testing.T) {
+	cond := func(status metav1.ConditionStatus, reason string) *metav1.Condition {
+		return &metav1.Condition{Type: seiv1alpha1.ConditionMaintenanceInProgress, Status: status, Reason: reason, Message: reason}
+	}
+	cases := []struct {
+		name      string
+		prev, cur *metav1.Condition
+		want      string
+	}{
+		{"held", cond(metav1.ConditionTrue, seiv1alpha1.ReasonHoldPending), cond(metav1.ConditionTrue, seiv1alpha1.ReasonHeld), "MaintenanceHeld"},
+		{"armed", cond(metav1.ConditionTrue, seiv1alpha1.ReasonHoldPending), cond(metav1.ConditionTrue, seiv1alpha1.ReasonArmed), "MaintenanceArmed"},
+		{"released via HoldPending", cond(metav1.ConditionTrue, seiv1alpha1.ReasonHoldPending), cond(metav1.ConditionFalse, seiv1alpha1.ReasonNotHeld), "MaintenanceReleased"},
+		{"released from Held", cond(metav1.ConditionTrue, seiv1alpha1.ReasonHeld), cond(metav1.ConditionFalse, seiv1alpha1.ReasonNotHeld), "MaintenanceReleased"},
+		{"seeded, never held", nil, cond(metav1.ConditionFalse, seiv1alpha1.ReasonNotHeld), ""},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			g := NewWithT(t)
+			rec := record.NewFakeRecorder(4)
+			r := &SeiNodeReconciler{Recorder: rec}
+			node := &seiv1alpha1.SeiNode{}
+			apimeta.SetStatusCondition(&node.Status.Conditions, *tc.cur)
+
+			r.emitMaintenanceEvent(node, tc.prev)
+			if tc.want == "" {
+				g.Expect(rec.Events).To(BeEmpty())
+				return
+			}
+			g.Expect(rec.Events).To(Receive(ContainSubstring(tc.want)))
+		})
+	}
+}

internal/controller/node/envtest/maintenance_validation_test.go

@@ -0,0 +1,63 @@
+//go:build envtest
+
+package envtest_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/gomega"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+
+	seiv1alpha1 "github.com/sei-protocol/sei-k8s-controller/api/v1alpha1"
+)
+
+// Admission coverage of spec.maintenance.hold (spec 010 Requirement 1,
+// SC-001). These cases need no controller.
+
+// 010 Req 1.1: both values are accepted on a nodeConfig node.
+func TestMaintenanceHold_Values_Accepted(t *testing.T) {
+	for _, hold := range []seiv1alpha1.MaintenanceHold{seiv1alpha1.MaintenanceHoldImmediate, seiv1alpha1.MaintenanceHoldAfterExit} {
+		t.Run(string(hold), func(t *testing.T) {
+			g := NewWithT(t)
+			ns := makeNamespace(t)
+			node := nodeConfigNode(ns, "hold-ok")
+			node.Spec.Maintenance = &seiv1alpha1.MaintenanceSpec{Hold: hold}
+			g.Expect(testCli.Create(testCtx, node)).To(Succeed())
+		})
+	}
+}
+
+// 010 Req 1.2: a hold needs spec.nodeConfig.
+func TestMaintenanceHold_RequiresNodeConfig(t *testing.T) {
+	g := NewWithT(t)
+	ns := makeNamespace(t)
+	node := nodeConfigNode(ns, "hold-no-nc")
+	node.Spec.NodeConfig = nil
+	node.Spec.Maintenance = &seiv1alpha1.MaintenanceSpec{Hold: seiv1alpha1.MaintenanceHoldImmediate}
+
+	err := testCli.Create(testCtx, node)
+	g.Expect(err).To(HaveOccurred())
+	g.Expect(err.Error()).To(ContainSubstring("needs spec.nodeConfig"))
+}
+
+// 010 Req 1.1: any other value is rejected by the enum.
+func TestMaintenanceHold_UnknownValue_Rejected(t *testing.T) {
+	g := NewWithT(t)
+	ns := makeNamespace(t)
+	obj := &unstructured.Unstructured{Object: map[string]any{
+		"apiVersion": "sei.io/v1alpha1",
+		"kind":       "SeiNode",
+		"metadata":   map[string]any{"name": "hold-bad", "namespace": ns},
+		"spec": map[string]any{
+			"chainId":  "envtest-1",
+			"image":    "sei:latest",
+			"fullNode": map[string]any{},
+			"nodeConfig": map[string]any{
+				"configRef": map[string]any{"name": "rpc-config-v1"},
+				"appRef":    map[string]any{"name": "rpc-app-v1"},
+			},
+			"maintenance": map[string]any{"hold": "Forever"},
+		},
+	}}
+	g.Expect(testCli.Create(testCtx, obj)).To(HaveOccurred())
+}