From 546c26ea2c2afcbc0e9f62f4a52487fdb926d38a Mon Sep 17 00:00:00 2001 From: John Ramsden Date: Fri, 18 Sep 2026 09:55:36 -0700 Subject: [PATCH 1/2] feat(db): add a table for each member's last applied RGW frontend This is scaffolding: the table and its helpers land now, unused, so later commits in this series build on a settled schema instead of bundling a migration with the feature that needs it. Each member applies its own RGW settings, so the cluster needs one place that remembers what every member last applied, including while that member is down. schemaUpdate10 adds the rgw_frontends table with the columns member_id, port, ssl_port and ssl. It never holds certificate material, so leaking a row leaks nothing secret, and a row is deleted together with its core_cluster_members row so it can never outlive the member it describes. UpsertRGWFrontend and DeleteRGWFrontendByMember will be called from the member-side apply and disable paths added later in this series, to record and clear a member's settings as they change. GetRGWFrontends will be used by the placement status read path added later, to report what each member is actually serving. The three helpers are hand-written rather than generated by the lxd-generate mapper because every caller identifies a member by name, not by its internal database id, and the mapper codegen does not support that lookup. Co-authored-by: Peter Sabaini Assisted-by: claude-code:claude-fable-5-1 Assisted-by: pi:gpt-6-astra Signed-off-by: John Ramsden --- microceph/database/rgw_frontend.go | 67 +++++++ .../database/rgw_frontend_extras_test.go | 188 ++++++++++++++++++ microceph/database/schema.go | 20 ++ 3 files changed, 275 insertions(+) create mode 100644 microceph/database/rgw_frontend.go create mode 100644 microceph/database/rgw_frontend_extras_test.go diff --git a/microceph/database/rgw_frontend.go b/microceph/database/rgw_frontend.go new file mode 100644 index 000000000..6cc59f298 --- /dev/null +++ b/microceph/database/rgw_frontend.go @@ -0,0 +1,67 @@ +package database + +import ( + "context" + "database/sql" + "fmt" +) + +// RgwFrontend holds a member's last successfully applied public frontend settings. +type RgwFrontend struct { + Member string + Port int + SSLPort int + SSL bool +} + +// UpsertRGWFrontend records frontend settings alongside the member's service row. +func UpsertRGWFrontend(ctx context.Context, tx *sql.Tx, member string, port, sslPort int, ssl bool) error { + _, err := tx.ExecContext(ctx, ` +INSERT INTO rgw_frontends (member_id, port, ssl_port, ssl) +VALUES ((SELECT id FROM core_cluster_members WHERE name = ?), ?, ?, ?) +ON CONFLICT(member_id) DO UPDATE SET + port = excluded.port, ssl_port = excluded.ssl_port, ssl = excluded.ssl +WHERE port != excluded.port OR ssl_port != excluded.ssl_port OR ssl != excluded.ssl`, member, port, sslPort, ssl) + if err != nil { + return fmt.Errorf("failed to upsert rgw frontend for %s: %w", member, err) + } + return nil +} + +// GetRGWFrontends returns the recorded frontend settings for cluster members. +func GetRGWFrontends(ctx context.Context, tx *sql.Tx) ([]RgwFrontend, error) { + rows, err := tx.QueryContext(ctx, ` +SELECT m.name, f.port, f.ssl_port, f.ssl + FROM rgw_frontends f + JOIN core_cluster_members m ON m.id = f.member_id + ORDER BY m.name`) + if err != nil { + return nil, fmt.Errorf("failed to query rgw frontends: %w", err) + } + defer func() { _ = rows.Close() }() + + var frontends []RgwFrontend + for rows.Next() { + var f RgwFrontend + err := rows.Scan(&f.Member, &f.Port, &f.SSLPort, &f.SSL) + if err != nil { + return nil, fmt.Errorf("failed to scan rgw frontend: %w", err) + } + frontends = append(frontends, f) + } + err = rows.Err() + if err != nil { + return nil, fmt.Errorf("rgw frontend rows error: %w", err) + } + return frontends, nil +} + +// DeleteRGWFrontendByMember removes a member's frontend record if present. +func DeleteRGWFrontendByMember(ctx context.Context, tx *sql.Tx, member string) error { + _, err := tx.ExecContext(ctx, ` +DELETE FROM rgw_frontends WHERE member_id = (SELECT id FROM core_cluster_members WHERE name = ?)`, member) + if err != nil { + return fmt.Errorf("failed to delete rgw frontend for %s: %w", member, err) + } + return nil +} diff --git a/microceph/database/rgw_frontend_extras_test.go b/microceph/database/rgw_frontend_extras_test.go new file mode 100644 index 000000000..0e42e6b51 --- /dev/null +++ b/microceph/database/rgw_frontend_extras_test.go @@ -0,0 +1,188 @@ +package database + +import ( + "context" + "database/sql" + "testing" + + _ "github.com/mattn/go-sqlite3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// setupRGWFrontendDB creates an in-memory SQLite database (with foreign keys +// enabled) containing a minimal core_cluster_members table (the FK target) and +// the real schemaUpdate11 migration that creates rgw_frontends. Two named +// members are seeded so name->id resolution and cascade-delete can be tested. +func setupRGWFrontendDB(t *testing.T) *sql.DB { + t.Helper() + // _foreign_keys=on is per-connection in SQLite; the DSN parameter ensures + // every pooled connection honours ON DELETE CASCADE. + db, err := sql.Open("sqlite3", ":memory:?_foreign_keys=on") + require.NoError(t, err) + db.SetMaxOpenConns(1) + t.Cleanup(func() { _ = db.Close() }) + + // Minimal core_cluster_members matching the columns the rgw_frontends + // migration and the name->id subqueries reference. + _, err = db.Exec(` +CREATE TABLE core_cluster_members ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name TEXT NOT NULL UNIQUE +); +INSERT INTO core_cluster_members (name) VALUES ('node-a'); +INSERT INTO core_cluster_members (name) VALUES ('node-b'); +`) + require.NoError(t, err) + + tx, err := db.BeginTx(context.Background(), nil) + require.NoError(t, err) + err = schemaUpdate11(context.Background(), tx) + require.NoError(t, err) + require.NoError(t, tx.Commit()) + + return db +} + +// TestUpsertRGWFrontendAndGetAll verifies an upsert is persisted and read back +// keyed by member name, with ports and the TLS flag intact. +func TestUpsertRGWFrontendAndGetAll(t *testing.T) { + db := setupRGWFrontendDB(t) + ctx := context.Background() + + tx, err := db.BeginTx(ctx, nil) + require.NoError(t, err) + err = UpsertRGWFrontend(ctx, tx, "node-a", 80, 0, false) + require.NoError(t, err) + require.NoError(t, tx.Commit()) + + tx2, err := db.BeginTx(ctx, nil) + require.NoError(t, err) + defer func() { _ = tx2.Rollback() }() + + frontends, err := GetRGWFrontends(ctx, tx2) + require.NoError(t, err) + require.Len(t, frontends, 1) + assert.Equal(t, "node-a", frontends[0].Member) + assert.Equal(t, 80, frontends[0].Port) + assert.Equal(t, 0, frontends[0].SSLPort) + assert.False(t, frontends[0].SSL) +} + +// TestUpsertRGWFrontendOverwrites verifies a second upsert for the same member +// replaces the row (cert rotation / port change), not a duplicate. +func TestUpsertRGWFrontendOverwrites(t *testing.T) { + db := setupRGWFrontendDB(t) + ctx := context.Background() + + lockTx(t, db, func(tx *sql.Tx) { + require.NoError(t, UpsertRGWFrontend(ctx, tx, "node-a", 80, 0, false)) + }) + lockTx(t, db, func(tx *sql.Tx) { + require.NoError(t, UpsertRGWFrontend(ctx, tx, "node-a", 8080, 443, true)) + }) + + tx, err := db.BeginTx(ctx, nil) + require.NoError(t, err) + defer func() { _ = tx.Rollback() }() + + frontends, err := GetRGWFrontends(ctx, tx) + require.NoError(t, err) + require.Len(t, frontends, 1, "upsert must replace, not append") + assert.Equal(t, 8080, frontends[0].Port) + assert.Equal(t, 443, frontends[0].SSLPort) + assert.True(t, frontends[0].SSL) +} + +// TestGetRGWFrontendsMultipleMembers verifies the read returns each member's +// frontend keyed by name. +func TestGetRGWFrontendsMultipleMembers(t *testing.T) { + db := setupRGWFrontendDB(t) + ctx := context.Background() + + lockTx(t, db, func(tx *sql.Tx) { + require.NoError(t, UpsertRGWFrontend(ctx, tx, "node-a", 80, 0, false)) + require.NoError(t, UpsertRGWFrontend(ctx, tx, "node-b", 8443, 443, true)) + }) + + tx, err := db.BeginTx(ctx, nil) + require.NoError(t, err) + defer func() { _ = tx.Rollback() }() + + frontends, err := GetRGWFrontends(ctx, tx) + require.NoError(t, err) + require.Len(t, frontends, 2) + + byName := map[string]RgwFrontend{} + for _, f := range frontends { + byName[f.Member] = f + } + assert.Contains(t, byName, "node-a") + assert.Contains(t, byName, "node-b") + assert.True(t, byName["node-b"].SSL) +} + +// TestDeleteRGWFrontendByMember verifies the row is removed and that deleting a +// missing row is not an error (idempotent scale-down / retry). +func TestDeleteRGWFrontendByMember(t *testing.T) { + db := setupRGWFrontendDB(t) + ctx := context.Background() + + lockTx(t, db, func(tx *sql.Tx) { + require.NoError(t, UpsertRGWFrontend(ctx, tx, "node-a", 80, 0, false)) + }) + + lockTx(t, db, func(tx *sql.Tx) { + require.NoError(t, DeleteRGWFrontendByMember(ctx, tx, "node-a")) + }) + + // Read in its own (rolled-back) transaction, released before the re-delete + // below so the pool reuses the same :memory: connection. + func() { + tx, err := db.BeginTx(ctx, nil) + require.NoError(t, err) + frontends, err := GetRGWFrontends(ctx, tx) + require.NoError(t, err) + assert.Empty(t, frontends) + require.NoError(t, tx.Rollback()) + }() + + // Deleting an already-absent row must succeed (idempotent). + lockTx(t, db, func(tx *sql.Tx) { + require.NoError(t, DeleteRGWFrontendByMember(ctx, tx, "node-a")) + }) +} + +// TestRGWFrontendCascadeOnMemberRemoval verifies the ON DELETE CASCADE foreign +// key removes the frontend row when the cluster member is deleted. +func TestRGWFrontendCascadeOnMemberRemoval(t *testing.T) { + db := setupRGWFrontendDB(t) + ctx := context.Background() + + lockTx(t, db, func(tx *sql.Tx) { + require.NoError(t, UpsertRGWFrontend(ctx, tx, "node-a", 80, 0, false)) + }) + + _, err := db.Exec(`DELETE FROM core_cluster_members WHERE name = 'node-a'`) + require.NoError(t, err) + + tx, err := db.BeginTx(ctx, nil) + require.NoError(t, err) + defer func() { _ = tx.Rollback() }() + + frontends, err := GetRGWFrontends(ctx, tx) + require.NoError(t, err) + assert.Empty(t, frontends, "removing the member must cascade-delete its frontend row") +} + +func TestRGWFrontendRejectsUnknownMember(t *testing.T) { + db := setupRGWFrontendDB(t) + ctx := context.Background() + lockTx(t, db, func(tx *sql.Tx) { + err := UpsertRGWFrontend(ctx, tx, "unknown", 80, 0, false) + require.Error(t, err) + frontends, err := GetRGWFrontends(ctx, tx) + require.NoError(t, err) + assert.Empty(t, frontends) + }) +} diff --git a/microceph/database/schema.go b/microceph/database/schema.go index 9960f007d..e7d13b114 100644 --- a/microceph/database/schema.go +++ b/microceph/database/schema.go @@ -23,6 +23,7 @@ var SchemaExtensions = []cluster.Update{ schemaUpdate8, schemaUpdate9, schemaUpdate10, + schemaUpdate11, } // getClusterTableName returns the name of the table that holds the record of cluster members from sqlite_master. @@ -331,3 +332,22 @@ END; return err } + +// schemaUpdate11 records each member's last successfully applied RGW frontend. +// TLS material stays on the member and is never replicated in this table. +func schemaUpdate11(ctx context.Context, tx *sql.Tx) error { + stmt := ` +CREATE TABLE rgw_frontends ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + member_id INTEGER NOT NULL, + port INTEGER NOT NULL DEFAULT 0, + ssl_port INTEGER NOT NULL DEFAULT 0, + ssl INTEGER NOT NULL DEFAULT 0, + FOREIGN KEY (member_id) REFERENCES "core_cluster_members" (id) ON DELETE CASCADE, + UNIQUE(member_id) +); + ` + _, err := tx.ExecContext(ctx, stmt) + + return err +} From adae26f028662403a0463411216d48f1faa8b80f Mon Sep 17 00:00:00 2001 From: John Ramsden Date: Tue, 29 Sep 2026 18:19:22 +0000 Subject: [PATCH 2/2] fix: Assert cascade deletion directly. Assisted-by: omp:openai/gpt-6-astra Signed-off-by: John Ramsden --- microceph/database/rgw_frontend_extras_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/microceph/database/rgw_frontend_extras_test.go b/microceph/database/rgw_frontend_extras_test.go index 0e42e6b51..ddcfcabe1 100644 --- a/microceph/database/rgw_frontend_extras_test.go +++ b/microceph/database/rgw_frontend_extras_test.go @@ -170,9 +170,10 @@ func TestRGWFrontendCascadeOnMemberRemoval(t *testing.T) { require.NoError(t, err) defer func() { _ = tx.Rollback() }() - frontends, err := GetRGWFrontends(ctx, tx) + var count int + err = tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM rgw_frontends`).Scan(&count) require.NoError(t, err) - assert.Empty(t, frontends, "removing the member must cascade-delete its frontend row") + assert.Zero(t, count, "removing the member must cascade-delete its frontend row") } func TestRGWFrontendRejectsUnknownMember(t *testing.T) {