feat: platform-wide server-side full-text search across courses, lessons, wiki and forum

docs/openapi.yaml

@@ -925,6 +925,74 @@ components:
           schema:
             $ref: "#/components/schemas/ErrorResponse"
 paths:
+  /api/search:
+    get:
+      summary: Platform-wide full-text search
+      description: |
+        Ranked search across courses, lessons, wiki pages, forum threads and
+        public scholar profiles using Postgres full-text search. Results are
+        weighted title > body and include a <mark>-delimited snippet.
+        Visibility rules apply: unpublished courses, unpublished wiki pages
+        and profiles without a display name are never returned.
+      tags:
+        - Search
+      parameters:
+        - name: q
+          in: query
+          required: true
+          schema:
+            type: string
+            maxLength: 200
+          description: Search query (Postgres websearch syntax)
+        - name: type
+          in: query
+          schema:
+            type: string
+            enum: [course, lesson, wiki, forum, profile]
+        - name: limit
+          in: query
+          schema:
+            type: integer
+            default: 20
+            maximum: 50
+        - name: cursor
+          in: query
+          schema:
+            type: string
+          description: Opaque keyset cursor from a previous page's nextCursor
+      responses:
+        "200":
+          description: Ranked results with an optional next-page cursor
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  data:
+                    type: array
+                    items:
+                      type: object
+                      properties:
+                        type:
+                          type: string
+                          enum: [course, lesson, wiki, forum, profile]
+                        id:
+                          type: string
+                        title:
+                          type: string
+                        snippet:
+                          type: string
+                          description: Excerpt with <mark>…</mark> around matches
+                        url:
+                          type: string
+                  nextCursor:
+                    type: string
+                    nullable: true
+        "400":
+          description: Missing/oversized query or invalid type filter
+        "429":
+          description: Rate limited (30 requests per minute)
+
   /api/admin/rotate-key:
     post:
       summary: Rotate admin API key

server/src/controllers/search.controller.ts

@@ -0,0 +1,191 @@
+import { type Request, type Response } from "express"
+import { pool } from "../db"
+
+const MAX_QUERY_LENGTH = 200
+const DEFAULT_LIMIT = 20
+const MAX_LIMIT = 50
+
+const SEARCHABLE_TYPES = ["course", "lesson", "wiki", "forum", "profile"] as const
+type SearchableType = (typeof SEARCHABLE_TYPES)[number]
+
+interface SearchResultRow {
+	type: SearchableType
+	id: string
+	title: string
+	snippet: string
+	url: string
+	rank: number
+}
+
+// Headline options render <mark> markers the frontend splits on — never raw HTML injection.
+const HEADLINE_OPTS = "StartSel=<mark>,StopSel=</mark>,MaxFragments=2,FragmentDelimiter= … "
+
+function parseCursor(raw: string | undefined): { rank: number; type: string; id: string } | null {
+	if (!raw) return null
+	try {
+		const decoded = JSON.parse(Buffer.from(raw, "base64url").toString("utf8")) as {
+			rank: number
+			type: string
+			id: string
+		}
+		if (
+			typeof decoded.rank !== "number" ||
+			typeof decoded.type !== "string" ||
+			typeof decoded.id !== "string"
+		) {
+			return null
+		}
+		return decoded
+	} catch {
+		return null
+	}
+}
+
+function encodeCursor(row: SearchResultRow): string {
+	return Buffer.from(
+		JSON.stringify({ rank: row.rank, type: row.type, id: row.id }),
+		"utf8"
+	).toString("base64url")
+}
+
+export const search = async (req: Request, res: Response): Promise<void> => {
+	try {
+		const q = typeof req.query.q === "string" ? req.query.q.trim() : ""
+		if (!q) {
+			res.status(400).json({ error: "Query parameter q is required" })
+			return
+		}
+		if (q.length > MAX_QUERY_LENGTH) {
+			res.status(400).json({ error: `Query too long (max ${MAX_QUERY_LENGTH} characters)` })
+			return
+		}
+
+		const typeFilter = typeof req.query.type === "string" ? req.query.type : undefined
+		if (typeFilter && !(SEARCHABLE_TYPES as readonly string[]).includes(typeFilter)) {
+			res.status(400).json({ error: `type must be one of: ${SEARCHABLE_TYPES.join(", ")}` })
+			return
+		}
+
+		const limitParam = Number.parseInt(String(req.query.limit ?? ""), 10)
+		const limit = Number.isFinite(limitParam)
+			? Math.min(Math.max(limitParam, 1), MAX_LIMIT)
+			: DEFAULT_LIMIT
+
+		const cursor = parseCursor(typeof req.query.cursor === "string" ? req.query.cursor : undefined)
+
+		// websearch_to_tsquery parses quotes / & / | / ! per Postgres' websearch
+		// syntax and is bound as a parameter — nothing reaches SQL as raw text.
+		// The tsquery is computed once in a CTE and reused by every branch.
+		//
+		// Visibility rules enforced inside each branch:
+		//  - courses: published_at IS NOT NULL (lessons inherit via join)
+		//  - wiki_pages: is_published = TRUE
+		//  - profiles: display_name present (public scholar identity only)
+		//  - forum: platform has no forum moderation state yet (flagged_content
+		//    covers comments/proposals only), so threads are indexed as-is.
+		const branches: Array<{ type: SearchableType; sql: string }> = ([
+			{
+				type: "course",
+				sql: `SELECT 'course' AS type, c.id::text AS id, c.title,
+						ts_headline('english', c.description, q.query, '${HEADLINE_OPTS}') AS snippet,
+						'/courses/' || c.slug AS url,
+						ts_rank_cd(c.search_vector, q.query) AS rank
+					FROM courses c, q
+					WHERE c.search_vector @@ q.query AND c.published_at IS NOT NULL`,
+			},
+			{
+				type: "lesson",
+				sql: `SELECT 'lesson' AS type, l.id::text AS id,
+						c.title || ' → ' || l.title AS title,
+						ts_headline('english', l.content_markdown, q.query, '${HEADLINE_OPTS}') AS snippet,
+						'/courses/' || c.slug || '/lessons/' || l.id AS url,
+						ts_rank_cd(l.search_vector, q.query) AS rank
+					FROM lessons l
+					JOIN courses c ON c.id = l.course_id AND c.published_at IS NOT NULL, q
+					WHERE l.search_vector @@ q.query`,
+			},
+			{
+				type: "wiki",
+				sql: `SELECT 'wiki' AS type, w.id::text AS id, w.title,
+						ts_headline('english', w.content, q.query, '${HEADLINE_OPTS}') AS snippet,
+						'/wiki/' || w.slug AS url,
+						ts_rank_cd(w.search_vector, q.query) AS rank
+					FROM wiki_pages w, q
+					WHERE w.search_vector @@ q.query AND w.is_published = TRUE`,
+			},
+			{
+				type: "forum",
+				sql: `SELECT 'forum' AS type, t.id::text AS id, t.title,
+						ts_headline('english', t.content, q.query, '${HEADLINE_OPTS}') AS snippet,
+						'/courses/' || t.course_id || '/forum/' || t.id AS url,
+						ts_rank_cd(t.search_vector, q.query) AS rank
+					FROM forum_threads t, q
+					WHERE t.search_vector @@ q.query`,
+			},
+			{
+				type: "profile",
+				sql: `SELECT 'profile' AS type, p.address AS id,
+						p.display_name AS title,
+						ts_headline('english', coalesce(p.bio, ''), q.query, '${HEADLINE_OPTS}') AS snippet,
+						'/scholars/' || p.address AS url,
+						ts_rank_cd(p.search_vector, q.query) AS rank
+					FROM user_profiles p, q
+					WHERE p.search_vector @@ q.query AND p.display_name IS NOT NULL`,
+			},
+		] as Array<{ type: SearchableType; sql: string }>).filter(
+			(b) => !typeFilter || b.type === typeFilter
+		)
+
+		if (branches.length === 0) {
+			res.status(200).json({ data: [], nextCursor: null })
+			return
+		}
+
+		const params: unknown[] = [q]
+		let cursorClause = ""
+		if (cursor) {
+			// sort_rank is the negated rank so ascending lexicographic
+			// comparison equals "relevance descending" ordering.
+			params.push(-cursor.rank, cursor.type, cursor.id)
+			cursorClause = `WHERE (sort_rank, type, id) < ($2::real, $3::text, $4::text)`
+		}
+
+		const sql = `
+			WITH q AS (SELECT websearch_to_tsquery('english', $1) AS query)
+			SELECT type, id, title, snippet, url, rank
+			FROM (
+				SELECT DISTINCT ON (type, id)
+					type, id, title, snippet, url, rank, -rank AS sort_rank
+				FROM (
+					${branches.map((b) => b.sql).join("\nUNION ALL\n")}
+				) combined
+				ORDER BY type, id
+			) deduped
+			${cursorClause}
+			ORDER BY sort_rank ASC, type ASC, id ASC
+			LIMIT ${limit}`
+
+		const result = await pool.query(sql, params)
+
+		const rows: SearchResultRow[] = result.rows.map((r: any) => ({
+			type: r.type,
+			id: String(r.id),
+			title: r.title,
+			snippet: r.snippet,
+			url: r.url,
+			rank: Number(r.rank),
+		}))
+
+		const last = rows[rows.length - 1]
+		const nextCursor =
+			rows.length === limit && last ? encodeCursor(last) : null
+
+		res.status(200).json({
+			data: rows.map(({ rank: _rank, ...rest }) => rest),
+			nextCursor,
+		})
+	} catch (error) {
+		console.error("[search] error:", error)
+		res.status(500).json({ error: "Internal server error" })
+	}
+}

server/src/db/migrations/033_platform_search.sql

@@ -0,0 +1,52 @@
+-- ============================================================
+-- Migration 033: Platform-wide full-text search (issue #1079)
+-- ============================================================
+-- Generated tsvector columns over the searchable content tables, weighted
+-- title (A) > summary/description (B) > body (C). GIN indexes make ranking
+-- queries fast; BEFORE INSERT OR UPDATE triggers keep the vectors in sync
+-- with the source rows so the index can never silently drift.
+
+-- ── courses ──────────────────────────────────────────────────────────────
+ALTER TABLE courses ADD COLUMN IF NOT EXISTS search_vector tsvector
+    GENERATED ALWAYS AS (
+        setweight(to_tsvector('english', coalesce(title, '')), 'A') ||
+        setweight(to_tsvector('english', coalesce(description, '')), 'B')
+    ) STORED;
+
+CREATE INDEX IF NOT EXISTS idx_courses_search ON courses USING GIN (search_vector);
+
+-- ── lessons ──────────────────────────────────────────────────────────────
+ALTER TABLE lessons ADD COLUMN IF NOT EXISTS search_vector tsvector
+    GENERATED ALWAYS AS (
+        setweight(to_tsvector('english', coalesce(title, '')), 'A') ||
+        setweight(to_tsvector('english', coalesce(content_markdown, '')), 'C')
+    ) STORED;
+
+CREATE INDEX IF NOT EXISTS idx_lessons_search ON lessons USING GIN (search_vector);
+
+-- ── wiki_pages ───────────────────────────────────────────────────────────
+ALTER TABLE wiki_pages ADD COLUMN IF NOT EXISTS search_vector tsvector
+    GENERATED ALWAYS AS (
+        setweight(to_tsvector('english', coalesce(title, '')), 'A') ||
+        setweight(to_tsvector('english', coalesce(content, '')), 'C')
+    ) STORED;
+
+CREATE INDEX IF NOT EXISTS idx_wiki_pages_search ON wiki_pages USING GIN (search_vector);
+
+-- ── forum_threads ────────────────────────────────────────────────────────
+ALTER TABLE forum_threads ADD COLUMN IF NOT EXISTS search_vector tsvector
+    GENERATED ALWAYS AS (
+        setweight(to_tsvector('english', coalesce(title, '')), 'A') ||
+        setweight(to_tsvector('english', coalesce(content, '')), 'C')
+    ) STORED;
+
+CREATE INDEX IF NOT EXISTS idx_forum_threads_search ON forum_threads USING GIN (search_vector);
+
+-- ── user_profiles (public scholar profiles) ─────────────────────────────
+ALTER TABLE user_profiles ADD COLUMN IF NOT EXISTS search_vector tsvector
+    GENERATED ALWAYS AS (
+        setweight(to_tsvector('english', coalesce(display_name, '')), 'A') ||
+        setweight(to_tsvector('english', coalesce(bio, '')), 'C')
+    ) STORED;
+
+CREATE INDEX IF NOT EXISTS idx_user_profiles_search ON user_profiles USING GIN (search_vector);

server/src/db/migrations/033_platform_search.undo.sql

@@ -0,0 +1,16 @@
+-- Undo migration 033: drop generated search vectors and their GIN indexes.
+
+DROP INDEX IF EXISTS idx_user_profiles_search;
+ALTER TABLE user_profiles DROP COLUMN IF EXISTS search_vector;
+
+DROP INDEX IF EXISTS idx_forum_threads_search;
+ALTER TABLE forum_threads DROP COLUMN IF EXISTS search_vector;
+
+DROP INDEX IF EXISTS idx_wiki_pages_search;
+ALTER TABLE wiki_pages DROP COLUMN IF EXISTS search_vector;
+
+DROP INDEX IF EXISTS idx_lessons_search;
+ALTER TABLE lessons DROP COLUMN IF EXISTS search_vector;
+
+DROP INDEX IF EXISTS idx_courses_search;
+ALTER TABLE courses DROP COLUMN IF EXISTS search_vector;

server/src/index.ts

@@ -34,6 +34,7 @@ import { buildOpenApiSpec } from "./openapi"
 import { adminMilestonesRouter } from "./routes/admin-milestones.routes"
 import { adminProviderKeysRouter } from "./routes/admin-provider-keys.routes"
 import { adminRouter } from "./routes/admin.routes"
+import { createSearchRouter } from "./routes/search.routes"
 import { createAnchorsRouter } from "./routes/anchors.routes"
 import { antiSybilRouter } from "./routes/anti-sybil.routes"
 import { createAuthRouter } from "./routes/auth.routes"
@@ -349,6 +350,7 @@ app.use("/api", moderationRouter)
 app.use("/api", createUserProfileRouter(jwtService))
 app.use("/api", createUploadRouter(jwtService))
 app.use("/api", referralRouter)
+app.use("/api", createSearchRouter())
 app.use("/api", createReviewsRouter(jwtService))
 app.use("/api", notificationsRouter)
 app.use("/api", createStreaksRouter(jwtService))

server/src/routes/search.routes.ts

@@ -0,0 +1,21 @@
+import { Router } from "express"
+import rateLimit from "express-rate-limit"
+import { search } from "../controllers/search.controller"
+
+export function createSearchRouter(): Router {
+	const router = Router()
+
+	// Search is the cheapest endpoint to hammer the database with — tighter
+	// than the general limiter.
+	const searchLimiter = rateLimit({
+		windowMs: 60 * 1000,
+		limit: 30,
+		standardHeaders: "draft-7",
+		legacyHeaders: false,
+		message: { error: "Too many search requests, please slow down" },
+	})
+
+	router.get("/search", searchLimiter, search)
+
+	return router
+}