feat(stt): add cross-platform Silero VAD pre-pass via whisper.cpp

Add Silero VAD (v6.2.0) pre-pass hooked into the vendored whisper.cpp
helper via whisper_vad_segments_from_samples, fully cross-platform with
no new dependencies or external runtimes.

- electron/native/whisper-stt/src/main.cpp:
  - Add --vad-model flag and OPENSCREEN_VAD_MODEL env var support
  - Initialize whisper_vad_context with auto GPU detection and CPU fallback
  - Pass wparams.vad and wparams.vad_model_path to whisper_full
  - Add POST /vad endpoint returning speech boundaries via whisper_vad_segments_from_samples
  - Clean up VAD context via whisper_vad_free on exit
- electron/stt/modelManager.ts:
  - Add silero-vad model descriptor (ggml-silero-v6.2.0.bin, ~885 KB)
  - Map model cache paths and check model presence
- electron/stt/whisperServer.ts:
  - Pass --vad-model to helper process args
  - Add detectVadSegments method calling POST /vad
- electron/stt/index.ts:
  - Download silero-vad alongside whisper in prepare()
  - Pass vadModelPath on startup and chunk retry
  - Add detectSpeech method and register stt:vad IPC handler
- electron/stt/*.test.ts:
  - Add unit tests for descriptor, paths, caching, --vad-model argument, and /vad client

Refs #626
Follows feedback on #631

Author: CyberSparkx

electron/native/whisper-stt/src/main.cpp

@@ -44,6 +44,7 @@
 #include <cstdint>
 #include <cstdlib>
 #include <cstring>
+#include <filesystem>
 #include <fstream>
 #include <iostream>
 #include <mutex>
@@ -222,6 +223,7 @@ struct Word {
 
 int main(int argc, char** argv) {
 	std::string model_path;
+	std::string vad_model_path;
 	std::string host = "127.0.0.1";
 	bool host_from_flag = false;
 	bool force_cpu = false;
@@ -231,6 +233,7 @@ int main(int argc, char** argv) {
 	for (int i = 1; i < argc; ++i) {
 		const std::string a = argv[i];
 		if (a == "--model"   && i + 1 < argc) model_path = argv[++i];
+		else if (a == "--vad-model" && i + 1 < argc) vad_model_path = argv[++i];
 		else if (a == "--host" && i + 1 < argc) { host = argv[++i]; host_from_flag = true; }
 		else if (a == "--port" && i + 1 < argc) port = std::atoi(argv[++i]);
 		else if (a == "--threads" && i + 1 < argc) threads = std::atoi(argv[++i]);
@@ -241,6 +244,10 @@ int main(int argc, char** argv) {
 	if (model_path.empty()) {
 		if (const char* p = std::getenv("OPENSCREEN_WHISPER_MODEL")) model_path = p;
 	}
+	if (vad_model_path.empty()) {
+		if (const char* p = std::getenv("OPENSCREEN_VAD_MODEL")) vad_model_path = p;
+		else if (const char* p = std::getenv("OPENSCREEN_WHISPER_VAD_MODEL")) vad_model_path = p;
+	}
 	if (port == 0) {
 		if (const char* p = std::getenv("OPENSCREEN_WHISPER_PORT")) port = std::atoi(p);
 	}
@@ -264,7 +271,9 @@ int main(int argc, char** argv) {
 		             "OPENSCREEN_WHISPER_MODEL is required" << std::endl;
 		return 2;
 	}
-	log("boot: model=" + model_path + " host=" + host +
+	log("boot: model=" + model_path +
+	    (!vad_model_path.empty() ? (" vad_model=" + vad_model_path) : "") +
+	    " host=" + host +
 	    " port=" + (port > 0 ? std::to_string(port) : "(any)") +
 	    " threads=" + std::to_string(threads));
 
@@ -293,6 +302,25 @@ int main(int argc, char** argv) {
 	const std::string active_backend = cparams.use_gpu ? detect_active_backend() : "whispercpp-cpu";
 	log("model loaded; backend=" + active_backend);
 
+	// ---- Init VAD context (Silero VAD v6.2.0) ----
+	struct whisper_vad_context* vctx = nullptr;
+	if (!vad_model_path.empty()) {
+		struct whisper_vad_context_params vad_ctx_params = whisper_vad_default_context_params();
+		vad_ctx_params.n_threads = threads;
+		vad_ctx_params.use_gpu   = !force_cpu;
+		vctx = whisper_vad_init_from_file_with_params(vad_model_path.c_str(), vad_ctx_params);
+		if (!vctx && vad_ctx_params.use_gpu) {
+			log("GPU VAD initialization failed; retrying with CPU inference");
+			vad_ctx_params.use_gpu = false;
+			vctx = whisper_vad_init_from_file_with_params(vad_model_path.c_str(), vad_ctx_params);
+		}
+		if (vctx) {
+			log("VAD model loaded; path=" + vad_model_path);
+		} else {
+			log("WARNING: failed to load VAD model from " + vad_model_path);
+		}
+	}
+
 	// ---- HTTP server ----
 	httplib::Server svr;
 	std::mutex infer_mu;  // whisper contexts are not thread-safe
@@ -366,6 +394,11 @@ int main(int argc, char** argv) {
 		wparams.print_realtime   = false;
 		wparams.print_timestamps = false;
 		wparams.n_threads        = threads;
+		if (vctx && !vad_model_path.empty()) {
+			wparams.vad            = true;
+			wparams.vad_model_path = vad_model_path.c_str();
+			wparams.vad_params     = whisper_vad_default_params();
+		}
 
 		const auto t0 = std::chrono::steady_clock::now();
 		const int rc  = whisper_full(ctx, wparams, pcm.data(), static_cast<int>(pcm.size()));
@@ -540,12 +573,80 @@ int main(int argc, char** argv) {
 		res.set_content(reply.dump(), "application/json");
 	});
 
+	// POST /vad — multipart form with `file` (WAV: 16 kHz mono PCM16).
+	// Runs Silero VAD segmentation and returns speech intervals [start, end] in seconds.
+	svr.Post("/vad", [&](const httplib::Request& req, httplib::Response& res) {
+		if (!vctx) {
+			res.status = 400;
+			res.set_content(R"({"error":"VAD model was not loaded on server startup"})", "application/json");
+			return;
+		}
+		auto it = req.files.find("file");
+		if (it == req.files.end()) {
+			res.status = 400;
+			res.set_content(R"({"error":"missing 'file' form field"})", "application/json");
+			return;
+		}
+		const auto& file_entry = it->second;
+
+		static std::atomic<uint64_t> tmp_vad_counter{0};
+		const auto tmp_vad_id = tmp_vad_counter.fetch_add(1, std::memory_order_relaxed);
+		const auto tmp_vad_ts = std::chrono::high_resolution_clock::now().time_since_epoch().count();
+		const std::string tmp_wav = (std::filesystem::temp_directory_path() /
+		                             ("openscreen-vad-" + std::to_string(tmp_vad_ts) +
+		                              "-" + std::to_string(tmp_vad_id) + ".wav")).string();
+		{
+			std::ofstream out(tmp_wav, std::ios::binary);
+			out.write(file_entry.content.data(),
+			          static_cast<std::streamsize>(file_entry.content.size()));
+		}
+		std::vector<float> pcm;
+		int sample_rate = 0, channels = 0;
+		const bool ok = read_wav_pcm16(tmp_wav, pcm, sample_rate, channels);
+		std::error_code ec;
+		std::filesystem::remove(tmp_wav, ec);
+		if (!ok) {
+			res.status = 400;
+			res.set_content(R"({"error":"failed to parse WAV"})", "application/json");
+			return;
+		}
+		if (sample_rate != 16000 || channels != 1) {
+			res.status = 400;
+			res.set_content(
+				R"({"error":"expected 16 kHz mono PCM16 WAV"})",
+				"application/json");
+			return;
+		}
+
+		const std::lock_guard<std::mutex> lk(infer_mu);
+		struct whisper_vad_params vad_params = whisper_vad_default_params();
+		struct whisper_vad_segments* vad_segments =
+			whisper_vad_segments_from_samples(vctx, vad_params, pcm.data(), static_cast<int>(pcm.size()));
+
+		nlohmann::json reply;
+		reply["segments"] = nlohmann::json::array();
+		if (vad_segments) {
+			const int n_segs = whisper_vad_segments_n_segments(vad_segments);
+			for (int i = 0; i < n_segs; ++i) {
+				const float t0 = whisper_vad_segments_get_segment_t0(vad_segments, i);
+				const float t1 = whisper_vad_segments_get_segment_t1(vad_segments, i);
+				reply["segments"].push_back({
+					{"start", static_cast<double>(t0)},
+					{"end",   static_cast<double>(t1)}
+				});
+			}
+			whisper_vad_free_segments(vad_segments);
+		}
+		res.set_content(reply.dump(), "application/json");
+	});
+
 	// ---- bind + listen ----
 	int bound_port = port;
 	if (bound_port == 0) {
 		bound_port = svr.bind_to_any_port(host);
 	} else if (!svr.bind_to_port(host, bound_port)) {
 		std::cerr << "FATAL: bind_to_port(" << host << ":" << bound_port << ") failed" << std::endl;
+		if (vctx) whisper_vad_free(vctx);
 		whisper_free(ctx);
 		return 4;
 	}
@@ -554,6 +655,7 @@ int main(int argc, char** argv) {
 	if (rc != 0) {
 		std::cerr << "FATAL: listen_after_bind failed" << std::endl;
 	}
+	if (vctx) whisper_vad_free(vctx);
 	whisper_free(ctx);
 	return rc;
 }

electron/stt/index.ts

@@ -9,6 +9,8 @@ import type {
 	SttTiming,
 	SttTranscribeRequest,
 	SttTranscribeResponse,
+	SttVadResponse,
+	SttVadSegment,
 	SttWordSegment,
 } from "./transcriptionContract";
 import { WhisperServerManager } from "./whisperServer";
@@ -97,6 +99,7 @@ export class SttManager {
 	private initPromise: Promise<void> | null = null;
 	/** Kept from `prepare()` so a chunk retry can respawn a helper that died mid-run. */
 	private modelPath: string | null = null;
+	private vadModelPath: string | null = null;
 	/**
 	 * Bumped by `cancel()`. The chunk loop compares it against the value it
 	 * captured on entry, so a cancel that lands after a new run started cannot
@@ -194,8 +197,12 @@ export class SttManager {
 
 		const paths = modelPaths(modelsDir);
 		this.modelPath = paths.whisper;
+		this.vadModelPath = paths["silero-vad"];
 		try {
-			await this.server.start({ modelPath: paths.whisper });
+			await this.server.start({
+				modelPath: paths.whisper,
+				vadModelPath: paths["silero-vad"],
+			});
 		} catch (error) {
 			if (this.shuttingDown) throw cancelledError();
 			throw error;
@@ -236,7 +243,9 @@ export class SttManager {
 				if (this.shuttingDown) throw cancelledError();
 				if (attempt === CHUNK_ATTEMPTS) break;
 				if (this.modelPath) {
-					await this.server.start({ modelPath: this.modelPath }).catch(() => undefined);
+					await this.server
+						.start({ modelPath: this.modelPath, vadModelPath: this.vadModelPath })
+						.catch(() => undefined);
 				}
 				if (this.shuttingDown) throw cancelledError();
 				await new Promise((resolve) => setTimeout(resolve, 500 * attempt));
@@ -433,6 +442,15 @@ export class SttManager {
 		this.cancelEpoch++;
 		await this.server.shutdown();
 	}
+
+	/**
+	 * Run Voice Activity Detection (Silero VAD) to detect speech segments in samples.
+	 */
+	async detectSpeech(samples: Float32Array): Promise<SttVadSegment[]> {
+		if (this.shuttingDown) throw cancelledError();
+		await this.init();
+		return this.server.detectVadSegments({ samples });
+	}
 }
 
 let singleton: SttManager | null = null;
@@ -491,6 +509,13 @@ export function registerSttIpc(ipcMain: IpcMain): void {
 			}
 		},
 	);
+	ipcMain.handle(
+		"stt:vad",
+		async (_event, req: { samples: Float32Array }): Promise<SttVadResponse> => {
+			const segments = await manager.detectSpeech(req.samples);
+			return { segments };
+		},
+	);
 	ipcMain.handle("stt:cancel", () => {
 		manager.cancel();
 	});

electron/stt/modelManager.test.ts

@@ -30,20 +30,39 @@ describe("modelManager", () => {
 		}
 	});
 
+	it("exposes the silero-vad model descriptor with a single GGML file", () => {
+		expect(STT_MODELS["silero-vad"].cacheDir).toBe("whisper-ggml");
+		expect(STT_MODELS["silero-vad"].repoId).toBe("ggml-org/whisper-vad");
+		expect(STT_MODELS["silero-vad"].files.length).toBe(1);
+		expect(STT_MODELS["silero-vad"].files[0].name).toBe("ggml-silero-v6.2.0.bin");
+		expect(STT_MODELS["silero-vad"].files[0].expectedSha256).toBe(
+			"2aa269b785eeb53a82983a20501ddf7c1d9c48e33ab63a41391ac6c9f7fb6987",
+		);
+		for (const f of STT_MODELS["silero-vad"].files) {
+			expect(f.approximateBytes).toBeGreaterThan(0);
+			expect(f.url).toContain("huggingface.co");
+			expect(f.url).toMatch(/\/resolve\/[0-9a-f]{40}\//);
+		}
+	});
+
 	it("modelPaths places the GGML file under the cache directory", () => {
 		const paths = modelPaths(dir);
 		expect(paths.whisper).toBe(path.join(dir, "whisper-ggml", "ggml-small-q8_0.bin"));
+		expect(paths["silero-vad"]).toBe(path.join(dir, "whisper-ggml", "ggml-silero-v6.2.0.bin"));
 	});
 
 	it("areModelsPresent returns false when the model file is missing", async () => {
 		expect(await areModelsPresent(dir)).toBe(false);
 	});
 
-	it("areModelsPresent returns true once the GGML file is present", async () => {
+	it("areModelsPresent returns true once the GGML files are present", async () => {
 		const paths = modelPaths(dir);
 		await mkdir(path.dirname(paths.whisper), { recursive: true });
 		expect(await areModelsPresent(dir)).toBe(false);
 		await writeFile(paths.whisper, "dummy-ggml");
+		expect(await areModelsPresent(dir, ["whisper"])).toBe(true);
+		expect(await areModelsPresent(dir)).toBe(false);
+		await writeFile(paths["silero-vad"], "dummy-vad");
 		expect(await areModelsPresent(dir)).toBe(true);
 	});
 

electron/stt/modelManager.ts

@@ -24,7 +24,7 @@ import { pipeline } from "node:stream/promises";
  * separate VAD model is required. See `technical-documentation/architecture/transcription-and-captions.md`.
  */
 
-export type SttModelId = "whisper";
+export type SttModelId = "whisper" | "silero-vad";
 
 export interface SttModelFile {
 	/** Relative path within the model directory (e.g. "ggml-small-q8_0.bin"). */
@@ -62,6 +62,10 @@ const MODEL_FILE = "ggml-small-q8_0.bin";
 // LFS oid for MODEL_FILE is exactly the digest below.
 const MODEL_REVISION = "5359861c739e955e79d9a303bcbc70fb988958b1";
 
+const VAD_REPO = "ggml-org/whisper-vad";
+const VAD_FILE = "ggml-silero-v6.2.0.bin";
+const VAD_REVISION = "9ffd54a1e1ee413ddf265af9913beaf518d1639b";
+
 export const STT_MODELS: Record<SttModelId, SttModelDescriptor> = {
 	whisper: {
 		cacheDir: "whisper-ggml",
@@ -75,22 +79,43 @@ export const STT_MODELS: Record<SttModelId, SttModelDescriptor> = {
 			},
 		],
 	},
+	"silero-vad": {
+		cacheDir: "whisper-ggml",
+		repoId: VAD_REPO,
+		files: [
+			{
+				name: VAD_FILE,
+				url: `${MODEL_BASE}/${VAD_REPO}/resolve/${VAD_REVISION}/${VAD_FILE}`,
+				expectedSha256: "2aa269b785eeb53a82983a20501ddf7c1d9c48e33ab63a41391ac6c9f7fb6987",
+				approximateBytes: 885_098,
+			},
+		],
+	},
 };
 
 export function modelPaths(baseDir: string): Record<SttModelId, string> {
 	return {
 		whisper: path.join(baseDir, STT_MODELS.whisper.cacheDir, MODEL_FILE),
+		"silero-vad": path.join(baseDir, STT_MODELS["silero-vad"].cacheDir, VAD_FILE),
 	};
 }
 
 /**
- * True when the GGML model file exists and is non-empty.
+ * True when the GGML model files exist and are non-empty.
  */
-export async function areModelsPresent(baseDir: string): Promise<boolean> {
+export async function areModelsPresent(
+	baseDir: string,
+	only: SttModelId[] = ["whisper", "silero-vad"],
+): Promise<boolean> {
 	const paths = modelPaths(baseDir);
 	try {
-		const s = await stat(paths.whisper);
-		return s.isFile() && s.size > 0;
+		const results = await Promise.all(
+			only.map(async (id) => {
+				const s = await stat(paths[id]);
+				return s.isFile() && s.size > 0;
+			}),
+		);
+		return results.every(Boolean);
 	} catch {
 		return false;
 	}
@@ -217,7 +242,7 @@ async function ensureFile(
 
 export interface EnsureModelsOptions {
 	baseDir: string;
-	/** Models to ensure; defaults to all (currently just `whisper`). */
+	/** Models to ensure; defaults to all (`whisper`, `silero-vad`). */
 	only?: SttModelId[];
 	onProgress?: (event: {
 		id: SttModelId;
@@ -228,9 +253,9 @@ export interface EnsureModelsOptions {
 	fetcher?: typeof fetch;
 }
 
-/** Ensure the GGML model file is present locally; downloads with progress + retry. */
+/** Ensure the GGML model files are present locally; downloads with progress + retry. */
 export async function ensureModels(opts: EnsureModelsOptions): Promise<void> {
-	const targets = (opts.only ?? (["whisper"] as SttModelId[])).map((id) => ({
+	const targets = (opts.only ?? (["whisper", "silero-vad"] as SttModelId[])).map((id) => ({
 		id,
 		descriptor: STT_MODELS[id],
 		filePath: modelPaths(opts.baseDir)[id],

electron/stt/transcriptionContract.ts

@@ -71,7 +71,7 @@ export interface SttStatusEvent {
 	/** Total bytes for the in-flight download. */
 	totalBytes?: number;
 	/** Which model is downloading. */
-	model?: "whisper";
+	model?: "whisper" | "silero-vad";
 	/**
 	 * Seconds of audio transcribed so far, and the total for this request. Only
 	 * when `phase === "transcribe"`. Progress is reported per CHUNK (see
@@ -158,3 +158,14 @@ export interface SttTranscribeResponse {
 
 /** IPC success envelope; thrown errors cross as a rejection. */
 export type SttTranscribeResult = SttTranscribeResponse;
+
+/** A detected speech segment with timestamps in seconds. */
+export interface SttVadSegment {
+	startSec: number;
+	endSec: number;
+}
+
+/** Response from the VAD speech detection pass. */
+export interface SttVadResponse {
+	segments: SttVadSegment[];
+}