diff --git a/packages/daemon/src/core/state.test.ts b/packages/daemon/src/core/state.test.ts index 9844b4e..3e68859 100644 --- a/packages/daemon/src/core/state.test.ts +++ b/packages/daemon/src/core/state.test.ts @@ -23,7 +23,11 @@ vi.mock('./engines.js', async (importOriginal) => { }; }); -import { CoreState } from './state.js'; +import { + CoreState, + credentialUnavailableReason, + isModelUsable, +} from './state.js'; // ── Helpers ────────────────────────────────────────────────────────────────── @@ -405,12 +409,68 @@ describe('CoreState.listEngines', () => { }); }); +describe('credentialUnavailableReason', () => { + it('describes missing env credentials', () => { + const msg = credentialUnavailableReason( + { engine: 'openrouter', secret_store: 'env', api_key_env_var_name: 'OR_KEY', models: {} }, + { defaultEnvVar: 'OPENROUTER_API_KEY' }, + ); + expect(msg).toContain('OR_KEY'); + expect(msg).toContain('daemon process'); + }); + + it('describes missing store credentials per backend', () => { + expect( + credentialUnavailableReason( + { engine: 'openrouter', secret_store: 'keychain', secret_name: 'K', models: {} }, + {}, + ), + ).toMatch(/keychain/i); + + expect( + credentialUnavailableReason( + { engine: 'openrouter', secret_store: 'file', secret_name: 'F', models: {} }, + {}, + ), + ).toMatch(/file secrets/i); + + expect( + credentialUnavailableReason( + { engine: 'openrouter', secret_store: 'memory', secret_name: 'M', models: {} }, + {}, + ), + ).toMatch(/memory/i); + }); + + it('falls back to engine default env or generic message', () => { + expect( + credentialUnavailableReason({ engine: 'openrouter', models: {} }, { defaultEnvVar: 'OPENROUTER_API_KEY' }), + ).toContain('OPENROUTER_API_KEY'); + + expect(credentialUnavailableReason({ engine: 'mock', models: {} }, {})).toContain('provider wizard'); + }); +}); + +describe('isModelUsable', () => { + it('treats undefined available as usable', () => { + expect(isModelUsable({ id: 'p/m', name: 'm', engineModelId: 'm', provider: 'p', engine: 'mock', contextWindow: 0 })).toBe(true); + }); + + it('rejects explicitly unavailable models', () => { + expect(isModelUsable({ + id: 'p/m', name: 'm', engineModelId: 'm', provider: 'p', engine: 'mock', contextWindow: 0, available: false, + })).toBe(false); + }); +}); + describe('CoreState.listModels', () => { it('returns composite model ids for configured mock provider', async () => { const core = createCore({ config: mockProviderConfig }); const models = await core.listModels(); - expect(models.map((m) => m.id).sort()).toEqual(['my-mock/echo', 'my-mock/fixed']); - expect(models[0].engineModelId).toBeDefined(); + const mockModels = models.filter((m) => m.provider === 'my-mock'); + expect(mockModels.map((m) => m.id).sort()).toEqual(['my-mock/echo', 'my-mock/fixed']); + expect(mockModels[0].engineModelId).toBeDefined(); + expect(mockModels.every((m) => m.available !== false)).toBe(true); }); it('applies an explicit vision capability override', async () => { @@ -429,10 +489,17 @@ describe('CoreState.listModels', () => { expect(models.find((model) => model.id === 'my-mock/vision')?.capabilities.supportsVision).toBe(true); }); - it('skips providers without keys when engine requires key', async () => { + it('lists saved models as unavailable when engine requires key but key is missing', async () => { + mockFetchModels.mockClear(); const core = createCore({ config: mockProviderConfig }); const models = await core.listModels(); - expect(models.every((m) => m.provider !== 'openrouter')).toBe(true); + const orModels = models.filter((m) => m.provider === 'openrouter'); + expect(orModels.length).toBeGreaterThan(0); + expect(orModels.every((m) => m.available === false)).toBe(true); + expect(orModels[0].unavailableReason).toMatch(/keychain|API key/i); + // fetchModels should have been called for mock (no key needed) but NOT for openrouter + const calls = mockFetchModels.mock.calls; + expect(calls.every((c: unknown[]) => c[0] !== 'openrouter')).toBe(true); }); it('continues when fetchModels throws for one provider', async () => { @@ -484,6 +551,28 @@ describe('CoreState.chat', () => { expect(chunks.at(-1)?.finishReason).toBe('error'); }); + it('errors when model is not in the provider enabled list', async () => { + const core = createCore({ config: mockProviderConfig }); + const chunks = await collectChat(core, 'my-mock/not-in-config'); + expect(chunks.some((c) => c.type === 'error' && c.error?.includes('not enabled'))).toBe(true); + }); + + it('allows chat when modelName matches a configured model_id alias', async () => { + const core = createCore({ + config: { + providers: { + 'my-mock': { + engine: 'mock', + models: { 'claude-precise': { model_id: 'echo' } }, + }, + }, + }, + }); + const chunks = await collectChat(core, 'my-mock/echo'); + expect(chunks.some((c) => c.type === 'error' && c.error?.includes('not enabled'))).toBe(false); + expect(mockStreamChat).toHaveBeenCalled(); + }); + it('errors when provider or api key missing', async () => { const core = createCore({ config: mockProviderConfig }); const missingProvider = await collectChat(core, 'missing/echo'); diff --git a/packages/daemon/src/core/state.ts b/packages/daemon/src/core/state.ts index 95f74b9..2db2724 100644 --- a/packages/daemon/src/core/state.ts +++ b/packages/daemon/src/core/state.ts @@ -96,6 +96,46 @@ export interface ModelInfo { }; /** Per-model parameter overrides from config */ params?: ModelConfig; + /** False when config exists but credentials cannot be resolved at runtime */ + available?: boolean; + /** Human-readable reason when available is false */ + unavailableReason?: string; +} + +/** Models that can be selected for chat / OpenAI-compatible listing */ +export function isModelUsable(model: ModelInfo): boolean { + return model.available !== false; +} + +/** + * Explain why a provider's API key could not be resolved (for UI and listModels). + */ +export function credentialUnavailableReason( + providerCfg: ProviderConfig, + engineInfo: { defaultEnvVar?: string }, +): string { + const source = providerCredentialSource(providerCfg); + if (source?.kind === 'env') { + return ( + `Environment variable "${source.name}" is not set in the daemon process. ` + + 'Set it in the shell before starting the daemon, or use Keychain/File and enter the key in the provider wizard.' + ); + } + if (source?.kind === 'store') { + const storeLabel = + source.backend === 'file' ? 'file secrets' : source.backend === 'memory' ? 'memory' : 'keychain'; + return ( + `API key "${source.name}" is not available in ${storeLabel}. ` + + 'Re-enter the API key in the provider settings, or fix secret storage.' + ); + } + if (engineInfo.defaultEnvVar) { + return ( + `No API key found (expected env "${engineInfo.defaultEnvVar}" or a saved secret). ` + + 'Complete provider setup in the dashboard.' + ); + } + return 'No API key available for this provider. Add credentials via the provider wizard.'; } // ── Chat tool options ────────────────────────────────────────────────── @@ -468,15 +508,15 @@ export class CoreState { } const apiKey = await this.resolveApiKey(providerId, providerCfg); - if (engineInfo.requiresKey && !apiKey) { - continue; - } + const keyMissing = engineInfo.requiresKey && !apiKey; let discoveredModels: DiscoveredModel[] = []; - try { - discoveredModels = await fetchModels(providerCfg.engine, apiKey || undefined, providerCfg.base_url); - } catch (error) { - console.error(`[State] Failed to fetch models for provider ${providerId}:`, error); + if (!keyMissing) { + try { + discoveredModels = await fetchModels(providerCfg.engine, apiKey || undefined, providerCfg.base_url); + } catch (error) { + console.error(`[State] Failed to fetch models for provider ${providerId}:`, error); + } } const discoveryMap = new Map(); @@ -508,6 +548,10 @@ export class CoreState { contextWindow: discovered?.contextWindow || 0, capabilities, params: Object.keys(modelCfg).length > 0 ? modelCfg : undefined, + available: !keyMissing, + unavailableReason: keyMissing + ? credentialUnavailableReason(providerCfg, engineInfo) + : undefined, }); } } @@ -595,6 +639,25 @@ export class CoreState { return; } + const enabledModels = providerCfg.models; + const matchesEngineModelId = Object.entries(enabledModels || {}).some( + ([name, cfg]) => resolveEngineModelId(name, cfg) === modelName, + ); + if ( + enabledModels + && Object.keys(enabledModels).length > 0 + && !enabledModels[modelName] + && !matchesEngineModelId + ) { + console.error(`[State] Model not enabled in config: ${compositeModelId}`); + yield { + type: 'error', + error: `Model "${compositeModelId}" is not enabled. Add it in the provider settings or pick a listed model.`, + }; + yield { type: 'done', finishReason: 'error' }; + return; + } + // ── Resolve API key ── const apiKey = await this.resolveApiKey(providerId, providerCfg); if (engineInfo.requiresKey && !apiKey) { diff --git a/packages/daemon/src/daemon/model-picker.ts b/packages/daemon/src/daemon/model-picker.ts index b9b4421..b274859 100644 --- a/packages/daemon/src/daemon/model-picker.ts +++ b/packages/daemon/src/daemon/model-picker.ts @@ -26,10 +26,18 @@ export async function selectModel(): Promise { } const models = await state.listModels(); - if (models.length === 0) { - console.error( - `${YELLOW}No models configured.${RESET} Run: ${DIM}aby start${RESET} -> open the web UI to add a provider.`, - ); + const usable = models.filter((m) => m.available !== false); + if (usable.length === 0) { + if (models.length > 0) { + console.error( + `${YELLOW}No usable models — saved models exist but API keys are missing in the daemon.${RESET} ` + + `Open the web UI to fix provider credentials.`, + ); + } else { + console.error( + `${YELLOW}No models configured.${RESET} Run: ${DIM}aby start${RESET} -> open the web UI to add a provider.`, + ); + } return null; } @@ -40,7 +48,7 @@ export async function selectModel(): Promise { }); try { - const model = await promptModelPicker(models, rl); + const model = await promptModelPicker(usable, rl); if (!model) return null; return { model, state }; } finally { diff --git a/packages/daemon/src/daemon/select-model.test.ts b/packages/daemon/src/daemon/select-model.test.ts index f7a8527..7e51a16 100644 --- a/packages/daemon/src/daemon/select-model.test.ts +++ b/packages/daemon/src/daemon/select-model.test.ts @@ -74,7 +74,30 @@ describe('selectModel', () => { expect(result!.model).toBe('anthropic/claude-sonnet'); expect(result!.state).toBeDefined(); expect(result!.state.listModels).toBeDefined(); - expect(promptModelPicker).toHaveBeenCalledWith(models, expect.anything()); + expect(promptModelPicker).toHaveBeenCalledWith( + models.filter((m) => (m as { available?: boolean }).available !== false), + expect.anything(), + ); + }); + + it('returns null when saved models exist but none are usable', async () => { + const models = [ + { id: 'openrouter/m1', name: 'm1', provider: 'openrouter', available: false }, + ]; + const { DaemonState } = await import('./state.js'); + vi.mocked(DaemonState).mockImplementation(function () { + return mockDaemonState(models); + }); + + const { promptModelPicker } = await import('./chat.js'); + const { selectModel } = await import('./model-picker.js'); + const result = await selectModel(); + + expect(result).toBeNull(); + expect(promptModelPicker).not.toHaveBeenCalled(); + const output = consoleSpy.mock.calls.map((c) => c[0]).join(''); + expect(output).toContain('No usable models'); + expect(output).toContain('API keys are missing'); }); it('returns null when user cancels the picker (Ctrl+C)', async () => { diff --git a/packages/daemon/src/daemon/server/abbenay-service.test.ts b/packages/daemon/src/daemon/server/abbenay-service.test.ts index c23dd30..1d8835d 100644 --- a/packages/daemon/src/daemon/server/abbenay-service.test.ts +++ b/packages/daemon/src/daemon/server/abbenay-service.test.ts @@ -801,6 +801,36 @@ describe('createAbbenayService handlers', () => { expect(response?.workspaces).toEqual(['/ws/a']); }); + it('ListModels omits models that are not usable at runtime', async () => { + const state = createMockState({ + listModels: vi.fn().mockResolvedValue([ + { + id: 'mock/echo', + name: 'echo', + engineModelId: 'echo', + provider: 'mock', + engine: 'mock', + contextWindow: 8192, + available: true, + }, + { + id: 'or/missing-key', + name: 'missing-key', + engineModelId: 'missing-key', + provider: 'or', + engine: 'openrouter', + contextWindow: 0, + available: false, + unavailableReason: 'API key missing', + }, + ]), + }); + const service = createServiceHandlers(state); + const models = await invokeUnary(service.ListModels, {}); + const ids = rpcArray<{ id?: string }>(models.response, 'models').map((m) => m.id); + expect(ids).toEqual(['mock/echo']); + }); + it('ListProviders and ListModels map response fields', async () => { const state = createMockState({ listModels: vi.fn().mockResolvedValue([{ diff --git a/packages/daemon/src/daemon/server/abbenay-service.ts b/packages/daemon/src/daemon/server/abbenay-service.ts index 84e8308..f940b4c 100644 --- a/packages/daemon/src/daemon/server/abbenay-service.ts +++ b/packages/daemon/src/daemon/server/abbenay-service.ts @@ -644,8 +644,9 @@ export function createAbbenayService( ): void { const workspacePaths = call.request.workspace_paths || call.request.workspacePaths || []; state.listModels(workspacePaths).then((models) => { + const usable = models.filter((m) => m.available !== false); callback(null, { - models: models.map((m) => ({ + models: usable.map((m) => ({ id: m.id, name: m.name, engine_model_id: m.engineModelId, diff --git a/packages/daemon/src/daemon/web/openai-compat.test.ts b/packages/daemon/src/daemon/web/openai-compat.test.ts index df82cab..efac6ca 100644 --- a/packages/daemon/src/daemon/web/openai-compat.test.ts +++ b/packages/daemon/src/daemon/web/openai-compat.test.ts @@ -662,6 +662,38 @@ describe('registerOpenAIRoutes', () => { vi.restoreAllMocks(); }); + it('GET /v1/models omits models that are not usable at runtime', async () => { + const state = createOpenAIState([], async () => ([ + { + id: 'mock/echo', + name: 'echo', + engineModelId: 'echo', + provider: 'mock', + engine: 'mock', + contextWindow: 8192, + available: true, + } as ModelInfo, + { + id: 'or/missing', + name: 'missing', + engineModelId: 'missing', + provider: 'or', + engine: 'openrouter', + contextWindow: 0, + available: false, + } as ModelInfo, + ])); + const { server, baseUrl } = await startOpenAIApp(state); + try { + const res = await openaiRequest(baseUrl, 'GET', '/v1/models'); + expect(res.statusCode).toBe(200); + const data = (res.body as { data: Array<{ id: string }> }).data; + expect(data.map((m) => m.id)).toEqual(['mock/echo']); + } finally { + await stopOpenAIApp(server); + } + }); + it('GET /v1/models returns OpenAI-formatted models', async () => { const { server, baseUrl } = await startOpenAIApp(createOpenAIState([])); try { diff --git a/packages/daemon/src/daemon/web/openai-compat.ts b/packages/daemon/src/daemon/web/openai-compat.ts index 14b40a4..e078ce8 100644 --- a/packages/daemon/src/daemon/web/openai-compat.ts +++ b/packages/daemon/src/daemon/web/openai-compat.ts @@ -382,7 +382,7 @@ export function registerOpenAIRoutes(app: Express, state: DaemonState): void { */ app.get('/v1/models', async (_req: Request, res: Response) => { try { - const models = await state.listModels(); + const models = (await state.listModels()).filter((m) => m.available !== false); res.json({ object: 'list', data: models.map(mapModelToOpenAI), diff --git a/packages/daemon/src/daemon/web/server.test.ts b/packages/daemon/src/daemon/web/server.test.ts index ccb8262..0c75959 100644 --- a/packages/daemon/src/daemon/web/server.test.ts +++ b/packages/daemon/src/daemon/web/server.test.ts @@ -121,6 +121,7 @@ interface MockStateOptions { config?: { transport?: string; enabled?: boolean }; }>; throwOnListProviders?: boolean; + listProvidersSpy?: (workspacePaths: string[]) => void; } function createMockState(opts: MockStateOptions): DaemonState { @@ -143,6 +144,7 @@ function createMockState(opts: MockStateOptions): DaemonState { toolRegistryTools = [], mcpPoolStatuses = [], throwOnListProviders = false, + listProvidersSpy, } = opts; let running = mcpRunning; @@ -162,8 +164,9 @@ function createMockState(opts: MockStateOptions): DaemonState { }, notifyModelsChanged() {}, async refreshMcpConnections() {}, - async listProviders() { + async listProviders(workspacePaths: string[] = []) { if (throwOnListProviders) throw new Error('provider list failed'); + listProvidersSpy?.(workspacePaths); return providers; }, async listModels() { return models; }, @@ -523,12 +526,58 @@ describe('createWebApp routes', () => { expect((res.body as { providers: ProviderInfo[] }).providers[0].id).toBe('openai'); }); + it('GET /api/providers passes workspace query to listProviders', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'abbenay-providers-ws-')); + const workspacePaths: string[][] = []; + const state = createMockState({ + sessionsDir: dir, + listProvidersSpy: (paths) => { workspacePaths.push([...paths]); }, + }); + const started = await startTestApp(state); + try { + const res = await httpRequest(started.baseUrl, 'GET', '/api/providers?workspace=%2Ftmp%2Fmy-ws'); + expect(res.statusCode).toBe(200); + expect(workspacePaths).toEqual([['/tmp/my-ws']]); + } finally { + await stopTestApp(started.httpServer); + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + it('GET /api/models returns model list', async () => { const res = await httpRequest(baseUrl, 'GET', '/api/models'); expect(res.statusCode).toBe(200); expect((res.body as { models: ModelInfo[] }).models[0].id).toBe('openai/gpt-4o'); }); + it('GET /api/models exposes availability metadata for unavailable models', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'abbenay-models-avail-')); + const state = createMockState({ + sessionsDir: dir, + models: [{ + id: 'or/m1', + name: 'm1', + engineModelId: 'm1', + provider: 'or', + engine: 'openrouter', + contextWindow: 0, + available: false, + unavailableReason: 'API key missing', + } as ModelInfo], + }); + const started = await startTestApp(state); + try { + const res = await httpRequest(started.baseUrl, 'GET', '/api/models'); + expect(res.statusCode).toBe(200); + const model = (res.body as { models: Array<{ available: boolean; unavailableReason?: string }> }).models[0]; + expect(model.available).toBe(false); + expect(model.unavailableReason).toBe('API key missing'); + } finally { + await stopTestApp(started.httpServer); + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + it('GET /api/workspaces falls back to client workspace paths', async () => { const res = await httpRequest(baseUrl, 'GET', '/api/workspaces'); expect(res.statusCode).toBe(200); diff --git a/packages/daemon/src/daemon/web/server.ts b/packages/daemon/src/daemon/web/server.ts index a6eb379..fe3d185 100644 --- a/packages/daemon/src/daemon/web/server.ts +++ b/packages/daemon/src/daemon/web/server.ts @@ -671,7 +671,9 @@ export function createWebApp(state: DaemonState, options?: WebSecurityOptions): */ app.get('/api/providers', async (req, res) => { try { - const providers = await state.listProviders(); + const workspace = req.query.workspace as string | undefined; + const workspacePaths = workspace ? [workspace] : []; + const providers = await state.listProviders(workspacePaths); res.json({ providers: providers.map((p) => ({ id: p.id, @@ -819,6 +821,8 @@ export function createWebApp(state: DaemonState, options?: WebSecurityOptions): vision: m.capabilities?.supportsVision || false, }, params: m.params, + available: m.available !== false, + unavailableReason: m.unavailableReason, })), }); } catch (err: unknown) { diff --git a/packages/daemon/src/state.test.ts b/packages/daemon/src/state.test.ts index bdff76b..4909b1b 100644 --- a/packages/daemon/src/state.test.ts +++ b/packages/daemon/src/state.test.ts @@ -182,7 +182,7 @@ describe('DaemonState.listModels', () => { expect(models[0].engine).toBe('openrouter'); }); - it('should skip provider without key when engine requires key', async () => { + it('should list saved models as unavailable when key is missing', async () => { const config: ConfigFile = { providers: { openrouter: { @@ -198,7 +198,10 @@ describe('DaemonState.listModels', () => { const models = await state.listModels(); - expect(models).toEqual([]); + expect(models).toHaveLength(1); + expect(models[0].id).toBe('openrouter/model-a'); + expect(models[0].available).toBe(false); + expect(models[0].unavailableReason).toBeDefined(); expect(mockFetchModels).not.toHaveBeenCalled(); }); diff --git a/packages/daemon/static/index.html b/packages/daemon/static/index.html index f9ff3d9..83b99e6 100644 --- a/packages/daemon/static/index.html +++ b/packages/daemon/static/index.html @@ -119,6 +119,8 @@ .status-tag { display: inline-block; padding: 2px 8px; font-size: 12px; font-weight: 500; border-radius: 3px; } .status-tag.configured { background: #e7f1e4; color: #1e4f18; border: 1px solid #3e8635; } .status-tag.not-configured { background: #f0f0f0; color: #151515; border: 1px solid #d2d2d2; } + tr.model-unavailable td { color: #6a6e73; background: #fafafa; } + .model-unavailable-tag { background: #fdf7e7; color: #795600; border: 1px solid #f4c145; font-size: 0.75rem; padding: 2px 8px; border-radius: 3px; } .status-tag.key-missing { background: #fdf7e7; color: #795600; border: 1px solid #f0ab00; } .engine-badge { display: inline-block; padding: 1px 6px; font-size: 11px; font-weight: 500; border-radius: 3px; background: #e7f1fa; color: #0066cc; border: 1px solid #b8daff; } .source-pill { display: inline-block; padding: 1px 6px; font-size: 10px; font-weight: 500; border-radius: 3px; } @@ -483,6 +485,11 @@

Tools

+