Warn accurately about requested CVE IDs absent from the index

Author: afoote-mitre

README.md

@@ -133,6 +133,10 @@ The API omits the top-level `opensearch_ops` ingestion metadata from each `/sear
 
 CVE Records do not exist for years before 1999. A syntactically valid pre-1999 value in `cveIds` is therefore a valid search that contributes no matches rather than a request error. The response includes a `CVE_IDS_NOT_RETURNED` warning listing those IDs. When pre-1999 and supported-year IDs are requested together, matching supported-year records are still returned.
 
+For other requested IDs not already present in the returned page, the API performs at most one additional, ID-only lookup against the configured index. This lookup ignores other filters and pagination, fetches no full records, and handles duplicate documents without omitting another requested ID. `CVE_IDS_NOT_FOUND` lists IDs absent from the searchable index at lookup time; it does not mean those IDs do not exist globally. Existing IDs excluded by another filter or a different page do not receive this warning. Requests without `cveIds`, requests containing only pre-1999 IDs, and requests whose supported-year IDs all appear in the returned page need no additional lookup.
+
+The lookup adds search work and can add latency. Index refresh delays or changes between the main search and lookup can affect visibility; the two operations do not share a snapshot. If lookup verification fails, `CVE_ID_AVAILABILITY_UNVERIFIED` identifies the unchecked IDs while preserving an otherwise successful search, without reporting those IDs as missing. The overall search deadline and client cancellation still apply to the entire request.
+
 Use the optional `scope` field to control which parts of each record participate in matching. Values are case-insensitive: `FULL` (the default) searches base record fields, CNA data, and every ADP provider; `CNA_ONLY` searches base record fields and CNA data; and `ADP_ONLY` searches base record fields and every ADP provider. Scope changes matching only, so successful responses always contain the complete CVE record. CPE applicability filters use CNA data and return HTTP 400 when combined with `ADP_ONLY`.
 
 Date filters accept `YYYY-MM-DD` or timestamps with seconds and an explicit timezone (`Z` or `+/-HH:MM`), with optional fractional seconds of 1 to 3 digits. For example, `2024-01-01`, `2024-01-01T00:00:00Z`, and `2024-01-01T01:00:00+01:00` are accepted. Values are normalized to UTC. A date-only start means `00:00:00.000Z`; a date-only end means `23:59:59.999Z` on that day. Explicit timestamps retain their exact instant, including end boundaries. Impossible dates, timestamps without a timezone, leap seconds, and precision finer than milliseconds return HTTP 400. When both boundaries for publication, modification, SSVC timestamp, or KEV date-added filters are supplied, the start must be before or equal to the end after normalization. All boundaries are inclusive.

api-docs/openapi.json

@@ -685,7 +685,7 @@
             "type": "string",
             "maxLength": 4096,
             "example": "CVE-2024-1234,CVE-2024-5678",
-            "description": "Returns one or more CVE records identified by a comma-separated list of CVE IDs. CVE Records do not exist for years before 1999; those IDs produce no matches and are identified by a non-fatal response warning without preventing other requested IDs from matching. Limited to 100 CVE IDs, 64 characters per ID, and 4096 characters overall."
+            "description": "Returns one or more CVE records identified by a comma-separated list of CVE IDs. CVE Records do not exist for years before 1999; those IDs produce no matches and are identified by a non-fatal response warning (CVE_IDS_NOT_RETURNED) without preventing other requested IDs from matching. Other IDs absent from the returned page are verified through at most one additional ID-only lookup independent of scope, other filters, and pagination. CVE_IDS_NOT_FOUND lists IDs absent from the searchable configured index at lookup time, not necessarily nonexistent globally. Existing IDs excluded by filters or pagination do not receive this warning. Index refresh delays and changes between operations can affect visibility. CVE_ID_AVAILABILITY_UNVERIFIED warns when verification fails without discarding successful search results; overall deadlines and cancellation still apply. Limited to 100 CVE IDs, 64 characters per ID, and 4096 characters overall."
           },
           "noRejected": {
             "oneOf": [

api-docs/openapi.test.js

@@ -94,6 +94,9 @@ describe('OpenAPI contract', () => {
         expect(schema.properties.isVulnerable.oneOf).toStrictEqual(schema.properties.noRejected.oneOf);
         expect(schema.properties.scope.enum).toStrictEqual(['FULL', 'CNA_ONLY', 'ADP_ONLY']);
         expect(schema.properties.scope.default).toBe('FULL');
+        expect(schema.properties.cveIds.description).toContain('CVE_IDS_NOT_FOUND');
+        expect(schema.properties.cveIds.description).toContain('CVE_ID_AVAILABILITY_UNVERIFIED');
+        expect(schema.properties.cveIds.description).toContain('independent of scope, other filters, and pagination');
         expect(schema.properties.scope.description).toContain('does not remove data');
         expect(schema.properties.ssvcExploitation.enum).toStrictEqual(['NONE', 'POC', 'ACTIVE']);
         expect(schema.properties.ssvcAutomatable.enum).toStrictEqual(['YES', 'NO']);

controllers/searchController.js

@@ -5,6 +5,7 @@ const searchProvider = require('../utils/searchProvider');
 const searchQueryBuilder = require('../utils/searchQueryBuilder');
 const searchPostFilters = require('../utils/searchPostFilters');
 const searchResponseBuilder = require('../utils/searchResponseBuilder');
+const searchCveIdWarnings = require('../utils/searchCveIdWarnings');
 const webSearchQueryBuilder = require('../utils/webSearchQueryBuilder');
 const error = new errors.errors();
 const openSearchUnavailableErrorNames = new Set(['ConnectionError', 'NoLivingConnectionsError']);
@@ -162,7 +163,19 @@ async function searchHandler(req, res, next) {
         );
         lifecycle.throwIfAborted();
         const pagingOptions = responseOptions ?? searchOptions;
-        const warnings = [...searchResponseBuilder.getRequestWarnings(params), ...searchResult.warnings];
+        const availabilityWarnings = await searchCveIdWarnings.getAvailabilityWarnings(
+            params.cveIds,
+            searchResult.hits,
+            {
+                signal: lifecycle.signal
+            }
+        );
+        lifecycle.throwIfAborted();
+        const warnings = [
+            ...searchResponseBuilder.getRequestWarnings(params),
+            ...availabilityWarnings,
+            ...searchResult.warnings
+        ];
 
         return res.status(200).json(
             searchResponseBuilder.buildSearchResponse({

routes/search.integration.test.js

@@ -28,6 +28,73 @@ describe('search route integration', () => {
         mockSearchOpenSearch.mockReset();
     });
 
+    test('returns matching IDs and warns only about IDs absent from the index', async () => {
+        const record = { cveMetadata: { cveId: 'CVE-2024-0001' } };
+        mockSearchOpenSearch
+            .mockResolvedValueOnce(advancedResponse(1, [{ _source: record }]).data)
+            .mockResolvedValueOnce({
+                aggregations: { requested_ids: { buckets: { 'CVE-2099-99999999': { doc_count: 0 } } } }
+            });
+        const response = await request(createApp(), {
+            method: 'POST',
+            path: '/search',
+            body: { cveIds: 'CVE-1998-0001,cve-2024-0001,CVE-2099-99999999,CVE-2099-99999999', resultsPerPage: 1 }
+        });
+        expect(response.status).toBe(200);
+        expect(response.body.records).toStrictEqual([record]);
+        expect(response.body.pagination.totalResults).toBe(1);
+        expect(response.body.warnings).toStrictEqual([
+            expect.objectContaining({ code: 'CVE_IDS_NOT_RETURNED', cveIds: ['CVE-1998-0001'] }),
+            expect.objectContaining({ code: 'CVE_IDS_NOT_FOUND', cveIds: ['CVE-2099-99999999'] })
+        ]);
+        expect(Object.keys(response.body).indexOf('warnings')).toBeLessThan(
+            Object.keys(response.body).indexOf('records')
+        );
+        expect(mockSearchOpenSearch).toHaveBeenCalledTimes(2);
+    });
+
+    test.each([
+        { resultsPerPage: 1, pageNumber: 2 },
+        { keywordSearch: 'nonmatching-text', scope: 'ADP_ONLY' },
+        { noRejected: true },
+        { virtualMatchString: 'cpe:2.3:o:linux:linux_kernel' }
+    ])('does not report an existing ID as missing when excluded by filters or pagination: %p', async (filters) => {
+        mockSearchOpenSearch.mockResolvedValueOnce(advancedResponse(0).data).mockResolvedValueOnce({
+            aggregations: { requested_ids: { buckets: { 'CVE-2024-0001': { doc_count: 1 } } } }
+        });
+        const response = await request(createApp(), {
+            method: 'POST',
+            path: '/search',
+            body: { cveIds: 'CVE-2024-0001', ...filters }
+        });
+        expect(response.status).toBe(200);
+        expect(response.body.records).toStrictEqual([]);
+        expect(response.body.warnings).toStrictEqual([]);
+        expect(mockSearchOpenSearch.mock.calls[1][0]).toMatchObject({
+            size: 0,
+            _source: false,
+            query: { bool: { filter: [{ terms: { 'cveMetadata.cveId.keyword': ['CVE-2024-0001'] } }] } }
+        });
+    });
+
+    test('returns a successful search with a sanitized warning when only the ID lookup fails', async () => {
+        const record = { cveMetadata: { cveId: 'CVE-2024-0002' } };
+        mockSearchOpenSearch
+            .mockResolvedValueOnce(advancedResponse(1, [{ _source: record }]).data)
+            .mockRejectedValueOnce(new Error('private backend credentials'));
+        const response = await request(createApp(), {
+            method: 'POST',
+            path: '/search',
+            body: { cveIds: 'CVE-2024-0001,CVE-2024-0002' }
+        });
+        expect(response.status).toBe(200);
+        expect(response.body.records).toStrictEqual([record]);
+        expect(response.body.warnings).toStrictEqual([
+            expect.objectContaining({ code: 'CVE_ID_AVAILABILITY_UNVERIFIED', cveIds: ['CVE-2024-0001'] })
+        ]);
+        expect(response.text).not.toContain('private');
+    });
+
     test('date-only boundaries and offsets reach OpenSearch as inclusive UTC ranges', async () => {
         mockSearchOpenSearch.mockResolvedValue(advancedResponse(0).data);
         const response = await request(createApp(), {

routes/swagger.js

@@ -69,7 +69,7 @@ const searchRequestProperties = {
         coercibleBooleanSchema()
     ),
     cveIds: searchRequestProperty(
-        `Returns one or more CVE records identified by a comma-separated list of CVE IDs. CVE Records do not exist for years before ${constants.FIRST_CVE_RECORD_YEAR}; those IDs produce no matches and are identified by a non-fatal response warning without preventing other requested IDs from matching. Limited to ${constants.MAX_CVE_IDS} CVE IDs, ${constants.MAX_CVE_ID_LENGTH} characters per ID, and ${constants.SEARCH_STRING_LENGTH_LIMITS.cveIds} characters overall.`,
+        `Returns one or more CVE records identified by a comma-separated list of CVE IDs. CVE Records do not exist for years before ${constants.FIRST_CVE_RECORD_YEAR}; those IDs produce no matches and are identified by a non-fatal response warning (CVE_IDS_NOT_RETURNED) without preventing other requested IDs from matching. Other IDs absent from the returned page are verified through at most one additional ID-only lookup independent of scope, other filters, and pagination. CVE_IDS_NOT_FOUND lists IDs absent from the searchable configured index at lookup time, not necessarily nonexistent globally. Existing IDs excluded by filters or pagination do not receive this warning. Index refresh delays and changes between operations can affect visibility. CVE_ID_AVAILABILITY_UNVERIFIED warns when verification fails without discarding successful search results; overall deadlines and cancellation still apply. Limited to ${constants.MAX_CVE_IDS} CVE IDs, ${constants.MAX_CVE_ID_LENGTH} characters per ID, and ${constants.SEARCH_STRING_LENGTH_LIMITS.cveIds} characters overall.`,
         {
             type: 'string',
             maxLength: constants.SEARCH_STRING_LENGTH_LIMITS.cveIds,