From 7124899f34078485bbda3c69882916b4999d96fe Mon Sep 17 00:00:00 2001 From: Jialin Huang Date: Tue, 13 Oct 2026 16:02:42 +0800 Subject: [PATCH] Scope repository-relative knowledge ingestion to configured paths Preserve repository-relative blob identities while avoiding repeated shared-root scans. Add hermetic generic regressions and document global ingestion and cleanup migration impact. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../azure-sdk-qa-bot-knowledge-sync/README.md | 27 ++ .../src/DailySyncKnowledge.ts | 28 +- .../src/services/ConfigurationLoader.ts | 2 + .../tests/configurationLoader.test.ts | 23 ++ .../tests/sourcePathScoping.test.ts | 254 ++++++++++++++++++ 5 files changed, 325 insertions(+), 9 deletions(-) create mode 100644 tools/sdk-ai-bots/azure-sdk-qa-bot-knowledge-sync/tests/sourcePathScoping.test.ts diff --git a/tools/sdk-ai-bots/azure-sdk-qa-bot-knowledge-sync/README.md b/tools/sdk-ai-bots/azure-sdk-qa-bot-knowledge-sync/README.md index 1c47c3cde90..4c465b12502 100644 --- a/tools/sdk-ai-bots/azure-sdk-qa-bot-knowledge-sync/README.md +++ b/tools/sdk-ai-bots/azure-sdk-qa-bot-knowledge-sync/README.md @@ -39,6 +39,33 @@ The application uses `config/knowledge-config.json` to define which repositories - Documentation paths within repositories - Processing options and filters +`path` may select a directory or an individual Markdown/MDX file. +`relativeByRepoPath: true` keeps blob filenames relative to the repository root +for citations; scanning is limited to the selected `path`. Without a `path`, +processing still scans the entire available repository checkout. Ordinary sources +without `relativeByRepoPath` retain directory-relative filenames, or the basename +for an individual file. Existing Markdown/MDX, ignored-path, reference, dated +release-note, and metadata filters remain unchanged. Release-note index discovery +still checks the source root's `release-notes` directory, but skips it when outside +the selected path; it does not discover new nested release-note directories. + +Previously, `relativeByRepoPath: true` used `path` only for sparse checkout and +scanned the shared repository root for every entry. Multiple entries sharing a +destination folder could produce duplicate blob keys and overwrite metadata. +Each entry now ingests only its configured path while preserving repository-relative +citations. Selected paths sharing a destination folder should not overlap, +especially when assigning different metadata. + +This change applies globally to all `relativeByRepoPath` sources. Root or ancestor +Markdown incidentally fetched by sparse checkout but outside selected paths will +no longer be ingested. On a later authorized sync, `cleanupExpiredBlobs` can remove +previously indexed incidental blobs absent from the new results. Review the full +configuration and this migration impact before syncing; never run sync against a +partial source list, because cleanup removes blobs absent from its results. +Code/configuration changes alone do not sync, index, or deploy knowledge. +The companion .NET knowledge-source change, [#17253](https://github.com/Azure/azure-sdk-tools/pull/17253), +depends on this generic path-scoping change merging first. + ## Building and Running ### Prerequisites diff --git a/tools/sdk-ai-bots/azure-sdk-qa-bot-knowledge-sync/src/DailySyncKnowledge.ts b/tools/sdk-ai-bots/azure-sdk-qa-bot-knowledge-sync/src/DailySyncKnowledge.ts index 60ca85a9140..7e00ff2776c 100644 --- a/tools/sdk-ai-bots/azure-sdk-qa-bot-knowledge-sync/src/DailySyncKnowledge.ts +++ b/tools/sdk-ai-bots/azure-sdk-qa-bot-knowledge-sync/src/DailySyncKnowledge.ts @@ -103,8 +103,9 @@ export async function processDailySyncKnowledge(): Promise { const sourceDir = path.join(workingDir, source.path); const targetDir = path.join(tempDocsDir, source.folder); - if (!fs.existsSync(sourceDir)) { - console.warn(`Source directory not found: ${sourceDir}`); + const scanPath = source.includePath ? path.join(sourceDir, source.includePath) : sourceDir; + if (!fs.existsSync(scanPath)) { + console.warn(`Source path not found: ${scanPath}`); continue; } @@ -359,12 +360,12 @@ async function setupDocumentationRepositories(docsDir: string): Promise { /** * Process files in a source directory */ -async function processSourceDirectory( +export async function processSourceDirectory( sourceDir: string, source: DocumentationSource, targetDir: string, existingBlobs: Map, - blobService: BlobService + blobService: Pick ): Promise { let totalProcessed = 0; @@ -422,11 +423,12 @@ async function processSourceDirectory( } }; - // Check if sourceDir is actually a file on the filesystem - const isFile = fs.existsSync(sourceDir) && fs.statSync(sourceDir).isFile(); + // Selection stays narrow while filenames remain relative to the repository root. + const scanPath = source.includePath ? path.join(sourceDir, source.includePath) : sourceDir; + const isFile = fs.statSync(scanPath).isFile(); if (isFile) { - processSingleFile(sourceDir, path.dirname(sourceDir)); + processSingleFile(scanPath, source.includePath ? sourceDir : path.dirname(sourceDir)); return { totalProcessed, changedDocuments, unchangedDocuments, changedFiles, metadataChangedFiles, unchangedFiles }; } @@ -461,7 +463,7 @@ async function processSourceDirectory( } } - await walkDirectory(sourceDir); + await walkDirectory(scanPath); return { totalProcessed, @@ -476,13 +478,21 @@ async function processSourceDirectory( /** * Create release notes index with content from the 10 most recent release notes */ -async function createReleaseNotesIndex( +export async function createReleaseNotesIndex( source: DocumentationSource, sourceDir: string, targetDir: string ): Promise { // Path to release notes directory const releaseNotesDir = path.join(sourceDir, 'release-notes'); + + if (source.includePath) { + const selectedPath = path.join(sourceDir, source.includePath); + const relativeReleasePath = path.relative(selectedPath, releaseNotesDir); + if (relativeReleasePath.startsWith('..') || path.isAbsolute(relativeReleasePath)) { + return; + } + } // Check if release notes directory exists if (!fs.existsSync(releaseNotesDir)) { diff --git a/tools/sdk-ai-bots/azure-sdk-qa-bot-knowledge-sync/src/services/ConfigurationLoader.ts b/tools/sdk-ai-bots/azure-sdk-qa-bot-knowledge-sync/src/services/ConfigurationLoader.ts index b8728ea1e6e..ee6a14a000b 100644 --- a/tools/sdk-ai-bots/azure-sdk-qa-bot-knowledge-sync/src/services/ConfigurationLoader.ts +++ b/tools/sdk-ai-bots/azure-sdk-qa-bot-knowledge-sync/src/services/ConfigurationLoader.ts @@ -73,6 +73,7 @@ export interface KnowledgeConfig { */ export interface DocumentationSource { path: string; + includePath?: string; folder: string; fileNameLowerCase?: boolean; ignoredPaths?: string[]; @@ -138,6 +139,7 @@ export class ConfigurationLoader { for (const docPath of source.paths) { sources.push({ path: (docPath.relativeByRepoPath || docPath.path === undefined) ? `docs/${repoPath}` : `docs/${repoPath}/${docPath.path}`, + includePath: docPath.relativeByRepoPath ? docPath.path?.replace(/^[/\\]+/, '') : undefined, folder: docPath.folder, fileNameLowerCase: docPath.fileNameLowerCase, ignoredPaths: docPath.ignoredPaths, diff --git a/tools/sdk-ai-bots/azure-sdk-qa-bot-knowledge-sync/tests/configurationLoader.test.ts b/tools/sdk-ai-bots/azure-sdk-qa-bot-knowledge-sync/tests/configurationLoader.test.ts index d941a10843f..7488dbbc9d4 100644 --- a/tools/sdk-ai-bots/azure-sdk-qa-bot-knowledge-sync/tests/configurationLoader.test.ts +++ b/tools/sdk-ai-bots/azure-sdk-qa-bot-knowledge-sync/tests/configurationLoader.test.ts @@ -67,12 +67,35 @@ describe('ConfigurationLoader', () => { expect(repos[0].sparseCheckout).toEqual(['docs/part1', 'docs/part2']); }); + it('keeps repository-relative citation roots but selects only the configured path', () => { + const temp = writeTempConfig({ + version: '1.0.0', + sources: [{ + repository: { url: 'https://github.com/org/repo.git', branch: 'main', authType: 'public' }, + paths: [ + { description: 'Docs', path: '/doc', folder: 'docs', relativeByRepoPath: true }, + { description: 'Readme', path: '/eng/tool/README.md', folder: 'docs', relativeByRepoPath: true }, + { description: 'Entire repo', folder: 'whole_repo', relativeByRepoPath: true } + ] + }] + }); + ConfigurationLoader.setConfigPath(temp); + + const docs = ConfigurationLoader.getDocumentationSources(); + expect(docs.map(source => source.path)).toEqual(['docs/repo', 'docs/repo', 'docs/repo']); + expect(docs.map(source => source.includePath)).toEqual(['doc', 'eng/tool/README.md', undefined]); + expect(ConfigurationLoader.getRepositoryConfigs()[0].sparseCheckout).toEqual([ + '/doc', '/eng/tool/README.md' + ]); + }); + it('throws when file missing', () => { const missing = path.join(process.cwd(), 'definitely-missing-config.json'); ConfigurationLoader.setConfigPath(missing); expect(() => ConfigurationLoader.getDocumentationSources()).toThrow(); }); afterAll(() => { + ConfigurationLoader.setConfigPath(originalPath); for (const dir of tempDirs) { try { fs.rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ } } diff --git a/tools/sdk-ai-bots/azure-sdk-qa-bot-knowledge-sync/tests/sourcePathScoping.test.ts b/tools/sdk-ai-bots/azure-sdk-qa-bot-knowledge-sync/tests/sourcePathScoping.test.ts new file mode 100644 index 00000000000..91e00790bf4 --- /dev/null +++ b/tools/sdk-ai-bots/azure-sdk-qa-bot-knowledge-sync/tests/sourcePathScoping.test.ts @@ -0,0 +1,254 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { + ConfigurationLoader, + DocumentationPath, + DocumentationSource, + KnowledgeConfig, +} from '../src/services/ConfigurationLoader'; +import { createReleaseNotesIndex, processSourceDirectory } from '../src/DailySyncKnowledge'; + +const originalConfigPath = ConfigurationLoader.getConfigPath(); +const tempDirs: string[] = []; +const blobService = { + hasContentChanged: vi.fn(() => true), + hasMetadataChanged: vi.fn(() => false), +}; + +afterEach(() => { + ConfigurationLoader.setConfigPath(originalConfigPath); + vi.clearAllMocks(); + for (const dir of tempDirs.splice(0)) { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +function createFixture(): { root: string; repoRoot: string; target: string } { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'knowledge-path-scoping-')); + tempDirs.push(root); + const repoRoot = path.join(root, 'repo'); + const target = path.join(root, 'target'); + fs.mkdirSync(repoRoot); + fs.mkdirSync(target); + return { root, repoRoot, target }; +} + +function writeMarkdown(root: string, relativePath: string, content = '# Documentation\nContent'): void { + const file = path.join(root, relativePath); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, content); +} + +function loadSources(root: string, paths: DocumentationPath[]): DocumentationSource[] { + const config: KnowledgeConfig = { + version: '1.0.0', + sources: [{ + repository: { + url: 'https://github.com/example/fixture-repo.git', + path: 'fixture-checkout', + branch: 'main', + authType: 'public', + }, + paths, + }], + }; + const configPath = path.join(root, 'knowledge-config.json'); + fs.writeFileSync(configPath, JSON.stringify(config)); + ConfigurationLoader.setConfigPath(configPath); + return ConfigurationLoader.getDocumentationSources(); +} + +describe('source path scoping', () => { + it('separates repository citation roots from directory/file selections and sparse checkout', () => { + const { root } = createFixture(); + const sources = loadSources(root, [ + { name: 'Guide', description: '', path: '/guides', folder: 'shared', relativeByRepoPath: true }, + { name: 'Readme', description: '', path: '\\tools\\widget\\README.md', folder: 'shared', relativeByRepoPath: true }, + { name: 'Ordinary docs', description: '', path: 'ordinary/docs', folder: 'ordinary' }, + { name: 'Ordinary file', description: '', path: 'ordinary/README.md', folder: 'ordinary' }, + ]); + + expect(sources.map(source => source.path)).toEqual([ + 'docs/fixture-checkout', + 'docs/fixture-checkout', + 'docs/fixture-checkout/ordinary/docs', + 'docs/fixture-checkout/ordinary/README.md', + ]); + expect(sources.map(source => source.includePath)).toEqual([ + 'guides', 'tools\\widget\\README.md', undefined, undefined, + ]); + expect(ConfigurationLoader.getRepositoryConfigs()[0].sparseCheckout).toEqual([ + '/guides', '\\tools\\widget\\README.md', 'ordinary/docs', 'ordinary/README.md', + ]); + }); + + it('ingests disjoint shared-folder entries once with stable filenames and independent metadata', async () => { + const { root, repoRoot, target } = createFixture(); + const sources = loadSources(root, [ + { + name: 'Management', description: '', path: '/guides/management', folder: 'shared', + relativeByRepoPath: true, metadata: { scope: 'branded', service_type: 'management-plane' }, + }, + { + name: 'Data', description: '', path: '/guides/data', folder: 'shared', + relativeByRepoPath: true, metadata: { scope: 'branded', service_type: 'data-plane' }, + }, + { + name: 'Skill', description: '', path: '/skills/review', folder: 'shared', + relativeByRepoPath: true, metadata: { scope: 'unbranded' }, + }, + ]); + const selectedFiles = [ + 'guides/management/guide.md', + 'guides/management/index.mdx', + 'guides/data/guide.md', + 'skills/review/SKILL.md', + 'skills/review/references/details.md', + ]; + for (const file of [...selectedFiles, 'README.md', 'guides/README.md', 'unrelated/guide.md', 'skills/other/SKILL.md']) { + writeMarkdown(repoRoot, file); + } + + const blobs: string[] = []; + for (const source of sources) { + const result = await processSourceDirectory(repoRoot, source, target, new Map(), blobService); + expect(result.totalProcessed).toBe(result.changedFiles.length); + for (const file of result.changedFiles) { + blobs.push(file.blobPath); + expect(file.blobPath).toBe(`shared/${file.filename}`); + expect(file.metadata).toEqual(source.metadata); + expect(fs.readFileSync(path.join(target, file.filename), 'utf-8')).toBe(file.content); + } + } + expect(blobs.sort()).toEqual(selectedFiles.map(file => `shared/${file.replace(/\//g, '#')}`).sort()); + expect(new Set(blobs).size).toBe(blobs.length); + expect(blobService.hasContentChanged).toHaveBeenCalledTimes(selectedFiles.length); + expect(blobService.hasMetadataChanged).toHaveBeenCalledTimes(selectedFiles.length); + }); + + it('preserves repository-relative identities for selected files with the same basename', async () => { + const { root, repoRoot, target } = createFixture(); + const sources = loadSources(root, ['tools/one/README.md', 'tools/two/README.md'].map(selected => ({ + name: selected, description: '', path: `/${selected}`, folder: 'shared', relativeByRepoPath: true, + }))); + writeMarkdown(repoRoot, 'tools/one/README.md', '# One\nFirst content'); + writeMarkdown(repoRoot, 'tools/two/README.md', '# Two\nSecond content'); + writeMarkdown(repoRoot, 'tools/one/unselected.md'); + const filenames: string[] = []; + for (const source of sources) { + const result = await processSourceDirectory(repoRoot, source, target, new Map(), blobService); + expect(result.totalProcessed).toBe(1); + filenames.push(result.changedFiles[0].filename); + } + expect(filenames).toEqual(['tools#one#README.md', 'tools#two#README.md']); + expect(fs.readFileSync(path.join(target, filenames[0]), 'utf-8')).toContain('First content'); + expect(fs.readFileSync(path.join(target, filenames[1]), 'utf-8')).toContain('Second content'); + }); + + it('keeps pathless repository-relative sources scanning the whole available repository', async () => { + const { root, repoRoot, target } = createFixture(); + const [source] = loadSources(root, [{ + name: 'Entire repo', description: '', folder: 'whole_repo', relativeByRepoPath: true, + }]); + expect(source.path).toBe('docs/fixture-checkout'); + expect(source.includePath).toBeUndefined(); + expect(ConfigurationLoader.getRepositoryConfigs()[0].sparseCheckout).toBeUndefined(); + writeMarkdown(repoRoot, 'README.md'); + writeMarkdown(repoRoot, 'nested/guide.mdx'); + const result = await processSourceDirectory(repoRoot, source, target, new Map(), blobService); + expect(result.changedFiles.map(file => file.blobPath).sort()).toEqual([ + 'whole_repo/README.md', 'whole_repo/nested#guide.mdx', + ]); + }); + + it('preserves ordinary directory/file identities and ignored/reference/dated-release filters', async () => { + const { root, repoRoot, target } = createFixture(); + const [source, fileSource] = loadSources(root, [ + { name: 'Docs', description: '', path: 'docs', folder: 'ordinary', ignoredPaths: ['ignored'] }, + { name: 'File', description: '', path: 'docs/nested/guide.md', folder: 'ordinary' }, + ]); + for (const file of [ + 'nested/guide.md', 'nested/index.mdx', 'ignored/guide.md', 'reference/guide.md', + 'release-2026-01-01.md', 'nested/release-2026-01-02.mdx', + ]) { + writeMarkdown(repoRoot, `docs/${file}`); + } + fs.writeFileSync(path.join(repoRoot, 'docs', 'not-markdown.txt'), 'Unselected'); + const result = await processSourceDirectory( + path.join(repoRoot, 'docs'), source, target, new Map(), blobService); + expect(result.changedFiles.map(file => file.blobPath).sort()).toEqual([ + 'ordinary/nested#guide.md', 'ordinary/nested#index.mdx', + ]); + const fileResult = await processSourceDirectory( + path.join(repoRoot, 'docs', 'nested', 'guide.md'), fileSource, target, new Map(), blobService); + expect(fileResult.changedFiles.map(file => file.blobPath)).toEqual(['ordinary/guide.md']); + }); + + it('keeps ignored paths and metadata overrides relative to the repository root', async () => { + const { root, repoRoot, target } = createFixture(); + const [source] = loadSources(root, [{ + name: 'Selected docs', description: '', path: '/docs', folder: 'shared', relativeByRepoPath: true, + ignoredPaths: [path.join('docs', 'ignored')], + metadata: { scope: 'branded', service_type: 'data-plane' }, + overrides: [{ + pattern: 'docs/management/**', + metadata: { scope: 'branded', service_type: 'management-plane' }, + }], + }]); + writeMarkdown(repoRoot, 'docs/data/guide.md'); + writeMarkdown(repoRoot, 'docs/management/guide.md'); + writeMarkdown(repoRoot, 'docs/ignored/guide.md'); + writeMarkdown(repoRoot, 'docs/release-2026-01-01.mdx'); + const result = await processSourceDirectory(repoRoot, source, target, new Map(), blobService); + expect(result.changedFiles.map(file => [file.blobPath, file.metadata?.service_type]).sort()).toEqual([ + ['shared/docs#data#guide.md', 'data-plane'], + ['shared/docs#management#guide.md', 'management-plane'], + ]); + }); + + it('reports a missing selected path instead of falling back to scanning the repository', async () => { + const { repoRoot, target } = createFixture(); + writeMarkdown(repoRoot, 'README.md'); + await expect(processSourceDirectory( + repoRoot, { path: repoRoot, includePath: 'missing', folder: 'shared' }, + target, new Map(), blobService, + )).rejects.toMatchObject({ code: 'ENOENT' }); + expect(blobService.hasContentChanged).not.toHaveBeenCalled(); + expect(fs.readdirSync(target)).toEqual([]); + }); + + it('skips root release-note discovery when it is outside a selected directory or file', async () => { + const { repoRoot, target } = createFixture(); + writeMarkdown(repoRoot, 'release-notes/release-2026-01-01.md'); + writeMarkdown(repoRoot, 'docs/guide.md'); + for (const includePath of ['docs', 'docs/guide.md', 'release-notes/release-2026-01-01.md']) { + await createReleaseNotesIndex( + { path: repoRoot, includePath, folder: 'typespec_docs' }, repoRoot, target); + expect(fs.readdirSync(target)).toEqual([]); + } + }); + + it('retains existing root release-note discovery when the selection includes that directory', async () => { + const { repoRoot, target } = createFixture(); + writeMarkdown(repoRoot, 'release-notes/release-2026-01-01.md', + '---\ntitle: January\nreleaseDate: 2026-01-01\n---\n## Features\nNew feature'); + for (const includePath of [undefined, 'release-notes', '.']) { + await createReleaseNotesIndex( + { path: repoRoot, includePath, folder: 'typespec_docs' }, repoRoot, target); + const index = fs.readFileSync(path.join(target, 'version-release-notes-index.md'), 'utf-8'); + expect(index).toContain('January'); + expect(index).toContain('New feature'); + fs.unlinkSync(path.join(target, 'version-release-notes-index.md')); + } + }); + + it('does not introduce discovery of nested release-note directories', async () => { + const { repoRoot, target } = createFixture(); + writeMarkdown(repoRoot, 'docs/release-notes/release-2026-01-01.md'); + await createReleaseNotesIndex( + { path: repoRoot, includePath: 'docs', folder: 'typespec_docs' }, repoRoot, target); + expect(fs.readdirSync(target)).toEqual([]); + }); +});