From b2b508492242dc105456211d1a9ac729fa2297ef Mon Sep 17 00:00:00 2001 From: Alex Rawlings Date: Mon, 10 Aug 2026 15:52:30 -0600 Subject: [PATCH 01/18] Add the pure query core for the analysis catalog Row derivation, search fold, and facets, memoized as selectCatalogRows. Row building and query application are separate functions so searching and sorting never rebuild rows; the search fold stays apart from normalizeSurfaceForm, since folding a diacritic into identity would silently merge distinct analyses. --- src/__tests__/store/analysisSlice.test.ts | 23 + src/__tests__/utils/analysis-identity.test.ts | 8 + src/__tests__/utils/analysis-query.test.ts | 502 ++++++++++++++++++ src/__tests__/utils/search-fold.test.ts | 21 + src/store/analysisSlice.ts | 16 + src/utils/analysis-query.ts | 289 ++++++++++ src/utils/search-fold.ts | 19 + 7 files changed, 878 insertions(+) create mode 100644 src/__tests__/utils/analysis-query.test.ts create mode 100644 src/__tests__/utils/search-fold.test.ts create mode 100644 src/utils/analysis-query.ts create mode 100644 src/utils/search-fold.ts diff --git a/src/__tests__/store/analysisSlice.test.ts b/src/__tests__/store/analysisSlice.test.ts index 6363e0d2..e4ecf012 100644 --- a/src/__tests__/store/analysisSlice.test.ts +++ b/src/__tests__/store/analysisSlice.test.ts @@ -18,6 +18,7 @@ import { mergePhrases, selectApprovedGloss, selectApprovedMorphemes, + selectCatalogRows, selectMorphemeResetLosesGlosses, selectPhraseLinkByTokenRef, selectPhraseGloss, @@ -1855,6 +1856,28 @@ describe('selectPoolIndex', () => { }); }); +describe('selectCatalogRows', () => { + it('returns the same reference for repeated calls on unchanged state (memoized)', () => { + const store = createAnalysisStore(); + + const a = selectCatalogRows(store.getState().analysis, 'GEN'); + const b = selectCatalogRows(store.getState().analysis, 'GEN'); + + expect(a).toBe(b); + }); + + it('rebuilds the rows after a gloss write', () => { + const store = createAnalysisStore(); + const before = selectCatalogRows(store.getState().analysis, 'GEN'); + + store.dispatch(writeGloss('GEN 1:1:0', 'λόγος', 'word')); + + const after = selectCatalogRows(store.getState().analysis, 'GEN'); + expect(after).not.toBe(before); + expect(after.map((row) => row.gloss)).toEqual(['word']); + }); +}); + describe('selectResolvedTokenAnalysis', () => { it('returns the approved analysis when the token has one', () => { const ta: TokenAnalysis = { diff --git a/src/__tests__/utils/analysis-identity.test.ts b/src/__tests__/utils/analysis-identity.test.ts index bb1ad959..5584bbc5 100644 --- a/src/__tests__/utils/analysis-identity.test.ts +++ b/src/__tests__/utils/analysis-identity.test.ts @@ -3,6 +3,7 @@ import type { TokenAnalysis } from 'interlinearizer'; import { FIXTURE_STAMPS } from '../test-helpers'; import { analysesAreIdentical, normalizeSurfaceForm } from '../../utils/analysis-identity'; +import { foldForSearch } from '../../utils/search-fold'; describe('normalizeSurfaceForm', () => { it('lowercases so a sentence-initial form matches a mid-sentence form', () => { @@ -15,6 +16,13 @@ describe('normalizeSurfaceForm', () => { expect(decomposed).not.toBe(composed); // distinct raw strings... expect(normalizeSurfaceForm(decomposed)).toBe(normalizeSurfaceForm(composed)); // ...one key once normalized. }); + + // The two normalizations must stay separate: identity under-matches so a recorded distinction is + // never merged away, while the search fold over-matches so a query finds an accented form. + it('keeps a diacritic difference the search fold collapses', () => { + expect(normalizeSurfaceForm('ἀρχῇ')).not.toBe(normalizeSurfaceForm('αρχη')); + expect(foldForSearch('ἀρχῇ')).toBe(foldForSearch('αρχη')); + }); }); /** diff --git a/src/__tests__/utils/analysis-query.test.ts b/src/__tests__/utils/analysis-query.test.ts new file mode 100644 index 00000000..52c48663 --- /dev/null +++ b/src/__tests__/utils/analysis-query.test.ts @@ -0,0 +1,502 @@ +/// + +import type { AssignmentStatus, TextAnalysis, TokenAnalysisLink } from 'interlinearizer'; +import { emptyAnalysis } from '../../types/empty-factories'; +import { + applyCatalogQuery, + buildCatalogRows, + deriveFacets, + type CatalogQuery, + type CatalogScope, +} from '../../utils/analysis-query'; + +const scope: CatalogScope = { analysisLanguage: 'en', currentBook: 'GEN' }; + +/** Builds a query that neither searches nor filters, so each test varies one dimension. */ +function makeQuery(overrides: Partial = {}): CatalogQuery { + return { + search: '', + sort: 'usageCount', + filters: {}, + surfaceCollator: new Intl.Collator('el'), + glossCollator: new Intl.Collator('en'), + ...overrides, + }; +} + +/** Builds a link from `tokenRef` to the analysis, approved unless another status is given. */ +function link( + analysisId: string, + tokenRef: string, + status: AssignmentStatus = 'approved', +): TokenAnalysisLink { + return { analysisId, status, token: { tokenRef, surfaceText: 'word' } }; +} + +describe('buildCatalogRows', () => { + it('builds one row per token analysis, carrying its id and surface form', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { id: 'ta-1', surfaceText: 'ἀρχῇ' }, + { id: 'ta-2', surfaceText: 'λόγος' }, + ], + }; + + expect(buildCatalogRows(analysis, scope).map((r) => [r.analysisId, r.surfaceText])).toEqual([ + ['ta-1', 'ἀρχῇ'], + ['ta-2', 'λόγος'], + ]); + }); + + it('counts each approved link on a payload as a usage', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [{ id: 'ta-1', surfaceText: 'λόγος' }], + tokenAnalysisLinks: [link('ta-1', 'GEN 1:1:0'), link('ta-1', 'GEN 1:2:4')], + }; + + expect(buildCatalogRows(analysis, scope)[0].usageCount).toBe(2); + }); + + it('does not count a rejected link as a usage', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [{ id: 'ta-1', surfaceText: 'λόγος' }], + tokenAnalysisLinks: [link('ta-1', 'GEN 1:1:0', 'rejected')], + }; + + expect(buildCatalogRows(analysis, scope)[0].usageCount).toBe(0); + }); + + it('lists a payload with no links at all as a zero-usage row', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [{ id: 'ta-1', surfaceText: 'λόγος' }], + }; + + expect(buildCatalogRows(analysis, scope)).toHaveLength(1); + expect(buildCatalogRows(analysis, scope)[0].usageCount).toBe(0); + }); + + it('counts only usages in the scope book toward the per-book count', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [{ id: 'ta-1', surfaceText: 'λόγος' }], + tokenAnalysisLinks: [ + link('ta-1', 'GEN 1:1:0'), + link('ta-1', 'GEN 1:2:4'), + link('ta-1', 'JHN 1:1:0'), + ], + }; + + const [row] = buildCatalogRows(analysis, scope); + expect(row.usageCount).toBe(3); + expect(row.usageCountInBook).toBe(2); + }); + + it('parses a usage location out of the token ref', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [{ id: 'ta-1', surfaceText: 'λόγος' }], + tokenAnalysisLinks: [link('ta-1', 'GEN 1:5:12')], + }; + + expect(buildCatalogRows(analysis, scope)[0].usages).toEqual([ + { tokenRef: 'GEN 1:5:12', book: 'GEN', chapter: 1, verse: 5, charStart: 12 }, + ]); + }); + + // A USJ verse sid is verbatim, so a bridged verse reaches the catalog as "GEN 1:3-4". + it('resolves a bridged verse sid to its first verse', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [{ id: 'ta-1', surfaceText: 'λόγος' }], + tokenAnalysisLinks: [link('ta-1', 'GEN 1:3-4:0')], + }; + + expect(buildCatalogRows(analysis, scope)[0].usages[0].verse).toBe(3); + }); + + // EXO precedes ACT canonically but follows it alphabetically, so the order cannot come from the + // ref strings. + it('orders usages by document position across books', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [{ id: 'ta-1', surfaceText: 'λόγος' }], + tokenAnalysisLinks: [ + link('ta-1', 'ACT 1:1:0'), + link('ta-1', 'EXO 2:1:0'), + link('ta-1', 'EXO 1:2:5'), + link('ta-1', 'EXO 1:2:0'), + link('ta-1', 'EXO 1:1:0'), + ], + }; + + expect(buildCatalogRows(analysis, scope)[0].usages.map((u) => u.tokenRef)).toEqual([ + 'EXO 1:1:0', + 'EXO 1:2:0', + 'EXO 1:2:5', + 'EXO 2:1:0', + 'ACT 1:1:0', + ]); + }); +}); + +/** An analysis whose morpheme form and gloss share no text with the token's surface form. */ +const analyzedEimi: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { + id: 'ta-1', + surfaceText: 'ἦν', + morphemes: [{ id: 'm-1', form: 'εἰμί', writingSystem: 'grc', gloss: { en: 'to be' } }], + }, + ], +}; + +describe('applyCatalogQuery search', () => { + it('finds a row by a gloss in a language other than the active one', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [{ id: 'ta-1', surfaceText: 'λόγος', gloss: { en: 'word', fr: 'parole' } }], + }; + const rows = buildCatalogRows(analysis, scope); + + expect(applyCatalogQuery(rows, makeQuery({ search: 'parole' }))).toHaveLength(1); + }); + + // The morpheme form is suppletive, so a match cannot come from the surface form instead. + it('finds a row by a morpheme form', () => { + const rows = buildCatalogRows(analyzedEimi, scope); + + expect(applyCatalogQuery(rows, makeQuery({ search: 'ειμι' }))).toHaveLength(1); + }); + + it('finds a row by a morpheme gloss', () => { + const rows = buildCatalogRows(analyzedEimi, scope); + + expect(applyCatalogQuery(rows, makeQuery({ search: 'to be' }))).toHaveLength(1); + }); + + // Both words are present, but only as separate fields. + it('matches the whole query as one substring rather than splitting it into terms', () => { + const rows = buildCatalogRows(analyzedEimi, scope); + + expect(applyCatalogQuery(rows, makeQuery({ search: 'ην to be' }))).toHaveLength(0); + }); +}); + +describe('applyCatalogQuery sort', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { id: 'ta-1', surfaceText: 'a' }, + { id: 'ta-2', surfaceText: 'b' }, + { id: 'ta-3', surfaceText: 'c' }, + ], + tokenAnalysisLinks: [ + link('ta-1', 'GEN 1:1:0'), + link('ta-2', 'GEN 1:2:0'), + link('ta-2', 'GEN 1:3:0'), + link('ta-2', 'GEN 1:4:0'), + link('ta-3', 'GEN 1:5:0'), + link('ta-3', 'GEN 1:6:0'), + ], + }; + + it('orders by usage count, most used first', () => { + const rows = buildCatalogRows(analysis, scope); + + expect(applyCatalogQuery(rows, makeQuery({ sort: 'usageCount' })).map((r) => r.analysisId)) // + .toEqual(['ta-2', 'ta-3', 'ta-1']); + }); + + it('collates surface forms by the source language rather than by code point', () => { + const greek: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { id: 'ta-1', surfaceText: 'βίβλος' }, + { id: 'ta-2', surfaceText: 'ἀρχή' }, + ], + }; + // Code-point order would leave these as given: the accented alpha sits in the Greek Extended + // block, above beta. + expect('βίβλος' < 'ἀρχή').toBe(true); + const rows = buildCatalogRows(greek, scope); + + expect(applyCatalogQuery(rows, makeQuery({ sort: 'surfaceText' })).map((r) => r.analysisId)) // + .toEqual(['ta-2', 'ta-1']); + }); + + it('orders by usage count in the scope book, most used first', () => { + // ta-1 is the more used analysis overall, ta-2 the more used one in GEN. + const acrossBooks: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { id: 'ta-1', surfaceText: 'a' }, + { id: 'ta-2', surfaceText: 'b' }, + ], + tokenAnalysisLinks: [ + link('ta-1', 'GEN 1:1:0'), + link('ta-1', 'JHN 1:1:0'), + link('ta-1', 'JHN 1:2:0'), + link('ta-2', 'GEN 1:2:0'), + link('ta-2', 'GEN 1:3:0'), + ], + }; + const rows = buildCatalogRows(acrossBooks, scope); + + expect( + applyCatalogQuery(rows, makeQuery({ sort: 'usageCountInBook' })).map((r) => r.analysisId), + ).toEqual(['ta-2', 'ta-1']); + }); + + // Swedish collates "ä" after "z"; English collates it near "a". Passing the Swedish collator + // proves the query's collator decides, not a default. + it('collates glosses with the query collator', () => { + const glossed: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { id: 'ta-1', surfaceText: 'a', gloss: { en: 'äpple' } }, + { id: 'ta-2', surfaceText: 'b', gloss: { en: 'zebra' } }, + ], + }; + const rows = buildCatalogRows(glossed, scope); + const sortByGloss = (glossCollator: Intl.Collator) => + applyCatalogQuery(rows, makeQuery({ sort: 'gloss', glossCollator })).map((r) => r.analysisId); + + expect(sortByGloss(new Intl.Collator('sv'))).toEqual(['ta-2', 'ta-1']); + expect(sortByGloss(new Intl.Collator('en'))).toEqual(['ta-1', 'ta-2']); + }); + + // ta-1 is the more used analysis, so a count sort would put it first; ta-2 appears earlier. + it('orders by the first usage in document order', () => { + const spread: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { id: 'ta-1', surfaceText: 'a' }, + { id: 'ta-2', surfaceText: 'b' }, + ], + tokenAnalysisLinks: [ + link('ta-1', 'JHN 1:1:0'), + link('ta-1', 'JHN 1:2:0'), + link('ta-1', 'JHN 1:3:0'), + link('ta-2', 'GEN 1:1:0'), + ], + }; + const rows = buildCatalogRows(spread, scope); + + expect(applyCatalogQuery(rows, makeQuery({ sort: 'firstUsage' })).map((r) => r.analysisId)) // + .toEqual(['ta-2', 'ta-1']); + }); + + it('puts rows with no usages last in first-usage order', () => { + const someUnused: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { id: 'ta-1', surfaceText: 'a' }, + { id: 'ta-2', surfaceText: 'b' }, + { id: 'ta-3', surfaceText: 'c' }, + { id: 'ta-4', surfaceText: 'd' }, + ], + tokenAnalysisLinks: [ + link('ta-2', 'JHN 1:1:0'), + link('ta-2', 'JHN 1:2:0'), + link('ta-2', 'JHN 1:3:0'), + link('ta-3', 'GEN 1:1:0'), + ], + }; + const rows = buildCatalogRows(someUnused, scope); + + expect(applyCatalogQuery(rows, makeQuery({ sort: 'firstUsage' })).map((r) => r.analysisId)) // + .toEqual(['ta-3', 'ta-2', 'ta-1', 'ta-4']); + }); +}); + +/** One analysis with a morpheme breakdown and one without. */ +const mixedBreakdowns: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { + id: 'ta-1', + surfaceText: 'a', + morphemes: [{ id: 'm-1', form: 'a', writingSystem: 'grc' }], + }, + { id: 'ta-2', surfaceText: 'b' }, + ], +}; + +describe('applyCatalogQuery filters', () => { + it('keeps only rows used in one of the selected books', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { id: 'ta-1', surfaceText: 'a' }, + { id: 'ta-2', surfaceText: 'b' }, + { id: 'ta-3', surfaceText: 'c' }, + ], + tokenAnalysisLinks: [ + link('ta-1', 'GEN 1:1:0'), + link('ta-2', 'JHN 1:1:0'), + link('ta-3', 'GEN 1:2:0'), + link('ta-3', 'MRK 1:1:0'), + ], + }; + const rows = buildCatalogRows(analysis, scope); + const query = makeQuery({ filters: { books: ['JHN', 'MRK'] } }); + + // The query's default sort puts the more-used ta-3 first. + expect(applyCatalogQuery(rows, query).map((r) => r.analysisId)).toEqual(['ta-3', 'ta-2']); + }); + + it('keeps only unused rows when filtering for zero usages', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { id: 'ta-1', surfaceText: 'a' }, + { id: 'ta-2', surfaceText: 'b' }, + ], + tokenAnalysisLinks: [link('ta-1', 'GEN 1:1:0')], + }; + const rows = buildCatalogRows(analysis, scope); + const query = makeQuery({ filters: { zeroUsages: true } }); + + expect(applyCatalogQuery(rows, query).map((r) => r.analysisId)).toEqual(['ta-2']); + }); + + // ta-2 is glossed, just not in the scope's language, so an any-language check would drop it. + it('keeps a row glossed only in another language when filtering for a missing gloss', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { id: 'ta-1', surfaceText: 'a', gloss: { en: 'word' } }, + { id: 'ta-2', surfaceText: 'b', gloss: { fr: 'parole' } }, + ], + }; + const rows = buildCatalogRows(analysis, scope); + const query = makeQuery({ filters: { missingGloss: true } }); + + expect(applyCatalogQuery(rows, query).map((r) => r.analysisId)).toEqual(['ta-2']); + }); + + it('keeps only rows with a morpheme breakdown when filtering for one', () => { + const rows = buildCatalogRows(mixedBreakdowns, scope); + const query = makeQuery({ filters: { morphemes: 'has' } }); + + expect(applyCatalogQuery(rows, query).map((r) => r.analysisId)).toEqual(['ta-1']); + }); + + it('keeps only rows without a morpheme breakdown when filtering against one', () => { + const rows = buildCatalogRows(mixedBreakdowns, scope); + const query = makeQuery({ filters: { morphemes: 'lacks' } }); + + expect(applyCatalogQuery(rows, query).map((r) => r.analysisId)).toEqual(['ta-2']); + }); +}); + +describe('deriveFacets', () => { + it('offers a book facet once the rows span more than one book', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { id: 'ta-1', surfaceText: 'a' }, + { id: 'ta-2', surfaceText: 'b' }, + ], + tokenAnalysisLinks: [link('ta-1', 'JHN 1:1:0'), link('ta-2', 'GEN 1:1:0')], + }; + + expect(deriveFacets(buildCatalogRows(analysis, scope)).books).toEqual(['GEN', 'JHN']); + }); + + it('hides the book facet when every row sits in one book', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { id: 'ta-1', surfaceText: 'a' }, + { id: 'ta-2', surfaceText: 'b' }, + ], + tokenAnalysisLinks: [link('ta-1', 'GEN 1:1:0'), link('ta-2', 'GEN 1:2:0')], + }; + + expect(deriveFacets(buildCatalogRows(analysis, scope)).books).toBeUndefined(); + }); + + it('offers a part-of-speech facet once the rows disagree', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { id: 'ta-1', surfaceText: 'a', pos: 'noun' }, + { id: 'ta-2', surfaceText: 'b', pos: 'verb' }, + ], + }; + + expect(deriveFacets(buildCatalogRows(analysis, scope)).pos).toEqual(['noun', 'verb']); + }); + + it('offers a confidence facet once the rows disagree', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { id: 'ta-1', surfaceText: 'a', confidence: 'high' }, + { id: 'ta-2', surfaceText: 'b', confidence: 'guess' }, + ], + }; + + expect(deriveFacets(buildCatalogRows(analysis, scope)).confidence).toEqual(['guess', 'high']); + }); + + // Number varies across the rows; Case does not, so only Number earns a facet. + it('offers a facet per feature name the rows disagree on', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { id: 'ta-1', surfaceText: 'a', features: { Case: 'Nom', Number: 'Sg' } }, + { id: 'ta-2', surfaceText: 'b', features: { Case: 'Nom', Number: 'Pl' } }, + ], + }; + + expect(deriveFacets(buildCatalogRows(analysis, scope)).features).toEqual({ + Number: ['Pl', 'Sg'], + }); + }); + + it('draws a feature facet from the rows that carry the feature at all', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { id: 'ta-1', surfaceText: 'a', features: { Number: 'Sg' } }, + { id: 'ta-2', surfaceText: 'b', features: { Number: 'Pl' } }, + { id: 'ta-3', surfaceText: 'c' }, + ], + }; + + expect(deriveFacets(buildCatalogRows(analysis, scope)).features).toEqual({ + Number: ['Pl', 'Sg'], + }); + }); + + // Nothing in the app writes pos, features, or confidence yet, so an analysis shaped the way the + // gloss and morpheme write paths shape it must not raise a dropdown with nothing behind it. + it('derives no facet from fields no write path populates', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { id: 'ta-1', surfaceText: 'ἀρχῇ', gloss: { en: 'beginning' } }, + { + id: 'ta-2', + surfaceText: 'ἦν', + gloss: { en: 'was' }, + morphemes: [{ id: 'm-1', form: 'εἰμί', writingSystem: 'grc', gloss: { en: 'to be' } }], + }, + ], + tokenAnalysisLinks: [link('ta-1', 'GEN 1:1:0'), link('ta-2', 'JHN 1:1:0')], + }; + + const facets = deriveFacets(buildCatalogRows(analysis, scope)); + expect(facets.pos).toBeUndefined(); + expect(facets.confidence).toBeUndefined(); + expect(facets.features).toBeUndefined(); + expect(facets.books).toEqual(['GEN', 'JHN']); + }); +}); diff --git a/src/__tests__/utils/search-fold.test.ts b/src/__tests__/utils/search-fold.test.ts new file mode 100644 index 00000000..dd5dfdad --- /dev/null +++ b/src/__tests__/utils/search-fold.test.ts @@ -0,0 +1,21 @@ +/// + +import { foldForSearch } from '../../utils/search-fold'; + +describe('foldForSearch', () => { + it('folds Greek accents and the iota subscript away', () => { + expect(foldForSearch('ἀρχῇ')).toBe(foldForSearch('αρχη')); + }); + + it('folds Hebrew points away', () => { + expect(foldForSearch('שָׁלוֹם')).toBe(foldForSearch('שלום')); + }); + + it('strips a Latin diacritic rather than transliterating it', () => { + expect(foldForSearch('šālôm')).toBe('salom'); + }); + + it('lowercases so a query matches a capitalized form', () => { + expect(foldForSearch('Ἀρχή')).toBe(foldForSearch('αρχη')); + }); +}); diff --git a/src/store/analysisSlice.ts b/src/store/analysisSlice.ts index 4fce44e1..79f712a7 100644 --- a/src/store/analysisSlice.ts +++ b/src/store/analysisSlice.ts @@ -12,6 +12,7 @@ import type { } from 'interlinearizer'; import { emptyAnalysis } from '../types/empty-factories'; import { analysesAreIdentical } from '../utils/analysis-identity'; +import { buildCatalogRows } from '../utils/analysis-query'; import { isEmptyMultiString } from '../utils/multi-string'; import { buildPoolIndex, @@ -1034,6 +1035,21 @@ export const selectPoolIndex = createSelector( buildPoolIndex, ); +/** + * Memoized selector building the Analysis Catalog's rows — one per distinct token analysis, with + * its usage counts and locations — against the book named as the second argument. Recomputes only + * when `tokenAnalyses`, `tokenAnalysisLinks`, the analysis language, or that book change reference, + * so searching and sorting the result never rebuilds the rows. + */ +export const selectCatalogRows = createSelector( + selectTokenAnalyses, + selectTokenAnalysisLinks, + selectAnalysisLanguage, + (_state: AnalysisState, currentBook: string) => currentBook, + (tokenAnalyses, tokenAnalysisLinks, analysisLanguage, currentBook) => + buildCatalogRows({ tokenAnalyses, tokenAnalysisLinks }, { analysisLanguage, currentBook }), +); + /** * Returns the merged analysis the renderer shows for a token: its approved decision when one * exists, otherwise the engine's suggestion derived live from the approved-analysis pool, or diff --git a/src/utils/analysis-query.ts b/src/utils/analysis-query.ts new file mode 100644 index 00000000..525cf9bb --- /dev/null +++ b/src/utils/analysis-query.ts @@ -0,0 +1,289 @@ +import { Canon } from '@sillsdev/scripture'; +import type { + Confidence, + MorphemeAnalysis, + TextAnalysis, + TokenAnalysis, + TokenAnalysisLink, +} from 'interlinearizer'; +import { bookOfRef } from './analysis-book'; +import { foldForSearch } from './search-fold'; +import { firstVerseNumber } from './verse-ref'; + +/** The values every row is derived relative to. */ +export interface CatalogScope { + /** BCP 47 tag whose gloss each row displays. */ + analysisLanguage: string; + /** Book code the per-book usage count is taken against. */ + currentBook: string; +} + +/** One distinct token analysis, with the usage data the catalog lists it by. */ +export interface CatalogRow { + analysisId: string; + surfaceText: string; + /** Gloss in the scope's analysis language; `''` when the analysis has none. */ + gloss: string; + morphemes: readonly MorphemeAnalysis[]; + pos?: string; + features?: Record; + confidence?: Confidence; + /** Places in the whole draft where the analysis is applied. */ + usageCount: number; + /** Places in the scope's book where the analysis is applied. */ + usageCountInBook: number; + /** Every place the analysis is applied, in document order. */ + usages: readonly CatalogUsage[]; + /** Books the analysis is used in. */ + books: ReadonlySet; + /** Everything a search query is matched against, folded. */ + searchText: string; +} + +/** How the rows are ordered. */ +export type CatalogSort = + 'usageCount' | 'usageCountInBook' | 'surfaceText' | 'gloss' | 'firstUsage'; + +/** Which rows are kept. An absent filter is inactive. */ +export interface CatalogFilters { + /** Keeps rows used in at least one of these books. */ + books?: readonly string[]; + /** Keeps only rows nothing uses. */ + zeroUsages?: boolean; + /** Keeps only rows with no gloss in the scope's analysis language. */ + missingGloss?: boolean; + /** Keeps rows according to whether they carry a morpheme breakdown. */ + morphemes?: 'has' | 'lacks'; +} + +/** How the caller narrows and orders the rows. */ +export interface CatalogQuery { + /** Matched as a plain substring against each row's folded text; `''` matches everything. */ + search: string; + sort: CatalogSort; + filters: CatalogFilters; + /** Collates surface forms, so ordering follows the source language rather than code points. */ + surfaceCollator: Intl.Collator; + /** Collates glosses, so ordering follows the analysis language rather than code points. */ + glossCollator: Intl.Collator; +} + +/** + * One place in the text where an analysis is applied, read off the token ref alone — the catalog + * never resolves the token itself. + */ +export interface CatalogUsage { + tokenRef: string; + book: string; + chapter: number; + verse: number; + /** Zero-based UTF-16 offset of the token within its segment's baseline text. */ + charStart: number; +} + +/** + * Reads the location a token ref names. A ref is a verse SID plus the token's character offset + * (`"GEN 1:1:0"`), so the whole location is recoverable from the string. The SID's verse portion is + * verbatim USJ, so a bridged verse resolves to the first verse it names. + */ +function parseUsage(tokenRef: string): CatalogUsage { + const [chapterPart, versePart, charPart] = tokenRef.slice(tokenRef.indexOf(' ') + 1).split(':'); + return { + tokenRef, + book: bookOfRef(tokenRef), + chapter: Number(chapterPart), + /* v8 ignore next -- a sid whose verse portion starts with no digit cannot reach a token ref */ + verse: firstVerseNumber(versePart) ?? 0, + charStart: Number(charPart), + }; +} + +/** Orders two usages by document position, taking books in canonical rather than alphabetical order. */ +function compareDocumentOrder(a: CatalogUsage, b: CatalogUsage): number { + return ( + Canon.bookIdToNumber(a.book) - Canon.bookIdToNumber(b.book) || + a.chapter - b.chapter || + a.verse - b.verse || + a.charStart - b.charStart + ); +} + +/** + * Files each analysis's usages under its id, each list in document order. + * + * Only an approved link is a usage: a rejected link is by definition not a place the analysis is + * applied, and counting approvals alone keeps a row's count equal to the analysis's frequency in + * the suggestion pool, so the two can never disagree. + */ +function groupUsagesByAnalysisId( + links: readonly TokenAnalysisLink[], +): ReadonlyMap { + const byId = links.reduce((acc, l) => { + if (l.status !== 'approved') return acc; + const usages = acc.get(l.analysisId); + const usage = parseUsage(l.token.tokenRef); + if (usages) usages.push(usage); + else acc.set(l.analysisId, [usage]); + return acc; + }, new Map()); + byId.forEach((usages) => usages.sort(compareDocumentOrder)); + return byId; +} + +/** + * Assembles the folded text a search matches an analysis by, drawing on its forms and on every + * gloss it carries whatever the language, so a gloss stays findable in a language the project does + * not declare. No match spans two of the assembled fields. + */ +function buildSearchText(ta: TokenAnalysis): string { + const morphemeText = (ta.morphemes ?? []).flatMap((m) => [ + m.form, + ...Object.values(m.gloss ?? {}), + ]); + return foldForSearch( + [ta.surfaceText, ...Object.values(ta.gloss ?? {}), ...morphemeText].join('\n'), + ); +} + +/** + * Derives one row per distinct token analysis. Pure in the analysis records: nothing here reads the + * tokenized text, so the catalog reports what was recorded rather than whether the text a usage + * points at still says the same thing. + */ +export function buildCatalogRows( + analysis: Pick, + scope: CatalogScope, +): readonly CatalogRow[] { + const usagesByAnalysisId = groupUsagesByAnalysisId(analysis.tokenAnalysisLinks); + + return analysis.tokenAnalyses.map((ta) => { + const usages = usagesByAnalysisId.get(ta.id) ?? []; + return { + analysisId: ta.id, + surfaceText: ta.surfaceText, + gloss: ta.gloss?.[scope.analysisLanguage] ?? '', + morphemes: ta.morphemes ?? [], + pos: ta.pos, + features: ta.features, + confidence: ta.confidence, + usageCount: usages.length, + usageCountInBook: usages.filter((u) => u.book === scope.currentBook).length, + usages, + books: new Set(usages.map((u) => u.book)), + searchText: buildSearchText(ta), + }; + }); +} + +/** Orders two rows by where each is first applied, an unused analysis coming after every used one. */ +function compareFirstUsage(a: CatalogRow, b: CatalogRow): number { + const [firstA] = a.usages; + const [firstB] = b.usages; + if (!firstA) return firstB ? 1 : 0; + if (!firstB) return -1; + return compareDocumentOrder(firstA, firstB); +} + +/** + * The closed-vocabulary values present in the rows, each offered as a filter. + * + * A facet is absent unless the rows disagree on it — a dropdown with one choice filters nothing. + * Assignment status is not a facet: only approved links count as usages, so every row agrees by + * construction. + */ +export interface CatalogFacets { + /** Canonical order. */ + books?: readonly string[]; + pos?: readonly string[]; + confidence?: readonly Confidence[]; + /** Distinct values per feature name, each name judged on its own. */ + features?: Readonly>; +} + +/** + * Collects the distinct values the rows carry for one field, sorted, or `undefined` when fewer than + * two are present. + */ +function facetValues( + rows: readonly CatalogRow[], + valuesOf: (row: CatalogRow) => Iterable, + compare?: (a: T, b: T) => number, +): readonly T[] | undefined { + const distinct = new Set(rows.flatMap((row) => [...valuesOf(row)])); + return distinct.size < 2 ? undefined : [...distinct].sort(compare); +} + +/** Collects a facet per feature name that earns one, or `undefined` when no name does. */ +function featureFacets(rows: readonly CatalogRow[]): CatalogFacets['features'] { + const names = new Set(rows.flatMap((row) => Object.keys(row.features ?? {}))); + const facets = [...names].reduce>((acc, name) => { + const values = facetValues(rows, (row) => { + const value = row.features?.[name]; + return value === undefined ? [] : [value]; + }); + if (values) acc[name] = values; + return acc; + }, {}); + return Object.keys(facets).length === 0 ? undefined : facets; +} + +/** Derives the filter facets worth offering against these rows. */ +export function deriveFacets(rows: readonly CatalogRow[]): CatalogFacets { + return { + books: facetValues( + rows, + (row) => row.books, + (a, b) => Canon.bookIdToNumber(a) - Canon.bookIdToNumber(b), + ), + pos: facetValues(rows, (row) => (row.pos === undefined ? [] : [row.pos])), + confidence: facetValues(rows, (row) => (row.confidence === undefined ? [] : [row.confidence])), + features: featureFacets(rows), + }; +} + +/** Whether the row survives every active filter. */ +function passesFilters(row: CatalogRow, filters: CatalogFilters): boolean { + if (filters.books && !filters.books.some((book) => row.books.has(book))) return false; + if (filters.zeroUsages && row.usageCount > 0) return false; + if (filters.missingGloss && row.gloss !== '') return false; + if (filters.morphemes) { + const hasMorphemes = row.morphemes.length > 0; + if (hasMorphemes !== (filters.morphemes === 'has')) return false; + } + return true; +} + +/** + * Orders two rows by the query's sort key: counts descending, so the most used lead; text + * ascending. + */ +function compareBySort(a: CatalogRow, b: CatalogRow, query: CatalogQuery): number { + switch (query.sort) { + case 'usageCountInBook': + return b.usageCountInBook - a.usageCountInBook; + case 'surfaceText': + return query.surfaceCollator.compare(a.surfaceText, b.surfaceText); + case 'gloss': + return query.glossCollator.compare(a.gloss, b.gloss); + case 'firstUsage': + return compareFirstUsage(a, b); + default: + return b.usageCount - a.usageCount; + } +} + +/** + * Narrows rows to those the query's search and filters keep, in the order it asks for. + * + * Separate from {@link buildCatalogRows} so a keystroke re-runs only this pass: the analysis changes + * on every gloss blur, and rebuilding rows per keystroke would re-fold unchanged text. + */ +export function applyCatalogQuery( + rows: readonly CatalogRow[], + query: CatalogQuery, +): readonly CatalogRow[] { + const search = foldForSearch(query.search); + return rows + .filter((row) => row.searchText.includes(search) && passesFilters(row, query.filters)) + .sort((a, b) => compareBySort(a, b, query)); +} diff --git a/src/utils/search-fold.ts b/src/utils/search-fold.ts new file mode 100644 index 00000000..a4a37207 --- /dev/null +++ b/src/utils/search-fold.ts @@ -0,0 +1,19 @@ +/** + * @file The search fold, deliberately not merged with `normalizeSurfaceForm` in + * `analysis-identity`. The two look alike enough to invite it, and merging them would corrupt + * data rather than fail loudly: folding a diacritic into identity puts genuinely different words + * on one shared payload, silently dropping a distinction the user recorded. + */ + +/** + * Folds text down to the form a query is matched against: equal ignoring case and diacritics, so a + * query typed on an ordinary keyboard finds a fully pointed or accented form. Never use it to + * decide whether two analyses are the same. + */ +export function foldForSearch(text: string): string { + return text + .normalize('NFD') + .replace(/\p{Mn}/gu, '') + .normalize('NFC') + .toLowerCase(); +} From 7bf88db8416714b7e87afab23897cd70bea506cb Mon Sep 17 00:00:00 2001 From: Alex Rawlings Date: Mon, 10 Aug 2026 16:27:16 -0600 Subject: [PATCH 02/18] Let the catalog filter on every facet it derives Adds pos, confidence, and feature filters, treats an empty selection as inactive, and orders the confidence facet by strength. The search query is trimmed in applyCatalogQuery, not in foldForSearch, whose output separates the row fields a match must not span. --- src/__tests__/utils/analysis-query.test.ts | 74 +++++++++++++++++++++- src/__tests__/utils/search-fold.test.ts | 4 ++ src/utils/analysis-query.ts | 55 ++++++++++++++-- src/utils/search-fold.ts | 3 + 4 files changed, 130 insertions(+), 6 deletions(-) diff --git a/src/__tests__/utils/analysis-query.test.ts b/src/__tests__/utils/analysis-query.test.ts index 52c48663..b0220c3d 100644 --- a/src/__tests__/utils/analysis-query.test.ts +++ b/src/__tests__/utils/analysis-query.test.ts @@ -185,6 +185,12 @@ describe('applyCatalogQuery search', () => { expect(applyCatalogQuery(rows, makeQuery({ search: 'ην to be' }))).toHaveLength(0); }); + + it('ignores whitespace around the query, which a typed or pasted search carries', () => { + const rows = buildCatalogRows(analyzedEimi, scope); + + expect(applyCatalogQuery(rows, makeQuery({ search: ' to be ' }))).toHaveLength(1); + }); }); describe('applyCatalogQuery sort', () => { @@ -327,6 +333,22 @@ const mixedBreakdowns: TextAnalysis = { ], }; +/** Two analyses tagged across every closed vocabulary, and one tagged with none of them. */ +const tagged: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { + id: 'ta-1', + surfaceText: 'a', + pos: 'noun', + confidence: 'high', + features: { Number: 'Sg', Case: 'Nom' }, + }, + { id: 'ta-2', surfaceText: 'b', pos: 'verb', confidence: 'guess', features: { Number: 'Pl' } }, + { id: 'ta-3', surfaceText: 'c' }, + ], +}; + describe('applyCatalogQuery filters', () => { it('keeps only rows used in one of the selected books', () => { const analysis: TextAnalysis = { @@ -393,6 +415,56 @@ describe('applyCatalogQuery filters', () => { expect(applyCatalogQuery(rows, query).map((r) => r.analysisId)).toEqual(['ta-2']); }); + + it('keeps only rows carrying one of the selected parts of speech', () => { + const rows = buildCatalogRows(tagged, scope); + const query = makeQuery({ filters: { pos: ['noun'] } }); + + expect(applyCatalogQuery(rows, query).map((r) => r.analysisId)).toEqual(['ta-1']); + }); + + it('drops a row carrying no part of speech at all when filtering on one', () => { + const rows = buildCatalogRows(tagged, scope); + const query = makeQuery({ filters: { pos: ['noun', 'verb'] } }); + + expect(applyCatalogQuery(rows, query).map((r) => r.analysisId)).toEqual(['ta-1', 'ta-2']); + }); + + it('keeps only rows carrying one of the selected confidence levels', () => { + const rows = buildCatalogRows(tagged, scope); + const query = makeQuery({ filters: { confidence: ['guess'] } }); + + expect(applyCatalogQuery(rows, query).map((r) => r.analysisId)).toEqual(['ta-2']); + }); + + it('keeps only rows matching the selected value of a feature', () => { + const rows = buildCatalogRows(tagged, scope); + const query = makeQuery({ filters: { features: { Number: ['Sg'] } } }); + + expect(applyCatalogQuery(rows, query).map((r) => r.analysisId)).toEqual(['ta-1']); + }); + + // ta-1 is the only Sg row, and its Case is Nom rather than Acc, so nothing satisfies both. + it('requires a row to match every feature named in the selection', () => { + const rows = buildCatalogRows(tagged, scope); + const query = makeQuery({ filters: { features: { Number: ['Sg'], Case: ['Acc'] } } }); + + expect(applyCatalogQuery(rows, query)).toHaveLength(0); + }); + + it('treats an empty book selection as no filter rather than as an impossible one', () => { + const rows = buildCatalogRows(tagged, scope); + const query = makeQuery({ filters: { books: [] } }); + + expect(applyCatalogQuery(rows, query)).toHaveLength(rows.length); + }); + + it('treats an empty chosen-value selection as no filter', () => { + const rows = buildCatalogRows(tagged, scope); + const query = makeQuery({ filters: { pos: [], confidence: [], features: { Number: [] } } }); + + expect(applyCatalogQuery(rows, query)).toHaveLength(rows.length); + }); }); describe('deriveFacets', () => { @@ -443,7 +515,7 @@ describe('deriveFacets', () => { ], }; - expect(deriveFacets(buildCatalogRows(analysis, scope)).confidence).toEqual(['guess', 'high']); + expect(deriveFacets(buildCatalogRows(analysis, scope)).confidence).toEqual(['high', 'guess']); }); // Number varies across the rows; Case does not, so only Number earns a facet. diff --git a/src/__tests__/utils/search-fold.test.ts b/src/__tests__/utils/search-fold.test.ts index dd5dfdad..123b6d39 100644 --- a/src/__tests__/utils/search-fold.test.ts +++ b/src/__tests__/utils/search-fold.test.ts @@ -18,4 +18,8 @@ describe('foldForSearch', () => { it('lowercases so a query matches a capitalized form', () => { expect(foldForSearch('Ἀρχή')).toBe(foldForSearch('αρχη')); }); + + it('keeps a spacing combining mark, which spells a vowel rather than decorating one', () => { + expect(foldForSearch('कि')).not.toBe(foldForSearch('क')); + }); }); diff --git a/src/utils/analysis-query.ts b/src/utils/analysis-query.ts index 525cf9bb..6e4a0f7b 100644 --- a/src/utils/analysis-query.ts +++ b/src/utils/analysis-query.ts @@ -44,10 +44,16 @@ export interface CatalogRow { export type CatalogSort = 'usageCount' | 'usageCountInBook' | 'surfaceText' | 'gloss' | 'firstUsage'; -/** Which rows are kept. An absent filter is inactive. */ +/** Which rows are kept. An absent filter is inactive, as is an empty selection. */ export interface CatalogFilters { /** Keeps rows used in at least one of these books. */ books?: readonly string[]; + /** Keeps rows carrying one of these parts of speech. */ + pos?: readonly string[]; + /** Keeps rows carrying one of these confidence levels. */ + confidence?: readonly Confidence[]; + /** Keeps rows matching every named feature, each against its own accepted values. */ + features?: Readonly>; /** Keeps only rows nothing uses. */ zeroUsages?: boolean; /** Keeps only rows with no gloss in the scope's analysis language. */ @@ -58,7 +64,10 @@ export interface CatalogFilters { /** How the caller narrows and orders the rows. */ export interface CatalogQuery { - /** Matched as a plain substring against each row's folded text; `''` matches everything. */ + /** + * Matched as a plain substring against each row's folded text, ignoring surrounding whitespace; + * `''` matches everything. + */ search: string; sort: CatalogSort; filters: CatalogFilters; @@ -227,6 +236,9 @@ function featureFacets(rows: readonly CatalogRow[]): CatalogFacets['features'] { return Object.keys(facets).length === 0 ? undefined : facets; } +/** Confidence levels from strongest to weakest. */ +const CONFIDENCE_ORDER: readonly Confidence[] = ['high', 'medium', 'low', 'guess']; + /** Derives the filter facets worth offering against these rows. */ export function deriveFacets(rows: readonly CatalogRow[]): CatalogFacets { return { @@ -236,14 +248,45 @@ export function deriveFacets(rows: readonly CatalogRow[]): CatalogFacets { (a, b) => Canon.bookIdToNumber(a) - Canon.bookIdToNumber(b), ), pos: facetValues(rows, (row) => (row.pos === undefined ? [] : [row.pos])), - confidence: facetValues(rows, (row) => (row.confidence === undefined ? [] : [row.confidence])), + confidence: facetValues( + rows, + (row) => (row.confidence === undefined ? [] : [row.confidence]), + (a, b) => CONFIDENCE_ORDER.indexOf(a) - CONFIDENCE_ORDER.indexOf(b), + ), features: featureFacets(rows), }; } +/** + * Whether a chosen-value selection narrows anything. An empty selection is inactive rather than + * unsatisfiable, so clearing every choice restores the full list. + */ +function isActive(selected: readonly T[] | undefined): selected is readonly T[] { + return selected !== undefined && selected.length > 0; +} + +/** + * Whether a field holding one value at a time carries one its selection accepts. A field carrying + * no value at all fails an active selection. + */ +function passesValue(selected: readonly T[] | undefined, value: T | undefined): boolean { + return !isActive(selected) || (value !== undefined && selected.includes(value)); +} + +/** Whether the row satisfies every feature named in the selection, each judged on its own values. */ +function passesFeatures(row: CatalogRow, selected: CatalogFilters['features']): boolean { + if (!selected) return true; + return Object.entries(selected).every(([name, values]) => + passesValue(values, row.features?.[name]), + ); +} + /** Whether the row survives every active filter. */ function passesFilters(row: CatalogRow, filters: CatalogFilters): boolean { - if (filters.books && !filters.books.some((book) => row.books.has(book))) return false; + if (isActive(filters.books) && !filters.books.some((book) => row.books.has(book))) return false; + if (!passesValue(filters.pos, row.pos)) return false; + if (!passesValue(filters.confidence, row.confidence)) return false; + if (!passesFeatures(row, filters.features)) return false; if (filters.zeroUsages && row.usageCount > 0) return false; if (filters.missingGloss && row.gloss !== '') return false; if (filters.morphemes) { @@ -282,7 +325,9 @@ export function applyCatalogQuery( rows: readonly CatalogRow[], query: CatalogQuery, ): readonly CatalogRow[] { - const search = foldForSearch(query.search); + // Trimmed here rather than inside foldForSearch, which also folds each row's search text, where + // whitespace separates the fields a match must not span. + const search = foldForSearch(query.search.trim()); return rows .filter((row) => row.searchText.includes(search) && passesFilters(row, query.filters)) .sort((a, b) => compareBySort(a, b, query)); diff --git a/src/utils/search-fold.ts b/src/utils/search-fold.ts index a4a37207..12ee1a6c 100644 --- a/src/utils/search-fold.ts +++ b/src/utils/search-fold.ts @@ -9,6 +9,9 @@ * Folds text down to the form a query is matched against: equal ignoring case and diacritics, so a * query typed on an ordinary keyboard finds a fully pointed or accented form. Never use it to * decide whether two analyses are the same. + * + * Only nonspacing marks fold away: a spacing combining mark spells a dependent vowel, so dropping + * it would merge words that differ by their vowel. */ export function foldForSearch(text: string): string { return text From f15a96b3777f95a4a8b6ead36076fc40df93a085 Mon Sep 17 00:00:00 2001 From: Alex Rawlings Date: Mon, 10 Aug 2026 16:58:01 -0600 Subject: [PATCH 03/18] Let a search match a letter in any positional shape Compatibility normalization covers the shapes Unicode encodes as variants; Greek and Hebrew final letters, which it encodes as distinct letters, are mapped. --- src/__tests__/utils/analysis-query.test.ts | 11 +++++++ src/__tests__/utils/search-fold.test.ts | 30 +++++++++++++++++ src/utils/search-fold.ts | 38 ++++++++++++++++++---- 3 files changed, 72 insertions(+), 7 deletions(-) diff --git a/src/__tests__/utils/analysis-query.test.ts b/src/__tests__/utils/analysis-query.test.ts index b0220c3d..b022d620 100644 --- a/src/__tests__/utils/analysis-query.test.ts +++ b/src/__tests__/utils/analysis-query.test.ts @@ -191,6 +191,17 @@ describe('applyCatalogQuery search', () => { expect(applyCatalogQuery(rows, makeQuery({ search: ' to be ' }))).toHaveLength(1); }); + + // Both sides fold, so the word-final sigma the text carries meets the ordinary one typed. + it('finds a row whose surface form ends in a letter the query spells mid-word', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [{ id: 'ta-1', surfaceText: 'λόγος', gloss: { en: 'word' } }], + }; + const rows = buildCatalogRows(analysis, scope); + + expect(applyCatalogQuery(rows, makeQuery({ search: 'λογοσ' }))).toHaveLength(1); + }); }); describe('applyCatalogQuery sort', () => { diff --git a/src/__tests__/utils/search-fold.test.ts b/src/__tests__/utils/search-fold.test.ts index 123b6d39..089a11eb 100644 --- a/src/__tests__/utils/search-fold.test.ts +++ b/src/__tests__/utils/search-fold.test.ts @@ -22,4 +22,34 @@ describe('foldForSearch', () => { it('keeps a spacing combining mark, which spells a vowel rather than decorating one', () => { expect(foldForSearch('कि')).not.toBe(foldForSearch('क')); }); + + it('folds a Greek final sigma to the shape the letter takes mid-word', () => { + expect(foldForSearch('λόγος')).toBe(foldForSearch('λογοσ')); + }); + + it('folds a Hebrew final form to the shape the letter takes mid-word', () => { + expect(foldForSearch('שָׁלוֹם')).toBe(foldForSearch('שלומ')); + }); + + it('folds a final sigma reached by lowercasing a capitalized form', () => { + expect(foldForSearch('ΛΌΓΟΣ')).toBe(foldForSearch('λογοσ')); + }); + + // Arabic shapes its letters as it renders, so plain text needs no folding; text carrying the + // presentation forms an older encoding spelled out does. + it('folds Arabic presentation forms to the letters typed for them', () => { + expect(foldForSearch('ﺑﻴﺖ')).toBe(foldForSearch('بيت')); + }); + + it('folds a lunate sigma, which stands for the letter at either position', () => { + expect(foldForSearch('ϲοφία')).toBe(foldForSearch('σοφια')); + }); + + it('folds a long s, the word-medial shape an older Latin typography wrote', () => { + expect(foldForSearch('ſanctus')).toBe(foldForSearch('sanctus')); + }); + + it('keeps a stroked letter, which its script counts as a letter of its own', () => { + expect(foldForSearch('łuk')).not.toBe(foldForSearch('luk')); + }); }); diff --git a/src/utils/search-fold.ts b/src/utils/search-fold.ts index 12ee1a6c..e4611007 100644 --- a/src/utils/search-fold.ts +++ b/src/utils/search-fold.ts @@ -6,17 +6,41 @@ */ /** - * Folds text down to the form a query is matched against: equal ignoring case and diacritics, so a - * query typed on an ordinary keyboard finds a fully pointed or accented form. Never use it to - * decide whether two analyses are the same. + * Letters written with one shape at the end of a word and another shape elsewhere, mapped to the + * shape outside that position. These are the letters no normalization unifies, their scripts + * encoding the two shapes as distinct letters rather than as variants of one. + */ +const FINAL_LETTER_FORMS: Readonly> = { + ς: 'σ', + ך: 'כ', + ם: 'מ', + ן: 'נ', + ף: 'פ', + ץ: 'צ', +}; + +/** Matches the letters {@link FINAL_LETTER_FORMS} folds. */ +const FINAL_LETTER_PATTERN = new RegExp(`[${Object.keys(FINAL_LETTER_FORMS).join('')}]`, 'gu'); + +/** + * Folds text down to the form a query is matched against: equal ignoring case, diacritics, and the + * shape a letter takes at its position in a word, so a query typed on an ordinary keyboard finds a + * fully pointed or accented form wherever in a word it sits. Never use it to decide whether two + * analyses are the same. + * + * A positional shape matches the plain letter typed for it whatever script writes it, as do + * ligature and width variants; only {@link FINAL_LETTER_FORMS} needs listing. * - * Only nonspacing marks fold away: a spacing combining mark spells a dependent vowel, so dropping - * it would merge words that differ by their vowel. + * Some distinctions deliberately survive the fold. A spacing combining mark spells a dependent + * vowel, so dropping it would merge words differing only in that vowel; nonspacing marks alone go. + * And a letter written with a stroke or bar keeps it, matching how the scripts that write one count + * it a letter of its own rather than a decorated one. */ export function foldForSearch(text: string): string { return text - .normalize('NFD') + .normalize('NFKD') .replace(/\p{Mn}/gu, '') .normalize('NFC') - .toLowerCase(); + .toLowerCase() + .replace(FINAL_LETTER_PATTERN, (letter) => FINAL_LETTER_FORMS[letter]); } From 0d0f50119172009e4b3abcd51e2a9158a480ff25 Mon Sep 17 00:00:00 2001 From: Alex Rawlings Date: Tue, 11 Aug 2026 10:22:14 -0600 Subject: [PATCH 04/18] Document the catalog row cache by what callers can rely on --- src/store/analysisSlice.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/store/analysisSlice.ts b/src/store/analysisSlice.ts index 79f712a7..83487b40 100644 --- a/src/store/analysisSlice.ts +++ b/src/store/analysisSlice.ts @@ -1037,9 +1037,9 @@ export const selectPoolIndex = createSelector( /** * Memoized selector building the Analysis Catalog's rows — one per distinct token analysis, with - * its usage counts and locations — against the book named as the second argument. Recomputes only - * when `tokenAnalyses`, `tokenAnalysisLinks`, the analysis language, or that book change reference, - * so searching and sorting the result never rebuilds the rows. + * its usage counts and locations — against the book named as the second argument. Only a change to + * the analysis it reads or to the named book rebuilds the rows, so searching and sorting the result + * never does, and two components asking for different books keep each other's rows cached. */ export const selectCatalogRows = createSelector( selectTokenAnalyses, From c1d4d7972f5d6ffdcadec932891a82ccea479307 Mon Sep 17 00:00:00 2001 From: Alex Rawlings Date: Tue, 11 Aug 2026 10:53:18 -0600 Subject: [PATCH 05/18] Keep a search from matching across two catalog fields --- src/__tests__/utils/analysis-query.test.ts | 11 +++++++++++ src/utils/analysis-query.ts | 7 ++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/__tests__/utils/analysis-query.test.ts b/src/__tests__/utils/analysis-query.test.ts index b022d620..caaf01a6 100644 --- a/src/__tests__/utils/analysis-query.test.ts +++ b/src/__tests__/utils/analysis-query.test.ts @@ -186,6 +186,17 @@ describe('applyCatalogQuery search', () => { expect(applyCatalogQuery(rows, makeQuery({ search: 'ην to be' }))).toHaveLength(0); }); + // The two fields are adjacent in the folded text, so only the newline between them is in the way. + it('matches no row on a query carrying the newline that separates two fields', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [{ id: 'ta-1', surfaceText: 'λόγος', gloss: { en: 'word' } }], + }; + const rows = buildCatalogRows(analysis, scope); + + expect(applyCatalogQuery(rows, makeQuery({ search: 'λογος\nword' }))).toHaveLength(0); + }); + it('ignores whitespace around the query, which a typed or pasted search carries', () => { const rows = buildCatalogRows(analyzedEimi, scope); diff --git a/src/utils/analysis-query.ts b/src/utils/analysis-query.ts index 6e4a0f7b..0c8d7329 100644 --- a/src/utils/analysis-query.ts +++ b/src/utils/analysis-query.ts @@ -325,9 +325,10 @@ export function applyCatalogQuery( rows: readonly CatalogRow[], query: CatalogQuery, ): readonly CatalogRow[] { - // Trimmed here rather than inside foldForSearch, which also folds each row's search text, where - // whitespace separates the fields a match must not span. - const search = foldForSearch(query.search.trim()); + // A newline separates the fields a match must not span, so one typed into the query reads as an + // ordinary space instead. That and the trim happen here rather than inside foldForSearch, which + // also folds each row's search text, where the separators must survive. + const search = foldForSearch(query.search.replace(/\n/g, ' ').trim()); return rows .filter((row) => row.searchText.includes(search) && passesFilters(row, query.filters)) .sort((a, b) => compareBySort(a, b, query)); From 3e5b915c982f0833d9781939715d6d5d66f5dae8 Mon Sep 17 00:00:00 2001 From: Alex Rawlings Date: Tue, 11 Aug 2026 11:24:03 -0600 Subject: [PATCH 06/18] Hold the catalog listing still where a sort key ties Ties fell through to payload creation order, so a draft of mostly one-usage rows was ordered by when each analysis happened to be recorded; they now fall back to form, then gloss. --- src/__tests__/utils/analysis-query.test.ts | 30 ++++++++++++++++++++++ src/utils/analysis-query.ts | 18 ++++++++++++- src/utils/search-fold.ts | 16 +++++++----- 3 files changed, 57 insertions(+), 7 deletions(-) diff --git a/src/__tests__/utils/analysis-query.test.ts b/src/__tests__/utils/analysis-query.test.ts index caaf01a6..cb53808c 100644 --- a/src/__tests__/utils/analysis-query.test.ts +++ b/src/__tests__/utils/analysis-query.test.ts @@ -340,6 +340,36 @@ describe('applyCatalogQuery sort', () => { expect(applyCatalogQuery(rows, makeQuery({ sort: 'firstUsage' })).map((r) => r.analysisId)) // .toEqual(['ta-3', 'ta-2', 'ta-1', 'ta-4']); }); + + // Every row here is unused, so the count sort separates none of them. + it('orders rows the sort key cannot separate by surface form', () => { + const tied: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { id: 'ta-1', surfaceText: 'γ' }, + { id: 'ta-2', surfaceText: 'β' }, + { id: 'ta-3', surfaceText: 'α' }, + ], + }; + const rows = buildCatalogRows(tied, scope); + + expect(applyCatalogQuery(rows, makeQuery({ sort: 'usageCount' })).map((r) => r.analysisId)) // + .toEqual(['ta-3', 'ta-2', 'ta-1']); + }); + + it('orders two rows sharing a surface form by gloss', () => { + const homographs: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { id: 'ta-1', surfaceText: 'α', gloss: { en: 'second' } }, + { id: 'ta-2', surfaceText: 'α', gloss: { en: 'first' } }, + ], + }; + const rows = buildCatalogRows(homographs, scope); + + expect(applyCatalogQuery(rows, makeQuery({ sort: 'usageCount' })).map((r) => r.analysisId)) // + .toEqual(['ta-2', 'ta-1']); + }); }); /** One analysis with a morpheme breakdown and one without. */ diff --git a/src/utils/analysis-query.ts b/src/utils/analysis-query.ts index 0c8d7329..101f6462 100644 --- a/src/utils/analysis-query.ts +++ b/src/utils/analysis-query.ts @@ -300,7 +300,7 @@ function passesFilters(row: CatalogRow, filters: CatalogFilters): boolean { * Orders two rows by the query's sort key: counts descending, so the most used lead; text * ascending. */ -function compareBySort(a: CatalogRow, b: CatalogRow, query: CatalogQuery): number { +function compareBySortKey(a: CatalogRow, b: CatalogRow, query: CatalogQuery): number { switch (query.sort) { case 'usageCountInBook': return b.usageCountInBook - a.usageCountInBook; @@ -315,6 +315,22 @@ function compareBySort(a: CatalogRow, b: CatalogRow, query: CatalogQuery): numbe } } +/** Orders two rows by their form and then their gloss, both ascending. */ +function compareByText(a: CatalogRow, b: CatalogRow, query: CatalogQuery): number { + return ( + query.surfaceCollator.compare(a.surfaceText, b.surfaceText) || + query.glossCollator.compare(a.gloss, b.gloss) + ); +} + +/** + * Orders two rows for the listing, falling back to their text where the sort key cannot separate + * them, so the order holds still as unrelated analyses are recorded. + */ +function compareBySort(a: CatalogRow, b: CatalogRow, query: CatalogQuery): number { + return compareBySortKey(a, b, query) || compareByText(a, b, query); +} + /** * Narrows rows to those the query's search and filters keep, in the order it asks for. * diff --git a/src/utils/search-fold.ts b/src/utils/search-fold.ts index e4611007..8d2759cc 100644 --- a/src/utils/search-fold.ts +++ b/src/utils/search-fold.ts @@ -37,10 +37,14 @@ const FINAL_LETTER_PATTERN = new RegExp(`[${Object.keys(FINAL_LETTER_FORMS).join * it a letter of its own rather than a decorated one. */ export function foldForSearch(text: string): string { - return text - .normalize('NFKD') - .replace(/\p{Mn}/gu, '') - .normalize('NFC') - .toLowerCase() - .replace(FINAL_LETTER_PATTERN, (letter) => FINAL_LETTER_FORMS[letter]); + return ( + text + .normalize('NFKD') + .replace(/\p{Mn}/gu, '') + .normalize('NFC') + // Lowercasing a capitalized word is itself what puts its last letter into a final shape, so + // the fold of those shapes has to follow it. + .toLowerCase() + .replace(FINAL_LETTER_PATTERN, (letter) => FINAL_LETTER_FORMS[letter]) + ); } From de1730bafde2f7e4c6ec9901d9df9f23105ba001 Mon Sep 17 00:00:00 2001 From: Alex Rawlings Date: Tue, 11 Aug 2026 12:44:45 -0600 Subject: [PATCH 07/18] Stamp the catalog fixtures so they typecheck again The analyses and links these fixtures build gained required createdAt / updatedAt, so each now spreads FIXTURE_STAMPS. Also drops two fixture counts from the doc comments, which nothing kept in sync. --- src/__tests__/utils/analysis-query.test.ts | 151 +++++++++++++-------- 1 file changed, 92 insertions(+), 59 deletions(-) diff --git a/src/__tests__/utils/analysis-query.test.ts b/src/__tests__/utils/analysis-query.test.ts index cb53808c..8e211867 100644 --- a/src/__tests__/utils/analysis-query.test.ts +++ b/src/__tests__/utils/analysis-query.test.ts @@ -2,6 +2,7 @@ import type { AssignmentStatus, TextAnalysis, TokenAnalysisLink } from 'interlinearizer'; import { emptyAnalysis } from '../../types/empty-factories'; +import { FIXTURE_STAMPS } from '../test-helpers'; import { applyCatalogQuery, buildCatalogRows, @@ -30,7 +31,7 @@ function link( tokenRef: string, status: AssignmentStatus = 'approved', ): TokenAnalysisLink { - return { analysisId, status, token: { tokenRef, surfaceText: 'word' } }; + return { ...FIXTURE_STAMPS, analysisId, status, token: { tokenRef, surfaceText: 'word' } }; } describe('buildCatalogRows', () => { @@ -38,8 +39,8 @@ describe('buildCatalogRows', () => { const analysis: TextAnalysis = { ...emptyAnalysis(), tokenAnalyses: [ - { id: 'ta-1', surfaceText: 'ἀρχῇ' }, - { id: 'ta-2', surfaceText: 'λόγος' }, + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'ἀρχῇ' }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'λόγος' }, ], }; @@ -52,7 +53,7 @@ describe('buildCatalogRows', () => { it('counts each approved link on a payload as a usage', () => { const analysis: TextAnalysis = { ...emptyAnalysis(), - tokenAnalyses: [{ id: 'ta-1', surfaceText: 'λόγος' }], + tokenAnalyses: [{ ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'λόγος' }], tokenAnalysisLinks: [link('ta-1', 'GEN 1:1:0'), link('ta-1', 'GEN 1:2:4')], }; @@ -62,7 +63,7 @@ describe('buildCatalogRows', () => { it('does not count a rejected link as a usage', () => { const analysis: TextAnalysis = { ...emptyAnalysis(), - tokenAnalyses: [{ id: 'ta-1', surfaceText: 'λόγος' }], + tokenAnalyses: [{ ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'λόγος' }], tokenAnalysisLinks: [link('ta-1', 'GEN 1:1:0', 'rejected')], }; @@ -72,7 +73,7 @@ describe('buildCatalogRows', () => { it('lists a payload with no links at all as a zero-usage row', () => { const analysis: TextAnalysis = { ...emptyAnalysis(), - tokenAnalyses: [{ id: 'ta-1', surfaceText: 'λόγος' }], + tokenAnalyses: [{ ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'λόγος' }], }; expect(buildCatalogRows(analysis, scope)).toHaveLength(1); @@ -82,7 +83,7 @@ describe('buildCatalogRows', () => { it('counts only usages in the scope book toward the per-book count', () => { const analysis: TextAnalysis = { ...emptyAnalysis(), - tokenAnalyses: [{ id: 'ta-1', surfaceText: 'λόγος' }], + tokenAnalyses: [{ ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'λόγος' }], tokenAnalysisLinks: [ link('ta-1', 'GEN 1:1:0'), link('ta-1', 'GEN 1:2:4'), @@ -98,7 +99,7 @@ describe('buildCatalogRows', () => { it('parses a usage location out of the token ref', () => { const analysis: TextAnalysis = { ...emptyAnalysis(), - tokenAnalyses: [{ id: 'ta-1', surfaceText: 'λόγος' }], + tokenAnalyses: [{ ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'λόγος' }], tokenAnalysisLinks: [link('ta-1', 'GEN 1:5:12')], }; @@ -111,7 +112,7 @@ describe('buildCatalogRows', () => { it('resolves a bridged verse sid to its first verse', () => { const analysis: TextAnalysis = { ...emptyAnalysis(), - tokenAnalyses: [{ id: 'ta-1', surfaceText: 'λόγος' }], + tokenAnalyses: [{ ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'λόγος' }], tokenAnalysisLinks: [link('ta-1', 'GEN 1:3-4:0')], }; @@ -123,7 +124,7 @@ describe('buildCatalogRows', () => { it('orders usages by document position across books', () => { const analysis: TextAnalysis = { ...emptyAnalysis(), - tokenAnalyses: [{ id: 'ta-1', surfaceText: 'λόγος' }], + tokenAnalyses: [{ ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'λόγος' }], tokenAnalysisLinks: [ link('ta-1', 'ACT 1:1:0'), link('ta-1', 'EXO 2:1:0'), @@ -148,6 +149,7 @@ const analyzedEimi: TextAnalysis = { ...emptyAnalysis(), tokenAnalyses: [ { + ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'ἦν', morphemes: [{ id: 'm-1', form: 'εἰμί', writingSystem: 'grc', gloss: { en: 'to be' } }], @@ -159,7 +161,14 @@ describe('applyCatalogQuery search', () => { it('finds a row by a gloss in a language other than the active one', () => { const analysis: TextAnalysis = { ...emptyAnalysis(), - tokenAnalyses: [{ id: 'ta-1', surfaceText: 'λόγος', gloss: { en: 'word', fr: 'parole' } }], + tokenAnalyses: [ + { + ...FIXTURE_STAMPS, + id: 'ta-1', + surfaceText: 'λόγος', + gloss: { en: 'word', fr: 'parole' }, + }, + ], }; const rows = buildCatalogRows(analysis, scope); @@ -190,7 +199,9 @@ describe('applyCatalogQuery search', () => { it('matches no row on a query carrying the newline that separates two fields', () => { const analysis: TextAnalysis = { ...emptyAnalysis(), - tokenAnalyses: [{ id: 'ta-1', surfaceText: 'λόγος', gloss: { en: 'word' } }], + tokenAnalyses: [ + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'λόγος', gloss: { en: 'word' } }, + ], }; const rows = buildCatalogRows(analysis, scope); @@ -207,7 +218,9 @@ describe('applyCatalogQuery search', () => { it('finds a row whose surface form ends in a letter the query spells mid-word', () => { const analysis: TextAnalysis = { ...emptyAnalysis(), - tokenAnalyses: [{ id: 'ta-1', surfaceText: 'λόγος', gloss: { en: 'word' } }], + tokenAnalyses: [ + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'λόγος', gloss: { en: 'word' } }, + ], }; const rows = buildCatalogRows(analysis, scope); @@ -219,9 +232,9 @@ describe('applyCatalogQuery sort', () => { const analysis: TextAnalysis = { ...emptyAnalysis(), tokenAnalyses: [ - { id: 'ta-1', surfaceText: 'a' }, - { id: 'ta-2', surfaceText: 'b' }, - { id: 'ta-3', surfaceText: 'c' }, + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'a' }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'b' }, + { ...FIXTURE_STAMPS, id: 'ta-3', surfaceText: 'c' }, ], tokenAnalysisLinks: [ link('ta-1', 'GEN 1:1:0'), @@ -244,8 +257,8 @@ describe('applyCatalogQuery sort', () => { const greek: TextAnalysis = { ...emptyAnalysis(), tokenAnalyses: [ - { id: 'ta-1', surfaceText: 'βίβλος' }, - { id: 'ta-2', surfaceText: 'ἀρχή' }, + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'βίβλος' }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'ἀρχή' }, ], }; // Code-point order would leave these as given: the accented alpha sits in the Greek Extended @@ -262,8 +275,8 @@ describe('applyCatalogQuery sort', () => { const acrossBooks: TextAnalysis = { ...emptyAnalysis(), tokenAnalyses: [ - { id: 'ta-1', surfaceText: 'a' }, - { id: 'ta-2', surfaceText: 'b' }, + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'a' }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'b' }, ], tokenAnalysisLinks: [ link('ta-1', 'GEN 1:1:0'), @@ -286,8 +299,8 @@ describe('applyCatalogQuery sort', () => { const glossed: TextAnalysis = { ...emptyAnalysis(), tokenAnalyses: [ - { id: 'ta-1', surfaceText: 'a', gloss: { en: 'äpple' } }, - { id: 'ta-2', surfaceText: 'b', gloss: { en: 'zebra' } }, + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'a', gloss: { en: 'äpple' } }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'b', gloss: { en: 'zebra' } }, ], }; const rows = buildCatalogRows(glossed, scope); @@ -303,8 +316,8 @@ describe('applyCatalogQuery sort', () => { const spread: TextAnalysis = { ...emptyAnalysis(), tokenAnalyses: [ - { id: 'ta-1', surfaceText: 'a' }, - { id: 'ta-2', surfaceText: 'b' }, + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'a' }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'b' }, ], tokenAnalysisLinks: [ link('ta-1', 'JHN 1:1:0'), @@ -323,10 +336,10 @@ describe('applyCatalogQuery sort', () => { const someUnused: TextAnalysis = { ...emptyAnalysis(), tokenAnalyses: [ - { id: 'ta-1', surfaceText: 'a' }, - { id: 'ta-2', surfaceText: 'b' }, - { id: 'ta-3', surfaceText: 'c' }, - { id: 'ta-4', surfaceText: 'd' }, + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'a' }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'b' }, + { ...FIXTURE_STAMPS, id: 'ta-3', surfaceText: 'c' }, + { ...FIXTURE_STAMPS, id: 'ta-4', surfaceText: 'd' }, ], tokenAnalysisLinks: [ link('ta-2', 'JHN 1:1:0'), @@ -346,9 +359,9 @@ describe('applyCatalogQuery sort', () => { const tied: TextAnalysis = { ...emptyAnalysis(), tokenAnalyses: [ - { id: 'ta-1', surfaceText: 'γ' }, - { id: 'ta-2', surfaceText: 'β' }, - { id: 'ta-3', surfaceText: 'α' }, + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'γ' }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'β' }, + { ...FIXTURE_STAMPS, id: 'ta-3', surfaceText: 'α' }, ], }; const rows = buildCatalogRows(tied, scope); @@ -361,8 +374,8 @@ describe('applyCatalogQuery sort', () => { const homographs: TextAnalysis = { ...emptyAnalysis(), tokenAnalyses: [ - { id: 'ta-1', surfaceText: 'α', gloss: { en: 'second' } }, - { id: 'ta-2', surfaceText: 'α', gloss: { en: 'first' } }, + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'α', gloss: { en: 'second' } }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'α', gloss: { en: 'first' } }, ], }; const rows = buildCatalogRows(homographs, scope); @@ -372,32 +385,41 @@ describe('applyCatalogQuery sort', () => { }); }); -/** One analysis with a morpheme breakdown and one without. */ +/** Analyses with and without a morpheme breakdown. */ const mixedBreakdowns: TextAnalysis = { ...emptyAnalysis(), tokenAnalyses: [ { + ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'a', morphemes: [{ id: 'm-1', form: 'a', writingSystem: 'grc' }], }, - { id: 'ta-2', surfaceText: 'b' }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'b' }, ], }; -/** Two analyses tagged across every closed vocabulary, and one tagged with none of them. */ +/** Analyses tagged across every closed vocabulary, alongside an untagged analysis to filter out. */ const tagged: TextAnalysis = { ...emptyAnalysis(), tokenAnalyses: [ { + ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'a', pos: 'noun', confidence: 'high', features: { Number: 'Sg', Case: 'Nom' }, }, - { id: 'ta-2', surfaceText: 'b', pos: 'verb', confidence: 'guess', features: { Number: 'Pl' } }, - { id: 'ta-3', surfaceText: 'c' }, + { + ...FIXTURE_STAMPS, + id: 'ta-2', + surfaceText: 'b', + pos: 'verb', + confidence: 'guess', + features: { Number: 'Pl' }, + }, + { ...FIXTURE_STAMPS, id: 'ta-3', surfaceText: 'c' }, ], }; @@ -406,9 +428,9 @@ describe('applyCatalogQuery filters', () => { const analysis: TextAnalysis = { ...emptyAnalysis(), tokenAnalyses: [ - { id: 'ta-1', surfaceText: 'a' }, - { id: 'ta-2', surfaceText: 'b' }, - { id: 'ta-3', surfaceText: 'c' }, + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'a' }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'b' }, + { ...FIXTURE_STAMPS, id: 'ta-3', surfaceText: 'c' }, ], tokenAnalysisLinks: [ link('ta-1', 'GEN 1:1:0'), @@ -428,8 +450,8 @@ describe('applyCatalogQuery filters', () => { const analysis: TextAnalysis = { ...emptyAnalysis(), tokenAnalyses: [ - { id: 'ta-1', surfaceText: 'a' }, - { id: 'ta-2', surfaceText: 'b' }, + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'a' }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'b' }, ], tokenAnalysisLinks: [link('ta-1', 'GEN 1:1:0')], }; @@ -444,8 +466,8 @@ describe('applyCatalogQuery filters', () => { const analysis: TextAnalysis = { ...emptyAnalysis(), tokenAnalyses: [ - { id: 'ta-1', surfaceText: 'a', gloss: { en: 'word' } }, - { id: 'ta-2', surfaceText: 'b', gloss: { fr: 'parole' } }, + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'a', gloss: { en: 'word' } }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'b', gloss: { fr: 'parole' } }, ], }; const rows = buildCatalogRows(analysis, scope); @@ -524,8 +546,8 @@ describe('deriveFacets', () => { const analysis: TextAnalysis = { ...emptyAnalysis(), tokenAnalyses: [ - { id: 'ta-1', surfaceText: 'a' }, - { id: 'ta-2', surfaceText: 'b' }, + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'a' }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'b' }, ], tokenAnalysisLinks: [link('ta-1', 'JHN 1:1:0'), link('ta-2', 'GEN 1:1:0')], }; @@ -537,8 +559,8 @@ describe('deriveFacets', () => { const analysis: TextAnalysis = { ...emptyAnalysis(), tokenAnalyses: [ - { id: 'ta-1', surfaceText: 'a' }, - { id: 'ta-2', surfaceText: 'b' }, + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'a' }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'b' }, ], tokenAnalysisLinks: [link('ta-1', 'GEN 1:1:0'), link('ta-2', 'GEN 1:2:0')], }; @@ -550,8 +572,8 @@ describe('deriveFacets', () => { const analysis: TextAnalysis = { ...emptyAnalysis(), tokenAnalyses: [ - { id: 'ta-1', surfaceText: 'a', pos: 'noun' }, - { id: 'ta-2', surfaceText: 'b', pos: 'verb' }, + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'a', pos: 'noun' }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'b', pos: 'verb' }, ], }; @@ -562,8 +584,8 @@ describe('deriveFacets', () => { const analysis: TextAnalysis = { ...emptyAnalysis(), tokenAnalyses: [ - { id: 'ta-1', surfaceText: 'a', confidence: 'high' }, - { id: 'ta-2', surfaceText: 'b', confidence: 'guess' }, + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'a', confidence: 'high' }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'b', confidence: 'guess' }, ], }; @@ -575,8 +597,18 @@ describe('deriveFacets', () => { const analysis: TextAnalysis = { ...emptyAnalysis(), tokenAnalyses: [ - { id: 'ta-1', surfaceText: 'a', features: { Case: 'Nom', Number: 'Sg' } }, - { id: 'ta-2', surfaceText: 'b', features: { Case: 'Nom', Number: 'Pl' } }, + { + ...FIXTURE_STAMPS, + id: 'ta-1', + surfaceText: 'a', + features: { Case: 'Nom', Number: 'Sg' }, + }, + { + ...FIXTURE_STAMPS, + id: 'ta-2', + surfaceText: 'b', + features: { Case: 'Nom', Number: 'Pl' }, + }, ], }; @@ -589,9 +621,9 @@ describe('deriveFacets', () => { const analysis: TextAnalysis = { ...emptyAnalysis(), tokenAnalyses: [ - { id: 'ta-1', surfaceText: 'a', features: { Number: 'Sg' } }, - { id: 'ta-2', surfaceText: 'b', features: { Number: 'Pl' } }, - { id: 'ta-3', surfaceText: 'c' }, + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'a', features: { Number: 'Sg' } }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'b', features: { Number: 'Pl' } }, + { ...FIXTURE_STAMPS, id: 'ta-3', surfaceText: 'c' }, ], }; @@ -606,8 +638,9 @@ describe('deriveFacets', () => { const analysis: TextAnalysis = { ...emptyAnalysis(), tokenAnalyses: [ - { id: 'ta-1', surfaceText: 'ἀρχῇ', gloss: { en: 'beginning' } }, + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'ἀρχῇ', gloss: { en: 'beginning' } }, { + ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'ἦν', gloss: { en: 'was' }, From e051402433e7f18d1a3c4dc51fe7b27936e7c57a Mon Sep 17 00:00:00 2001 From: Alex Rawlings Date: Tue, 11 Aug 2026 13:23:30 -0600 Subject: [PATCH 08/18] Make an untagged analysis a facet choice of its own Absence now lists as undefined after a field's values, so a field one analysis is tagged with still raises a dropdown, and a search for what an analysis lacks is possible. Also folds a pasted Windows line ending in a catalog search. --- src/__tests__/utils/analysis-query.test.ts | 57 ++++++++++++- src/utils/analysis-query.ts | 97 +++++++++++++++------- 2 files changed, 121 insertions(+), 33 deletions(-) diff --git a/src/__tests__/utils/analysis-query.test.ts b/src/__tests__/utils/analysis-query.test.ts index 8e211867..092b3274 100644 --- a/src/__tests__/utils/analysis-query.test.ts +++ b/src/__tests__/utils/analysis-query.test.ts @@ -208,6 +208,14 @@ describe('applyCatalogQuery search', () => { expect(applyCatalogQuery(rows, makeQuery({ search: 'λογος\nword' }))).toHaveLength(0); }); + // The gloss is one field, so the query has to fold to the space inside it rather than to a + // carriage return no field can hold. + it('finds a row on a query carrying a pasted Windows line ending', () => { + const rows = buildCatalogRows(analyzedEimi, scope); + + expect(applyCatalogQuery(rows, makeQuery({ search: 'to\r\nbe' }))).toHaveLength(1); + }); + it('ignores whitespace around the query, which a typed or pasted search carries', () => { const rows = buildCatalogRows(analyzedEimi, scope); @@ -504,6 +512,24 @@ describe('applyCatalogQuery filters', () => { expect(applyCatalogQuery(rows, query).map((r) => r.analysisId)).toEqual(['ta-1', 'ta-2']); }); + it('keeps only rows carrying no part of speech when that is the selected choice', () => { + const rows = buildCatalogRows(tagged, scope); + const query = makeQuery({ filters: { pos: [undefined] } }); + + expect(applyCatalogQuery(rows, query).map((r) => r.analysisId)).toEqual(['ta-3']); + }); + + it('keeps rows carrying no feature value alongside those carrying a selected one', () => { + const rows = buildCatalogRows(tagged, scope); + const query = makeQuery({ filters: { features: { Case: ['Nom', undefined] } } }); + + expect(applyCatalogQuery(rows, query).map((r) => r.analysisId)).toEqual([ + 'ta-1', + 'ta-2', + 'ta-3', + ]); + }); + it('keeps only rows carrying one of the selected confidence levels', () => { const rows = buildCatalogRows(tagged, scope); const query = makeQuery({ filters: { confidence: ['guess'] } }); @@ -617,7 +643,8 @@ describe('deriveFacets', () => { }); }); - it('draws a feature facet from the rows that carry the feature at all', () => { + // ta-3 carries no Number, which is a choice of the facet rather than an omission from it. + it('offers the untagged choice on a feature only some rows carry', () => { const analysis: TextAnalysis = { ...emptyAnalysis(), tokenAnalyses: [ @@ -627,11 +654,35 @@ describe('deriveFacets', () => { ], }; - expect(deriveFacets(buildCatalogRows(analysis, scope)).features).toEqual({ - Number: ['Pl', 'Sg'], + expect(deriveFacets(buildCatalogRows(analysis, scope)).features).toStrictEqual({ + Number: ['Pl', 'Sg', undefined], }); }); + it('offers a part-of-speech facet where a single row is tagged and the rest are not', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'a', pos: 'noun' }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'b' }, + ], + }; + + expect(deriveFacets(buildCatalogRows(analysis, scope)).pos).toStrictEqual(['noun', undefined]); + }); + + it('hides the part-of-speech facet when every row carries the same one', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'a', pos: 'noun' }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'b', pos: 'noun' }, + ], + }; + + expect(deriveFacets(buildCatalogRows(analysis, scope)).pos).toBeUndefined(); + }); + // Nothing in the app writes pos, features, or confidence yet, so an analysis shaped the way the // gloss and morpheme write paths shape it must not raise a dropdown with nothing behind it. it('derives no facet from fields no write path populates', () => { diff --git a/src/utils/analysis-query.ts b/src/utils/analysis-query.ts index 101f6462..93416dc5 100644 --- a/src/utils/analysis-query.ts +++ b/src/utils/analysis-query.ts @@ -44,16 +44,20 @@ export interface CatalogRow { export type CatalogSort = 'usageCount' | 'usageCountInBook' | 'surfaceText' | 'gloss' | 'firstUsage'; -/** Which rows are kept. An absent filter is inactive, as is an empty selection. */ +/** + * Which rows are kept. An absent filter is inactive, as is an empty selection. A selection of a + * field an analysis holds one value of accepts `undefined` as the choice for rows carrying no value + * for it, so each such field can be filtered for what it lacks as well as for what it carries. + */ export interface CatalogFilters { /** Keeps rows used in at least one of these books. */ books?: readonly string[]; /** Keeps rows carrying one of these parts of speech. */ - pos?: readonly string[]; + pos?: readonly (string | undefined)[]; /** Keeps rows carrying one of these confidence levels. */ - confidence?: readonly Confidence[]; + confidence?: readonly (Confidence | undefined)[]; /** Keeps rows matching every named feature, each against its own accepted values. */ - features?: Readonly>; + features?: Readonly>; /** Keeps only rows nothing uses. */ zeroUsages?: boolean; /** Keeps only rows with no gloss in the scope's analysis language. */ @@ -194,26 +198,35 @@ function compareFirstUsage(a: CatalogRow, b: CatalogRow): number { } /** - * The closed-vocabulary values present in the rows, each offered as a filter. + * The choices the rows offer for each closed-vocabulary field, each field's list offered as a + * filter. + * + * For a field an analysis holds one value of, carrying no value is a choice of its own, listed as + * `undefined` after the values, so a field a single row is tagged with is still offered — that + * choice narrows the list to the tagged row, and its counterpart answers which analyses are still + * missing the field. A facet is absent only when the rows are all in one of these states, since a + * dropdown offering the state everything is already in filters nothing. + * + * Books are listed by value alone: an analysis in no book is one nothing uses, which + * {@link CatalogFilters} keeps on its own terms. * - * A facet is absent unless the rows disagree on it — a dropdown with one choice filters nothing. * Assignment status is not a facet: only approved links count as usages, so every row agrees by * construction. */ export interface CatalogFacets { /** Canonical order. */ books?: readonly string[]; - pos?: readonly string[]; - confidence?: readonly Confidence[]; - /** Distinct values per feature name, each name judged on its own. */ - features?: Readonly>; + pos?: readonly (string | undefined)[]; + confidence?: readonly (Confidence | undefined)[]; + /** Distinct choices per feature name, each name judged on its own. */ + features?: Readonly>; } /** - * Collects the distinct values the rows carry for one field, sorted, or `undefined` when fewer than - * two are present. + * Collects the distinct values the rows carry for one field holding several at a time, sorted, or + * `undefined` when fewer than two are present. */ -function facetValues( +function multiValueFacet( rows: readonly CatalogRow[], valuesOf: (row: CatalogRow) => Iterable, compare?: (a: T, b: T) => number, @@ -222,14 +235,33 @@ function facetValues( return distinct.size < 2 ? undefined : [...distinct].sort(compare); } +/** + * Collects the choices the rows offer for one field holding a value at a time — the distinct values + * sorted, then `undefined` where any row carries none — or `undefined` when the rows are all in one + * state. + */ +function singleValueFacet( + rows: readonly CatalogRow[], + valueOf: (row: CatalogRow) => T | undefined, + compare?: (a: T, b: T) => number, +): readonly (T | undefined)[] | undefined { + const present = new Set(); + let anyUntagged = false; + rows.forEach((row) => { + const value = valueOf(row); + if (value === undefined) anyUntagged = true; + else present.add(value); + }); + const sorted = [...present].sort(compare); + const choices = anyUntagged ? [...sorted, undefined] : sorted; + return choices.length < 2 ? undefined : choices; +} + /** Collects a facet per feature name that earns one, or `undefined` when no name does. */ function featureFacets(rows: readonly CatalogRow[]): CatalogFacets['features'] { const names = new Set(rows.flatMap((row) => Object.keys(row.features ?? {}))); - const facets = [...names].reduce>((acc, name) => { - const values = facetValues(rows, (row) => { - const value = row.features?.[name]; - return value === undefined ? [] : [value]; - }); + const facets = [...names].reduce>((acc, name) => { + const values = singleValueFacet(rows, (row) => row.features?.[name]); if (values) acc[name] = values; return acc; }, {}); @@ -242,15 +274,15 @@ const CONFIDENCE_ORDER: readonly Confidence[] = ['high', 'medium', 'low', 'guess /** Derives the filter facets worth offering against these rows. */ export function deriveFacets(rows: readonly CatalogRow[]): CatalogFacets { return { - books: facetValues( + books: multiValueFacet( rows, (row) => row.books, (a, b) => Canon.bookIdToNumber(a) - Canon.bookIdToNumber(b), ), - pos: facetValues(rows, (row) => (row.pos === undefined ? [] : [row.pos])), - confidence: facetValues( + pos: singleValueFacet(rows, (row) => row.pos), + confidence: singleValueFacet( rows, - (row) => (row.confidence === undefined ? [] : [row.confidence]), + (row) => row.confidence, (a, b) => CONFIDENCE_ORDER.indexOf(a) - CONFIDENCE_ORDER.indexOf(b), ), features: featureFacets(rows), @@ -266,11 +298,14 @@ function isActive(selected: readonly T[] | undefined): selected is readonly T } /** - * Whether a field holding one value at a time carries one its selection accepts. A field carrying - * no value at all fails an active selection. + * Whether a field holding one value at a time is in a state its selection accepts, carrying no + * value being the state the selection's `undefined` choice accepts. */ -function passesValue(selected: readonly T[] | undefined, value: T | undefined): boolean { - return !isActive(selected) || (value !== undefined && selected.includes(value)); +function passesValue( + selected: readonly (T | undefined)[] | undefined, + value: T | undefined, +): boolean { + return !isActive(selected) || selected.includes(value); } /** Whether the row satisfies every feature named in the selection, each judged on its own values. */ @@ -341,10 +376,12 @@ export function applyCatalogQuery( rows: readonly CatalogRow[], query: CatalogQuery, ): readonly CatalogRow[] { - // A newline separates the fields a match must not span, so one typed into the query reads as an - // ordinary space instead. That and the trim happen here rather than inside foldForSearch, which - // also folds each row's search text, where the separators must survive. - const search = foldForSearch(query.search.replace(/\n/g, ' ').trim()); + // A newline separates the fields a match must not span, so a line ending typed into the query + // reads as an ordinary space instead — including the carriage return a paste from a Windows + // source carries, which no row's search text holds. That and the trim happen here rather than + // inside foldForSearch, which also folds each row's search text, where the separators must + // survive. + const search = foldForSearch(query.search.replace(/\r\n?|\n/g, ' ').trim()); return rows .filter((row) => row.searchText.includes(search) && passesFilters(row, query.filters)) .sort((a, b) => compareBySort(a, b, query)); From e55559da11ca50f91b9e4d8a122a37f441517d3f Mon Sep 17 00:00:00 2001 From: Alex Rawlings Date: Tue, 11 Aug 2026 13:37:31 -0600 Subject: [PATCH 09/18] Name the conditions the catalog docs hold under Usage counts match the suggestion pool's frequency only while a token carries at most one approved link, and a single-book draft is offered no books facet even when unused rows sit alongside the used ones. --- src/utils/analysis-query.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/utils/analysis-query.ts b/src/utils/analysis-query.ts index 93416dc5..daf7e4dd 100644 --- a/src/utils/analysis-query.ts +++ b/src/utils/analysis-query.ts @@ -125,8 +125,10 @@ function compareDocumentOrder(a: CatalogUsage, b: CatalogUsage): number { * Files each analysis's usages under its id, each list in document order. * * Only an approved link is a usage: a rejected link is by definition not a place the analysis is - * applied, and counting approvals alone keeps a row's count equal to the analysis's frequency in - * the suggestion pool, so the two can never disagree. + * applied. Counting approvals alone leaves a row's count equal to the analysis's frequency in the + * suggestion pool, which counts the tokens an approval sits on rather than the approvals themselves + * — the two agree for as long as a token carries at most one approved link, as the data model + * requires. */ function groupUsagesByAnalysisId( links: readonly TokenAnalysisLink[], @@ -208,7 +210,9 @@ function compareFirstUsage(a: CatalogRow, b: CatalogRow): number { * dropdown offering the state everything is already in filters nothing. * * Books are listed by value alone: an analysis in no book is one nothing uses, which - * {@link CatalogFilters} keeps on its own terms. + * {@link CatalogFilters} keeps on its own terms. A draft confined to a single book therefore offers + * no books facet even where unused rows sit alongside the used ones: the unused are reachable on + * those terms, the used only once a second book is in play. * * Assignment status is not a facet: only approved links count as usages, so every row agrees by * construction. From 220cd09cce29fa1250678c1050dba685620488fa Mon Sep 17 00:00:00 2001 From: Alex Rawlings Date: Tue, 11 Aug 2026 14:19:57 -0600 Subject: [PATCH 10/18] Sort an analysis with no gloss to the end of the listing --- src/__tests__/utils/analysis-query.test.ts | 31 ++++++++++++++++++++++ src/utils/analysis-query.ts | 26 ++++++++++++++---- 2 files changed, 52 insertions(+), 5 deletions(-) diff --git a/src/__tests__/utils/analysis-query.test.ts b/src/__tests__/utils/analysis-query.test.ts index 092b3274..9b6f4822 100644 --- a/src/__tests__/utils/analysis-query.test.ts +++ b/src/__tests__/utils/analysis-query.test.ts @@ -319,6 +319,37 @@ describe('applyCatalogQuery sort', () => { expect(sortByGloss(new Intl.Collator('en'))).toEqual(['ta-1', 'ta-2']); }); + // Collating alone would open the list with ta-2: a missing gloss is the empty string, which sorts + // ahead of every word. + it('sorts a row with no gloss in the scope language after every glossed one', () => { + const partlyGlossed: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'a', gloss: { en: 'zebra' } }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'b', gloss: { fr: 'parole' } }, + { ...FIXTURE_STAMPS, id: 'ta-3', surfaceText: 'c', gloss: { en: 'apple' } }, + ], + }; + const rows = buildCatalogRows(partlyGlossed, scope); + + expect(applyCatalogQuery(rows, makeQuery({ sort: 'gloss' })).map((r) => r.analysisId)) // + .toEqual(['ta-3', 'ta-1', 'ta-2']); + }); + + it('orders two rows that both lack a gloss by surface form', () => { + const unglossed: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'β' }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'α' }, + ], + }; + const rows = buildCatalogRows(unglossed, scope); + + expect(applyCatalogQuery(rows, makeQuery({ sort: 'gloss' })).map((r) => r.analysisId)) // + .toEqual(['ta-2', 'ta-1']); + }); + // ta-1 is the more used analysis, so a count sort would put it first; ta-2 appears earlier. it('orders by the first usage in document order', () => { const spread: TextAnalysis = { diff --git a/src/utils/analysis-query.ts b/src/utils/analysis-query.ts index daf7e4dd..45f83aee 100644 --- a/src/utils/analysis-query.ts +++ b/src/utils/analysis-query.ts @@ -26,7 +26,7 @@ export interface CatalogRow { gloss: string; morphemes: readonly MorphemeAnalysis[]; pos?: string; - features?: Record; + features?: Readonly>; confidence?: Confidence; /** Places in the whole draft where the analysis is applied. */ usageCount: number; @@ -199,6 +199,16 @@ function compareFirstUsage(a: CatalogRow, b: CatalogRow): number { return compareDocumentOrder(firstA, firstB); } +/** + * Orders two rows by gloss, an analysis with none in the scope's language coming after every one + * that has one. + */ +function compareGloss(a: CatalogRow, b: CatalogRow, glossCollator: Intl.Collator): number { + if (!a.gloss) return b.gloss ? 1 : 0; + if (!b.gloss) return -1; + return glossCollator.compare(a.gloss, b.gloss); +} + /** * The choices the rows offer for each closed-vocabulary field, each field's list offered as a * filter. @@ -275,7 +285,13 @@ function featureFacets(rows: readonly CatalogRow[]): CatalogFacets['features'] { /** Confidence levels from strongest to weakest. */ const CONFIDENCE_ORDER: readonly Confidence[] = ['high', 'medium', 'low', 'guess']; -/** Derives the filter facets worth offering against these rows. */ +/** + * Derives the filter facets worth offering against these rows. + * + * @param rows Every row {@link buildCatalogRows} built, never what {@link applyCatalogQuery} narrowed + * them to: a facet judged against the rows its own selection kept collapses as soon as that + * selection is made, leaving nothing to widen it back by. + */ export function deriveFacets(rows: readonly CatalogRow[]): CatalogFacets { return { books: multiValueFacet( @@ -346,7 +362,7 @@ function compareBySortKey(a: CatalogRow, b: CatalogRow, query: CatalogQuery): nu case 'surfaceText': return query.surfaceCollator.compare(a.surfaceText, b.surfaceText); case 'gloss': - return query.glossCollator.compare(a.gloss, b.gloss); + return compareGloss(a, b, query.glossCollator); case 'firstUsage': return compareFirstUsage(a, b); default: @@ -354,11 +370,11 @@ function compareBySortKey(a: CatalogRow, b: CatalogRow, query: CatalogQuery): nu } } -/** Orders two rows by their form and then their gloss, both ascending. */ +/** Orders two rows by their form ascending, falling back to their gloss. */ function compareByText(a: CatalogRow, b: CatalogRow, query: CatalogQuery): number { return ( query.surfaceCollator.compare(a.surfaceText, b.surfaceText) || - query.glossCollator.compare(a.gloss, b.gloss) + compareGloss(a, b, query.glossCollator) ); } From abbdd3a7f76d0c49e7bbad3de0202ef293f4c6c9 Mon Sep 17 00:00:00 2001 From: Alex Rawlings Date: Tue, 11 Aug 2026 14:38:42 -0600 Subject: [PATCH 11/18] Document what a catalog row's feature map holds --- src/utils/analysis-query.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/utils/analysis-query.ts b/src/utils/analysis-query.ts index 45f83aee..e38125cc 100644 --- a/src/utils/analysis-query.ts +++ b/src/utils/analysis-query.ts @@ -26,6 +26,7 @@ export interface CatalogRow { gloss: string; morphemes: readonly MorphemeAnalysis[]; pos?: string; + /** Morphosyntactic features, each feature name mapped to the analysis's value for it. */ features?: Readonly>; confidence?: Confidence; /** Places in the whole draft where the analysis is applied. */ From d8fc7174c05e7cf4c3363937f310a03749860647 Mon Sep 17 00:00:00 2001 From: Alex Rawlings Date: Tue, 11 Aug 2026 15:03:44 -0600 Subject: [PATCH 12/18] Make a new catalog sort key fail to compile until it is ordered --- src/utils/analysis-query.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/utils/analysis-query.ts b/src/utils/analysis-query.ts index e38125cc..41b69c6a 100644 --- a/src/utils/analysis-query.ts +++ b/src/utils/analysis-query.ts @@ -355,9 +355,13 @@ function passesFilters(row: CatalogRow, filters: CatalogFilters): boolean { /** * Orders two rows by the query's sort key: counts descending, so the most used lead; text * ascending. + * + * A key added to {@link CatalogSort} fails to compile here until this orders it too. */ function compareBySortKey(a: CatalogRow, b: CatalogRow, query: CatalogQuery): number { switch (query.sort) { + case 'usageCount': + return b.usageCount - a.usageCount; case 'usageCountInBook': return b.usageCountInBook - a.usageCountInBook; case 'surfaceText': @@ -366,8 +370,7 @@ function compareBySortKey(a: CatalogRow, b: CatalogRow, query: CatalogQuery): nu return compareGloss(a, b, query.glossCollator); case 'firstUsage': return compareFirstUsage(a, b); - default: - return b.usageCount - a.usageCount; + // no default } } From 9c97efff7545dfe0094d7695768306a6c5e36399 Mon Sep 17 00:00:00 2001 From: Alex Rawlings Date: Tue, 11 Aug 2026 15:55:09 -0600 Subject: [PATCH 13/18] Document that the search fold keeps an eszett distinct --- src/__tests__/utils/search-fold.test.ts | 4 ++++ src/utils/search-fold.ts | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/__tests__/utils/search-fold.test.ts b/src/__tests__/utils/search-fold.test.ts index 089a11eb..3c82ba9f 100644 --- a/src/__tests__/utils/search-fold.test.ts +++ b/src/__tests__/utils/search-fold.test.ts @@ -52,4 +52,8 @@ describe('foldForSearch', () => { it('keeps a stroked letter, which its script counts as a letter of its own', () => { expect(foldForSearch('łuk')).not.toBe(foldForSearch('luk')); }); + + it('keeps an eszett distinct from the two letters that spell its uppercase', () => { + expect(foldForSearch('Straße')).not.toBe(foldForSearch('strasse')); + }); }); diff --git a/src/utils/search-fold.ts b/src/utils/search-fold.ts index 8d2759cc..b33817d6 100644 --- a/src/utils/search-fold.ts +++ b/src/utils/search-fold.ts @@ -34,7 +34,9 @@ const FINAL_LETTER_PATTERN = new RegExp(`[${Object.keys(FINAL_LETTER_FORMS).join * Some distinctions deliberately survive the fold. A spacing combining mark spells a dependent * vowel, so dropping it would merge words differing only in that vowel; nonspacing marks alone go. * And a letter written with a stroke or bar keeps it, matching how the scripts that write one count - * it a letter of its own rather than a decorated one. + * it a letter of its own rather than a decorated one. So does a letter whose uppercase is spelled + * with two: `ß` stays itself rather than expanding to `ss`, so neither spelling of a German word + * finds the other. */ export function foldForSearch(text: string): string { return ( From 9288c6d932aa84e74ca6613c538475e4121b5a07 Mon Sep 17 00:00:00 2001 From: Alex Rawlings Date: Tue, 11 Aug 2026 16:17:09 -0600 Subject: [PATCH 14/18] Catch a search fold that drops a letter it should keep --- src/__tests__/utils/search-fold.test.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/__tests__/utils/search-fold.test.ts b/src/__tests__/utils/search-fold.test.ts index 3c82ba9f..2a6b2b52 100644 --- a/src/__tests__/utils/search-fold.test.ts +++ b/src/__tests__/utils/search-fold.test.ts @@ -20,7 +20,9 @@ describe('foldForSearch', () => { }); it('keeps a spacing combining mark, which spells a vowel rather than decorating one', () => { - expect(foldForSearch('कि')).not.toBe(foldForSearch('क')); + const folded = foldForSearch('कि'); + expect(folded).toBe('कि'); + expect(folded).not.toBe(foldForSearch('क')); }); it('folds a Greek final sigma to the shape the letter takes mid-word', () => { @@ -50,10 +52,14 @@ describe('foldForSearch', () => { }); it('keeps a stroked letter, which its script counts as a letter of its own', () => { - expect(foldForSearch('łuk')).not.toBe(foldForSearch('luk')); + const folded = foldForSearch('łuk'); + expect(folded).toBe('łuk'); + expect(folded).not.toBe(foldForSearch('luk')); }); it('keeps an eszett distinct from the two letters that spell its uppercase', () => { - expect(foldForSearch('Straße')).not.toBe(foldForSearch('strasse')); + const folded = foldForSearch('Straße'); + expect(folded).toBe('straße'); + expect(folded).not.toBe(foldForSearch('strasse')); }); }); From 753e4faff590e78f5c65bde6d1866017ebc77587 Mon Sep 17 00:00:00 2001 From: Alex Rawlings Date: Tue, 11 Aug 2026 16:26:51 -0600 Subject: [PATCH 15/18] Link the identity fold the search fold warns against --- src/utils/search-fold.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/search-fold.ts b/src/utils/search-fold.ts index b33817d6..99bbce65 100644 --- a/src/utils/search-fold.ts +++ b/src/utils/search-fold.ts @@ -1,5 +1,5 @@ /** - * @file The search fold, deliberately not merged with `normalizeSurfaceForm` in + * @file The search fold, deliberately not merged with {@link normalizeSurfaceForm} in * `analysis-identity`. The two look alike enough to invite it, and merging them would corrupt * data rather than fail loudly: folding a diacritic into identity puts genuinely different words * on one shared payload, silently dropping a distinction the user recorded. From 4096003ffb3d07c308f2131195b949b631d22295 Mon Sep 17 00:00:00 2001 From: Alex Rawlings Date: Wed, 12 Aug 2026 09:31:29 -0600 Subject: [PATCH 16/18] Fold each catalog analysis's search text once Cache the fold per analysis record, count a token once however many approved links carry it, and keep a field's own line break from spelling the separator a match may not span. --- src/__tests__/store/analysisSlice.test.ts | 9 ++ src/__tests__/utils/analysis-query.test.ts | 39 ++++++ src/__tests__/utils/search-fold.test.ts | 10 ++ src/store/analysisSlice.ts | 6 +- src/utils/analysis-query.ts | 136 +++++++++++++++------ src/utils/search-fold.ts | 18 ++- 6 files changed, 172 insertions(+), 46 deletions(-) diff --git a/src/__tests__/store/analysisSlice.test.ts b/src/__tests__/store/analysisSlice.test.ts index e4ecf012..5172341d 100644 --- a/src/__tests__/store/analysisSlice.test.ts +++ b/src/__tests__/store/analysisSlice.test.ts @@ -1866,6 +1866,15 @@ describe('selectCatalogRows', () => { expect(a).toBe(b); }); + it('keeps a book cached across a call for another one', () => { + const store = createAnalysisStore(); + + const first = selectCatalogRows(store.getState().analysis, 'GEN'); + selectCatalogRows(store.getState().analysis, 'MAT'); + + expect(selectCatalogRows(store.getState().analysis, 'GEN')).toBe(first); + }); + it('rebuilds the rows after a gloss write', () => { const store = createAnalysisStore(); const before = selectCatalogRows(store.getState().analysis, 'GEN'); diff --git a/src/__tests__/utils/analysis-query.test.ts b/src/__tests__/utils/analysis-query.test.ts index 9b6f4822..d362a0fc 100644 --- a/src/__tests__/utils/analysis-query.test.ts +++ b/src/__tests__/utils/analysis-query.test.ts @@ -60,6 +60,18 @@ describe('buildCatalogRows', () => { expect(buildCatalogRows(analysis, scope)[0].usageCount).toBe(2); }); + // The data model allows a token only one approved analysis, so a second link on it is a + // duplicate rather than a second place the analysis is applied. + it('counts a token carrying the same approval twice as one usage', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [{ ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'λόγος' }], + tokenAnalysisLinks: [link('ta-1', 'GEN 1:1:0'), link('ta-1', 'GEN 1:1:0')], + }; + + expect(buildCatalogRows(analysis, scope)[0].usageCount).toBe(1); + }); + it('does not count a rejected link as a usage', () => { const analysis: TextAnalysis = { ...emptyAnalysis(), @@ -216,6 +228,33 @@ describe('applyCatalogQuery search', () => { expect(applyCatalogQuery(rows, makeQuery({ search: 'to\r\nbe' }))).toHaveLength(1); }); + // A gloss holding a line break of its own would otherwise spell the separator no match may span. + it('finds a row by a gloss whose own line break the query spells as a space', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'λόγος', gloss: { en: 'spoken\nword' } }, + ], + }; + const rows = buildCatalogRows(analysis, scope); + + expect(applyCatalogQuery(rows, makeQuery({ search: 'spoken word' }))).toHaveLength(1); + }); + + // The marks fold away, leaving the empty query rather than one nothing can match. + it('keeps every row on a query made only of combining marks', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'λόγος' }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'ἀρχῇ' }, + ], + }; + const rows = buildCatalogRows(analysis, scope); + + expect(applyCatalogQuery(rows, makeQuery({ search: '\u0301\u0308' }))).toHaveLength(2); + }); + it('ignores whitespace around the query, which a typed or pasted search carries', () => { const rows = buildCatalogRows(analyzedEimi, scope); diff --git a/src/__tests__/utils/search-fold.test.ts b/src/__tests__/utils/search-fold.test.ts index 2a6b2b52..f83edc43 100644 --- a/src/__tests__/utils/search-fold.test.ts +++ b/src/__tests__/utils/search-fold.test.ts @@ -25,6 +25,16 @@ describe('foldForSearch', () => { expect(folded).not.toBe(foldForSearch('क')); }); + // The virama that binds the conjunct is nonspacing. + it('merges a Devanagari conjunct with the cluster written without its joiner', () => { + expect(foldForSearch('क्ष')).toBe(foldForSearch('कष')); + }); + + // Thai writes this word's vowel and tone as nonspacing marks, leaving the consonant alone. + it('merges Thai words differing only in the marks that spell their vowels', () => { + expect(foldForSearch('ที่')).toBe(foldForSearch('ท')); + }); + it('folds a Greek final sigma to the shape the letter takes mid-word', () => { expect(foldForSearch('λόγος')).toBe(foldForSearch('λογοσ')); }); diff --git a/src/store/analysisSlice.ts b/src/store/analysisSlice.ts index 83487b40..426ef2d7 100644 --- a/src/store/analysisSlice.ts +++ b/src/store/analysisSlice.ts @@ -1039,7 +1039,11 @@ export const selectPoolIndex = createSelector( * Memoized selector building the Analysis Catalog's rows — one per distinct token analysis, with * its usage counts and locations — against the book named as the second argument. Only a change to * the analysis it reads or to the named book rebuilds the rows, so searching and sorting the result - * never does, and two components asking for different books keep each other's rows cached. + * never does. + * + * Rows are cached per book code asked for rather than in a single slot, so two components reading + * different books do not thrash. Nothing evicts an entry, and the canon bounds how many there can + * be. */ export const selectCatalogRows = createSelector( selectTokenAnalyses, diff --git a/src/utils/analysis-query.ts b/src/utils/analysis-query.ts index 41b69c6a..26a9231d 100644 --- a/src/utils/analysis-query.ts +++ b/src/utils/analysis-query.ts @@ -51,7 +51,10 @@ export type CatalogSort = * for it, so each such field can be filtered for what it lacks as well as for what it carries. */ export interface CatalogFilters { - /** Keeps rows used in at least one of these books. */ + /** + * Keeps rows used in at least one of these books. A row nothing uses sits in no book, so an + * active selection drops every unused row. + */ books?: readonly string[]; /** Keeps rows carrying one of these parts of speech. */ pos?: readonly (string | undefined)[]; @@ -59,7 +62,7 @@ export interface CatalogFilters { confidence?: readonly (Confidence | undefined)[]; /** Keeps rows matching every named feature, each against its own accepted values. */ features?: Readonly>; - /** Keeps only rows nothing uses. */ + /** Keeps only rows nothing uses. Those sit in no book, so a book selection alongside keeps none. */ zeroUsages?: boolean; /** Keeps only rows with no gloss in the scope's analysis language. */ missingGloss?: boolean; @@ -71,7 +74,8 @@ export interface CatalogFilters { export interface CatalogQuery { /** * Matched as a plain substring against each row's folded text, ignoring surrounding whitespace; - * `''` matches everything. + * `''` matches everything, as does anything the fold empties — a query of combining marks alone + * leaves the list at its full width rather than emptying it. */ search: string; sort: CatalogSort; @@ -99,6 +103,9 @@ export interface CatalogUsage { * Reads the location a token ref names. A ref is a verse SID plus the token's character offset * (`"GEN 1:1:0"`), so the whole location is recoverable from the string. The SID's verse portion is * verbatim USJ, so a bridged verse resolves to the first verse it names. + * + * Every part is taken as the tokenizer wrote it rather than validated: a ref reaching here names a + * token of a tokenized book, never anything a user typed. */ function parseUsage(tokenRef: string): CatalogUsage { const [chapterPart, versePart, charPart] = tokenRef.slice(tokenRef.indexOf(' ') + 1).split(':'); @@ -112,7 +119,13 @@ function parseUsage(tokenRef: string): CatalogUsage { }; } -/** Orders two usages by document position, taking books in canonical rather than alphabetical order. */ +/** + * Orders two usages by document position, taking books in canonical rather than alphabetical order. + * + * Total over the refs a tokenized book produces, whose parts are all present and whose book code is + * canonical. A code outside the canon has no number to be placed by and would lead the list rather + * than trail it. + */ function compareDocumentOrder(a: CatalogUsage, b: CatalogUsage): number { return ( Canon.bookIdToNumber(a.book) - Canon.bookIdToNumber(b.book) || @@ -126,39 +139,60 @@ function compareDocumentOrder(a: CatalogUsage, b: CatalogUsage): number { * Files each analysis's usages under its id, each list in document order. * * Only an approved link is a usage: a rejected link is by definition not a place the analysis is - * applied. Counting approvals alone leaves a row's count equal to the analysis's frequency in the - * suggestion pool, which counts the tokens an approval sits on rather than the approvals themselves - * — the two agree for as long as a token carries at most one approved link, as the data model - * requires. + * applied. A token counts once however many approved links carry it, so a row's count equals the + * analysis's frequency in the suggestion pool — which counts the tokens an approval sits on rather + * than the approvals themselves — whatever a duplicate link says. */ function groupUsagesByAnalysisId( links: readonly TokenAnalysisLink[], ): ReadonlyMap { const byId = links.reduce((acc, l) => { if (l.status !== 'approved') return acc; - const usages = acc.get(l.analysisId); - const usage = parseUsage(l.token.tokenRef); - if (usages) usages.push(usage); - else acc.set(l.analysisId, [usage]); - return acc; - }, new Map()); - byId.forEach((usages) => usages.sort(compareDocumentOrder)); - return byId; + const usages = acc.get(l.analysisId) ?? new Map(); + usages.set(l.token.tokenRef, parseUsage(l.token.tokenRef)); + return acc.set(l.analysisId, usages); + }, new Map>()); + return new Map( + [...byId].map(([id, usages]) => [id, [...usages.values()].sort(compareDocumentOrder)]), + ); +} + +/** Separates the fields a match may not span, and so the one character no field may hold. */ +const FIELD_SEPARATOR = '\n'; + +/** Reads a line ending as the ordinary space it stands for between words. */ +function collapseLineEndings(text: string): string { + return text.replace(/\r\n?|\n/g, ' '); } +/** + * The folded text each analysis is searched by, held against the record it was folded from, so a + * write re-folds the one analysis it replaced. A record edited in place rather than replaced would + * keep the fold it arrived with. + */ +const searchTextByAnalysis = new WeakMap(); + /** * Assembles the folded text a search matches an analysis by, drawing on its forms and on every * gloss it carries whatever the language, so a gloss stays findable in a language the project does - * not declare. No match spans two of the assembled fields. + * not declare. No match spans two of the assembled fields: a line ending inside a field contributes + * a space, so the separator between fields is the only one in the text. */ function buildSearchText(ta: TokenAnalysis): string { + const cached = searchTextByAnalysis.get(ta); + if (cached !== undefined) return cached; + const morphemeText = (ta.morphemes ?? []).flatMap((m) => [ m.form, ...Object.values(m.gloss ?? {}), ]); - return foldForSearch( - [ta.surfaceText, ...Object.values(ta.gloss ?? {}), ...morphemeText].join('\n'), + const searchText = foldForSearch( + [ta.surfaceText, ...Object.values(ta.gloss ?? {}), ...morphemeText] + .map(collapseLineEndings) + .join(FIELD_SEPARATOR), ); + searchTextByAnalysis.set(ta, searchText); + return searchText; } /** @@ -251,9 +285,23 @@ function multiValueFacet( } /** - * Collects the choices the rows offer for one field holding a value at a time — the distinct values - * sorted, then `undefined` where any row carries none — or `undefined` when the rows are all in one - * state. + * Assembles the choices for one field holding a value at a time — the distinct values sorted, then + * `undefined` where the field goes untagged — or `undefined` when fewer than two choices result, a + * lone choice being the state everything is already in. + */ +function valueChoices( + present: ReadonlySet, + anyUntagged: boolean, + compare?: (a: T, b: T) => number, +): readonly (T | undefined)[] | undefined { + const sorted = [...present].sort(compare); + const choices = anyUntagged ? [...sorted, undefined] : sorted; + return choices.length < 2 ? undefined : choices; +} + +/** + * Collects the choices the rows offer for one field holding a value at a time, or `undefined` when + * the rows are all in one state. */ function singleValueFacet( rows: readonly CatalogRow[], @@ -267,19 +315,31 @@ function singleValueFacet( if (value === undefined) anyUntagged = true; else present.add(value); }); - const sorted = [...present].sort(compare); - const choices = anyUntagged ? [...sorted, undefined] : sorted; - return choices.length < 2 ? undefined : choices; + return valueChoices(present, anyUntagged, compare); } -/** Collects a facet per feature name that earns one, or `undefined` when no name does. */ +/** + * Collects a facet per feature name that earns one, or `undefined` when no name does. A row counts + * as untagged for every name its own feature map omits. + */ function featureFacets(rows: readonly CatalogRow[]): CatalogFacets['features'] { - const names = new Set(rows.flatMap((row) => Object.keys(row.features ?? {}))); - const facets = [...names].reduce>((acc, name) => { - const values = singleValueFacet(rows, (row) => row.features?.[name]); - if (values) acc[name] = values; - return acc; - }, {}); + const tallies = new Map; taggedRows: number }>(); + rows.forEach((row) => { + Object.entries(row.features ?? {}).forEach(([name, value]) => { + const tally = tallies.get(name) ?? { values: new Set(), taggedRows: 0 }; + tally.values.add(value); + tally.taggedRows += 1; + tallies.set(name, tally); + }); + }); + const facets = [...tallies].reduce>( + (acc, [name, { values, taggedRows }]) => { + const choices = valueChoices(values, taggedRows < rows.length); + if (choices) acc[name] = choices; + return acc; + }, + {}, + ); return Object.keys(facets).length === 0 ? undefined : facets; } @@ -400,13 +460,11 @@ export function applyCatalogQuery( rows: readonly CatalogRow[], query: CatalogQuery, ): readonly CatalogRow[] { - // A newline separates the fields a match must not span, so a line ending typed into the query - // reads as an ordinary space instead — including the carriage return a paste from a Windows - // source carries, which no row's search text holds. That and the trim happen here rather than - // inside foldForSearch, which also folds each row's search text, where the separators must - // survive. - const search = foldForSearch(query.search.replace(/\r\n?|\n/g, ' ').trim()); + // A line ending typed or pasted into the query reads as the ordinary space it stands for, rather + // than as the separator a match may not span. Collapsing and trimming happen here rather than + // inside foldForSearch, which also folds each row's search text, where the separators survive. + const search = foldForSearch(collapseLineEndings(query.search).trim()); return rows .filter((row) => row.searchText.includes(search) && passesFilters(row, query.filters)) - .sort((a, b) => compareBySort(a, b, query)); + .toSorted((a, b) => compareBySort(a, b, query)); } diff --git a/src/utils/search-fold.ts b/src/utils/search-fold.ts index 99bbce65..4c861fed 100644 --- a/src/utils/search-fold.ts +++ b/src/utils/search-fold.ts @@ -23,16 +23,22 @@ const FINAL_LETTER_FORMS: Readonly> = { const FINAL_LETTER_PATTERN = new RegExp(`[${Object.keys(FINAL_LETTER_FORMS).join('')}]`, 'gu'); /** - * Folds text down to the form a query is matched against: equal ignoring case, diacritics, and the - * shape a letter takes at its position in a word, so a query typed on an ordinary keyboard finds a - * fully pointed or accented form wherever in a word it sits. Never use it to decide whether two - * analyses are the same. + * Folds text down to the form a query is matched against: equal ignoring case, diacritics, and + * every distinction a compatibility decomposition erases — the shape a letter takes at its position + * in a word, but equally a ligature, a full-width or superscript digit, a circled number, a + * non-breaking space. A query typed on an ordinary keyboard therefore finds a fully pointed or + * accented form wherever in a word it sits. Never use it to decide whether two analyses are the + * same. * - * A positional shape matches the plain letter typed for it whatever script writes it, as do - * ligature and width variants; only {@link FINAL_LETTER_FORMS} needs listing. + * Only the letters {@link FINAL_LETTER_FORMS} lists need folding by hand; every other positional + * shape decomposes to the plain letter typed for it whatever script writes it. * * Some distinctions deliberately survive the fold. A spacing combining mark spells a dependent * vowel, so dropping it would merge words differing only in that vowel; nonspacing marks alone go. + * Where a script writes a vowel or a cluster joiner as a nonspacing mark, as Thai and Devanagari + * do, that costs real distinctions: words differing only in such a mark fold together, and a query + * for one finds the other. + * * And a letter written with a stroke or bar keeps it, matching how the scripts that write one count * it a letter of its own rather than a decorated one. So does a letter whose uppercase is spelled * with two: `ß` stays itself rather than expanding to `ss`, so neither spelling of a German word From 94ff8831990286270bf3788593fb484926c709a8 Mon Sep 17 00:00:00 2001 From: Alex Rawlings Date: Wed, 12 Aug 2026 09:54:29 -0600 Subject: [PATCH 17/18] Take whitespace at the catalog's edges as no text at all A gloss of only whitespace counts as missing, the reading the rest of the analysis layer takes, and the query is trimmed after folding, by which point a spacing diacritic has decomposed to a bare space. --- src/__tests__/utils/analysis-query.test.ts | 22 +++++++++++++++++++ src/utils/analysis-query.ts | 25 +++++++++++++++++----- 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/src/__tests__/utils/analysis-query.test.ts b/src/__tests__/utils/analysis-query.test.ts index d362a0fc..d3302e49 100644 --- a/src/__tests__/utils/analysis-query.test.ts +++ b/src/__tests__/utils/analysis-query.test.ts @@ -261,6 +261,13 @@ describe('applyCatalogQuery search', () => { expect(applyCatalogQuery(rows, makeQuery({ search: ' to be ' }))).toHaveLength(1); }); + // The tonos folds to a bare space, which is whitespace only once the fold has run. + it('ignores a leading character of the query that folds to a space', () => { + const rows = buildCatalogRows(analyzedEimi, scope); + + expect(applyCatalogQuery(rows, makeQuery({ search: '΄to be' }))).toHaveLength(1); + }); + // Both sides fold, so the word-final sigma the text carries meets the ordinary one typed. it('finds a row whose surface form ends in a letter the query spells mid-word', () => { const analysis: TextAnalysis = { @@ -554,6 +561,21 @@ describe('applyCatalogQuery filters', () => { expect(applyCatalogQuery(rows, query).map((r) => r.analysisId)).toEqual(['ta-2']); }); + // A blank gloss reaches storage only in a project file, never from an edit. + it('keeps a row whose stored gloss is only whitespace when filtering for a missing gloss', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'a', gloss: { en: 'word' } }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'b', gloss: { en: ' ' } }, + ], + }; + const rows = buildCatalogRows(analysis, scope); + const query = makeQuery({ filters: { missingGloss: true } }); + + expect(applyCatalogQuery(rows, query).map((r) => r.analysisId)).toEqual(['ta-2']); + }); + it('keeps only rows with a morpheme breakdown when filtering for one', () => { const rows = buildCatalogRows(mixedBreakdowns, scope); const query = makeQuery({ filters: { morphemes: 'has' } }); diff --git a/src/utils/analysis-query.ts b/src/utils/analysis-query.ts index 26a9231d..12b837bf 100644 --- a/src/utils/analysis-query.ts +++ b/src/utils/analysis-query.ts @@ -22,7 +22,10 @@ export interface CatalogScope { export interface CatalogRow { analysisId: string; surfaceText: string; - /** Gloss in the scope's analysis language; `''` when the analysis has none. */ + /** + * Gloss in the scope's analysis language; `''` when the analysis has none, blank counting as + * none. + */ gloss: string; morphemes: readonly MorphemeAnalysis[]; pos?: string; @@ -195,6 +198,16 @@ function buildSearchText(ta: TokenAnalysis): string { return searchText; } +/** + * Reads the gloss a row is listed and filtered by, a blank one reading as no gloss at all — the + * reading the analysis layer takes everywhere else. A write clears a blank gloss rather than + * storing it, so a stored blank is one a project file arrived carrying. + */ +function glossForScope(ta: TokenAnalysis, analysisLanguage: string): string { + const gloss = ta.gloss?.[analysisLanguage] ?? ''; + return gloss.trim() === '' ? '' : gloss; +} + /** * Derives one row per distinct token analysis. Pure in the analysis records: nothing here reads the * tokenized text, so the catalog reports what was recorded rather than whether the text a usage @@ -211,7 +224,7 @@ export function buildCatalogRows( return { analysisId: ta.id, surfaceText: ta.surfaceText, - gloss: ta.gloss?.[scope.analysisLanguage] ?? '', + gloss: glossForScope(ta, scope.analysisLanguage), morphemes: ta.morphemes ?? [], pos: ta.pos, features: ta.features, @@ -461,9 +474,11 @@ export function applyCatalogQuery( query: CatalogQuery, ): readonly CatalogRow[] { // A line ending typed or pasted into the query reads as the ordinary space it stands for, rather - // than as the separator a match may not span. Collapsing and trimming happen here rather than - // inside foldForSearch, which also folds each row's search text, where the separators survive. - const search = foldForSearch(collapseLineEndings(query.search).trim()); + // than as the separator a match may not span, so it is collapsed here rather than inside + // foldForSearch, which also folds each row's search text, where the separators survive. Trimming + // follows the fold rather than preceding it: a spacing diacritic decomposes to a bare space, so + // it is whitespace only once folded. + const search = foldForSearch(collapseLineEndings(query.search)).trim(); return rows .filter((row) => row.searchText.includes(search) && passesFilters(row, query.filters)) .toSorted((a, b) => compareBySort(a, b, query)); From 9defbdfc2c6a548c4bb70e6f425d3bd479d0e573 Mon Sep 17 00:00:00 2001 From: Alex Rawlings Date: Thu, 13 Aug 2026 09:34:55 -0600 Subject: [PATCH 18/18] Read any line break as a space in the catalog's search Only CR and LF collapsed, so a gloss broken with a vertical tab or a line separator was unfindable by a spaced query. Also settles the direction of the final-letter map and narrows the usage-count and fold-cache docs to what they guarantee. --- src/__tests__/utils/analysis-query.test.ts | 54 +++++++++++++++++----- src/utils/analysis-query.ts | 20 +++++--- src/utils/search-fold.ts | 6 +-- 3 files changed, 60 insertions(+), 20 deletions(-) diff --git a/src/__tests__/utils/analysis-query.test.ts b/src/__tests__/utils/analysis-query.test.ts index d3302e49..51bb0a52 100644 --- a/src/__tests__/utils/analysis-query.test.ts +++ b/src/__tests__/utils/analysis-query.test.ts @@ -169,6 +169,27 @@ const analyzedEimi: TextAnalysis = { ], }; +/** + * One analysis holding the same single letter in every field a search reads, so a query for more + * than that letter can only match by reaching across fields — whichever pair it reaches for, and + * whatever order they are assembled in. + */ +const singleLetterFields: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { + ...FIXTURE_STAMPS, + id: 'ta-1', + surfaceText: 'a', + gloss: { en: 'a', fr: 'a' }, + morphemes: [ + { id: 'm-1', form: 'a', writingSystem: 'en', gloss: { en: 'a' } }, + { id: 'm-2', form: 'a', writingSystem: 'en', gloss: { en: 'a' } }, + ], + }, + ], +}; + describe('applyCatalogQuery search', () => { it('finds a row by a gloss in a language other than the active one', () => { const analysis: TextAnalysis = { @@ -187,7 +208,6 @@ describe('applyCatalogQuery search', () => { expect(applyCatalogQuery(rows, makeQuery({ search: 'parole' }))).toHaveLength(1); }); - // The morpheme form is suppletive, so a match cannot come from the surface form instead. it('finds a row by a morpheme form', () => { const rows = buildCatalogRows(analyzedEimi, scope); @@ -207,17 +227,16 @@ describe('applyCatalogQuery search', () => { expect(applyCatalogQuery(rows, makeQuery({ search: 'ην to be' }))).toHaveLength(0); }); - // The two fields are adjacent in the folded text, so only the newline between them is in the way. - it('matches no row on a query carrying the newline that separates two fields', () => { - const analysis: TextAnalysis = { - ...emptyAnalysis(), - tokenAnalyses: [ - { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'λόγος', gloss: { en: 'word' } }, - ], - }; - const rows = buildCatalogRows(analysis, scope); + // The lone letter matching is what makes the misses meaningful: the text is searchable, and only + // the reach across a boundary is not. + it('matches no row on a query reaching past a single field, however it spells the gap', () => { + const rows = buildCatalogRows(singleLetterFields, scope); + const hits = (search: string) => applyCatalogQuery(rows, makeQuery({ search })).length; - expect(applyCatalogQuery(rows, makeQuery({ search: 'λογος\nword' }))).toHaveLength(0); + expect(hits('a')).toBe(1); + expect(hits('aa')).toBe(0); + expect(hits('a a')).toBe(0); + expect(hits('a\na')).toBe(0); }); // The gloss is one field, so the query has to fold to the space inside it rather than to a @@ -241,6 +260,19 @@ describe('applyCatalogQuery search', () => { expect(applyCatalogQuery(rows, makeQuery({ search: 'spoken word' }))).toHaveLength(1); }); + // No keyboard types a vertical tab, so a gloss carrying one arrived as imported text. + it('finds a row by a gloss whose line break is not a newline', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'λόγος', gloss: { en: 'spoken\vword' } }, + ], + }; + const rows = buildCatalogRows(analysis, scope); + + expect(applyCatalogQuery(rows, makeQuery({ search: 'spoken word' }))).toHaveLength(1); + }); + // The marks fold away, leaving the empty query rather than one nothing can match. it('keeps every row on a query made only of combining marks', () => { const analysis: TextAnalysis = { diff --git a/src/utils/analysis-query.ts b/src/utils/analysis-query.ts index 12b837bf..e523611d 100644 --- a/src/utils/analysis-query.ts +++ b/src/utils/analysis-query.ts @@ -142,9 +142,12 @@ function compareDocumentOrder(a: CatalogUsage, b: CatalogUsage): number { * Files each analysis's usages under its id, each list in document order. * * Only an approved link is a usage: a rejected link is by definition not a place the analysis is - * applied. A token counts once however many approved links carry it, so a row's count equals the - * analysis's frequency in the suggestion pool — which counts the tokens an approval sits on rather - * than the approvals themselves — whatever a duplicate link says. + * applied. A token counts once however many approved links carry it to the same analysis, so a + * duplicate link leaves a row's count equal to the analysis's frequency in the suggestion pool, + * which counts the tokens an approval sits on rather than the approvals themselves. The two part + * company only over a token approved to two analyses at once: both rows count it, while the pool + * credits one. No write path builds that state, so it arrives only in imported or hand-edited data, + * and no later write repairs it. */ function groupUsagesByAnalysisId( links: readonly TokenAnalysisLink[], @@ -163,15 +166,20 @@ function groupUsagesByAnalysisId( /** Separates the fields a match may not span, and so the one character no field may hold. */ const FIELD_SEPARATOR = '\n'; -/** Reads a line ending as the ordinary space it stands for between words. */ +/** + * Reads a line ending as the ordinary space it stands for between words, taking every line and + * paragraph break Unicode defines, so a field arriving from another format reads as a typed one + * does. + */ function collapseLineEndings(text: string): string { - return text.replace(/\r\n?|\n/g, ' '); + return text.replace(/\r\n?|[\n\v\f\u0085\u2028\u2029]/g, ' '); } /** * The folded text each analysis is searched by, held against the record it was folded from, so a * write re-folds the one analysis it replaced. A record edited in place rather than replaced would - * keep the fold it arrived with. + * keep the fold it arrived with: the store's own records are frozen and throw on such an edit, so + * the fragility is confined to records assembled outside it. */ const searchTextByAnalysis = new WeakMap(); diff --git a/src/utils/search-fold.ts b/src/utils/search-fold.ts index 4c861fed..9430cd2f 100644 --- a/src/utils/search-fold.ts +++ b/src/utils/search-fold.ts @@ -6,9 +6,9 @@ */ /** - * Letters written with one shape at the end of a word and another shape elsewhere, mapped to the - * shape outside that position. These are the letters no normalization unifies, their scripts - * encoding the two shapes as distinct letters rather than as variants of one. + * Word-final letter shapes, each mapped to the shape its letter takes anywhere else. These are the + * letters no normalization unifies, their scripts encoding the two shapes as distinct letters + * rather than as variants of one. */ const FINAL_LETTER_FORMS: Readonly> = { ς: 'σ',