diff --git a/src/__tests__/store/analysisSlice.test.ts b/src/__tests__/store/analysisSlice.test.ts index 6363e0d2..5172341d 100644 --- a/src/__tests__/store/analysisSlice.test.ts +++ b/src/__tests__/store/analysisSlice.test.ts @@ -18,6 +18,7 @@ import { mergePhrases, selectApprovedGloss, selectApprovedMorphemes, + selectCatalogRows, selectMorphemeResetLosesGlosses, selectPhraseLinkByTokenRef, selectPhraseGloss, @@ -1855,6 +1856,37 @@ describe('selectPoolIndex', () => { }); }); +describe('selectCatalogRows', () => { + it('returns the same reference for repeated calls on unchanged state (memoized)', () => { + const store = createAnalysisStore(); + + const a = selectCatalogRows(store.getState().analysis, 'GEN'); + const b = selectCatalogRows(store.getState().analysis, 'GEN'); + + expect(a).toBe(b); + }); + + it('keeps a book cached across a call for another one', () => { + const store = createAnalysisStore(); + + const first = selectCatalogRows(store.getState().analysis, 'GEN'); + selectCatalogRows(store.getState().analysis, 'MAT'); + + expect(selectCatalogRows(store.getState().analysis, 'GEN')).toBe(first); + }); + + it('rebuilds the rows after a gloss write', () => { + const store = createAnalysisStore(); + const before = selectCatalogRows(store.getState().analysis, 'GEN'); + + store.dispatch(writeGloss('GEN 1:1:0', 'λόγος', 'word')); + + const after = selectCatalogRows(store.getState().analysis, 'GEN'); + expect(after).not.toBe(before); + expect(after.map((row) => row.gloss)).toEqual(['word']); + }); +}); + describe('selectResolvedTokenAnalysis', () => { it('returns the approved analysis when the token has one', () => { const ta: TokenAnalysis = { diff --git a/src/__tests__/utils/analysis-identity.test.ts b/src/__tests__/utils/analysis-identity.test.ts index bb1ad959..5584bbc5 100644 --- a/src/__tests__/utils/analysis-identity.test.ts +++ b/src/__tests__/utils/analysis-identity.test.ts @@ -3,6 +3,7 @@ import type { TokenAnalysis } from 'interlinearizer'; import { FIXTURE_STAMPS } from '../test-helpers'; import { analysesAreIdentical, normalizeSurfaceForm } from '../../utils/analysis-identity'; +import { foldForSearch } from '../../utils/search-fold'; describe('normalizeSurfaceForm', () => { it('lowercases so a sentence-initial form matches a mid-sentence form', () => { @@ -15,6 +16,13 @@ describe('normalizeSurfaceForm', () => { expect(decomposed).not.toBe(composed); // distinct raw strings... expect(normalizeSurfaceForm(decomposed)).toBe(normalizeSurfaceForm(composed)); // ...one key once normalized. }); + + // The two normalizations must stay separate: identity under-matches so a recorded distinction is + // never merged away, while the search fold over-matches so a query finds an accented form. + it('keeps a diacritic difference the search fold collapses', () => { + expect(normalizeSurfaceForm('ἀρχῇ')).not.toBe(normalizeSurfaceForm('αρχη')); + expect(foldForSearch('ἀρχῇ')).toBe(foldForSearch('αρχη')); + }); }); /** diff --git a/src/__tests__/utils/analysis-query.test.ts b/src/__tests__/utils/analysis-query.test.ts new file mode 100644 index 00000000..51bb0a52 --- /dev/null +++ b/src/__tests__/utils/analysis-query.test.ts @@ -0,0 +1,834 @@ +/// + +import type { AssignmentStatus, TextAnalysis, TokenAnalysisLink } from 'interlinearizer'; +import { emptyAnalysis } from '../../types/empty-factories'; +import { FIXTURE_STAMPS } from '../test-helpers'; +import { + applyCatalogQuery, + buildCatalogRows, + deriveFacets, + type CatalogQuery, + type CatalogScope, +} from '../../utils/analysis-query'; + +const scope: CatalogScope = { analysisLanguage: 'en', currentBook: 'GEN' }; + +/** Builds a query that neither searches nor filters, so each test varies one dimension. */ +function makeQuery(overrides: Partial = {}): CatalogQuery { + return { + search: '', + sort: 'usageCount', + filters: {}, + surfaceCollator: new Intl.Collator('el'), + glossCollator: new Intl.Collator('en'), + ...overrides, + }; +} + +/** Builds a link from `tokenRef` to the analysis, approved unless another status is given. */ +function link( + analysisId: string, + tokenRef: string, + status: AssignmentStatus = 'approved', +): TokenAnalysisLink { + return { ...FIXTURE_STAMPS, analysisId, status, token: { tokenRef, surfaceText: 'word' } }; +} + +describe('buildCatalogRows', () => { + it('builds one row per token analysis, carrying its id and surface form', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'ἀρχῇ' }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'λόγος' }, + ], + }; + + expect(buildCatalogRows(analysis, scope).map((r) => [r.analysisId, r.surfaceText])).toEqual([ + ['ta-1', 'ἀρχῇ'], + ['ta-2', 'λόγος'], + ]); + }); + + it('counts each approved link on a payload as a usage', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [{ ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'λόγος' }], + tokenAnalysisLinks: [link('ta-1', 'GEN 1:1:0'), link('ta-1', 'GEN 1:2:4')], + }; + + expect(buildCatalogRows(analysis, scope)[0].usageCount).toBe(2); + }); + + // The data model allows a token only one approved analysis, so a second link on it is a + // duplicate rather than a second place the analysis is applied. + it('counts a token carrying the same approval twice as one usage', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [{ ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'λόγος' }], + tokenAnalysisLinks: [link('ta-1', 'GEN 1:1:0'), link('ta-1', 'GEN 1:1:0')], + }; + + expect(buildCatalogRows(analysis, scope)[0].usageCount).toBe(1); + }); + + it('does not count a rejected link as a usage', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [{ ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'λόγος' }], + tokenAnalysisLinks: [link('ta-1', 'GEN 1:1:0', 'rejected')], + }; + + expect(buildCatalogRows(analysis, scope)[0].usageCount).toBe(0); + }); + + it('lists a payload with no links at all as a zero-usage row', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [{ ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'λόγος' }], + }; + + expect(buildCatalogRows(analysis, scope)).toHaveLength(1); + expect(buildCatalogRows(analysis, scope)[0].usageCount).toBe(0); + }); + + it('counts only usages in the scope book toward the per-book count', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [{ ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'λόγος' }], + tokenAnalysisLinks: [ + link('ta-1', 'GEN 1:1:0'), + link('ta-1', 'GEN 1:2:4'), + link('ta-1', 'JHN 1:1:0'), + ], + }; + + const [row] = buildCatalogRows(analysis, scope); + expect(row.usageCount).toBe(3); + expect(row.usageCountInBook).toBe(2); + }); + + it('parses a usage location out of the token ref', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [{ ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'λόγος' }], + tokenAnalysisLinks: [link('ta-1', 'GEN 1:5:12')], + }; + + expect(buildCatalogRows(analysis, scope)[0].usages).toEqual([ + { tokenRef: 'GEN 1:5:12', book: 'GEN', chapter: 1, verse: 5, charStart: 12 }, + ]); + }); + + // A USJ verse sid is verbatim, so a bridged verse reaches the catalog as "GEN 1:3-4". + it('resolves a bridged verse sid to its first verse', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [{ ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'λόγος' }], + tokenAnalysisLinks: [link('ta-1', 'GEN 1:3-4:0')], + }; + + expect(buildCatalogRows(analysis, scope)[0].usages[0].verse).toBe(3); + }); + + // EXO precedes ACT canonically but follows it alphabetically, so the order cannot come from the + // ref strings. + it('orders usages by document position across books', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [{ ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'λόγος' }], + tokenAnalysisLinks: [ + link('ta-1', 'ACT 1:1:0'), + link('ta-1', 'EXO 2:1:0'), + link('ta-1', 'EXO 1:2:5'), + link('ta-1', 'EXO 1:2:0'), + link('ta-1', 'EXO 1:1:0'), + ], + }; + + expect(buildCatalogRows(analysis, scope)[0].usages.map((u) => u.tokenRef)).toEqual([ + 'EXO 1:1:0', + 'EXO 1:2:0', + 'EXO 1:2:5', + 'EXO 2:1:0', + 'ACT 1:1:0', + ]); + }); +}); + +/** An analysis whose morpheme form and gloss share no text with the token's surface form. */ +const analyzedEimi: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { + ...FIXTURE_STAMPS, + id: 'ta-1', + surfaceText: 'ἦν', + morphemes: [{ id: 'm-1', form: 'εἰμί', writingSystem: 'grc', gloss: { en: 'to be' } }], + }, + ], +}; + +/** + * One analysis holding the same single letter in every field a search reads, so a query for more + * than that letter can only match by reaching across fields — whichever pair it reaches for, and + * whatever order they are assembled in. + */ +const singleLetterFields: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { + ...FIXTURE_STAMPS, + id: 'ta-1', + surfaceText: 'a', + gloss: { en: 'a', fr: 'a' }, + morphemes: [ + { id: 'm-1', form: 'a', writingSystem: 'en', gloss: { en: 'a' } }, + { id: 'm-2', form: 'a', writingSystem: 'en', gloss: { en: 'a' } }, + ], + }, + ], +}; + +describe('applyCatalogQuery search', () => { + it('finds a row by a gloss in a language other than the active one', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { + ...FIXTURE_STAMPS, + id: 'ta-1', + surfaceText: 'λόγος', + gloss: { en: 'word', fr: 'parole' }, + }, + ], + }; + const rows = buildCatalogRows(analysis, scope); + + expect(applyCatalogQuery(rows, makeQuery({ search: 'parole' }))).toHaveLength(1); + }); + + it('finds a row by a morpheme form', () => { + const rows = buildCatalogRows(analyzedEimi, scope); + + expect(applyCatalogQuery(rows, makeQuery({ search: 'ειμι' }))).toHaveLength(1); + }); + + it('finds a row by a morpheme gloss', () => { + const rows = buildCatalogRows(analyzedEimi, scope); + + expect(applyCatalogQuery(rows, makeQuery({ search: 'to be' }))).toHaveLength(1); + }); + + // Both words are present, but only as separate fields. + it('matches the whole query as one substring rather than splitting it into terms', () => { + const rows = buildCatalogRows(analyzedEimi, scope); + + expect(applyCatalogQuery(rows, makeQuery({ search: 'ην to be' }))).toHaveLength(0); + }); + + // The lone letter matching is what makes the misses meaningful: the text is searchable, and only + // the reach across a boundary is not. + it('matches no row on a query reaching past a single field, however it spells the gap', () => { + const rows = buildCatalogRows(singleLetterFields, scope); + const hits = (search: string) => applyCatalogQuery(rows, makeQuery({ search })).length; + + expect(hits('a')).toBe(1); + expect(hits('aa')).toBe(0); + expect(hits('a a')).toBe(0); + expect(hits('a\na')).toBe(0); + }); + + // The gloss is one field, so the query has to fold to the space inside it rather than to a + // carriage return no field can hold. + it('finds a row on a query carrying a pasted Windows line ending', () => { + const rows = buildCatalogRows(analyzedEimi, scope); + + expect(applyCatalogQuery(rows, makeQuery({ search: 'to\r\nbe' }))).toHaveLength(1); + }); + + // A gloss holding a line break of its own would otherwise spell the separator no match may span. + it('finds a row by a gloss whose own line break the query spells as a space', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'λόγος', gloss: { en: 'spoken\nword' } }, + ], + }; + const rows = buildCatalogRows(analysis, scope); + + expect(applyCatalogQuery(rows, makeQuery({ search: 'spoken word' }))).toHaveLength(1); + }); + + // No keyboard types a vertical tab, so a gloss carrying one arrived as imported text. + it('finds a row by a gloss whose line break is not a newline', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'λόγος', gloss: { en: 'spoken\vword' } }, + ], + }; + const rows = buildCatalogRows(analysis, scope); + + expect(applyCatalogQuery(rows, makeQuery({ search: 'spoken word' }))).toHaveLength(1); + }); + + // The marks fold away, leaving the empty query rather than one nothing can match. + it('keeps every row on a query made only of combining marks', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'λόγος' }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'ἀρχῇ' }, + ], + }; + const rows = buildCatalogRows(analysis, scope); + + expect(applyCatalogQuery(rows, makeQuery({ search: '\u0301\u0308' }))).toHaveLength(2); + }); + + it('ignores whitespace around the query, which a typed or pasted search carries', () => { + const rows = buildCatalogRows(analyzedEimi, scope); + + expect(applyCatalogQuery(rows, makeQuery({ search: ' to be ' }))).toHaveLength(1); + }); + + // The tonos folds to a bare space, which is whitespace only once the fold has run. + it('ignores a leading character of the query that folds to a space', () => { + const rows = buildCatalogRows(analyzedEimi, scope); + + expect(applyCatalogQuery(rows, makeQuery({ search: '΄to be' }))).toHaveLength(1); + }); + + // Both sides fold, so the word-final sigma the text carries meets the ordinary one typed. + it('finds a row whose surface form ends in a letter the query spells mid-word', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'λόγος', gloss: { en: 'word' } }, + ], + }; + const rows = buildCatalogRows(analysis, scope); + + expect(applyCatalogQuery(rows, makeQuery({ search: 'λογοσ' }))).toHaveLength(1); + }); +}); + +describe('applyCatalogQuery sort', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'a' }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'b' }, + { ...FIXTURE_STAMPS, id: 'ta-3', surfaceText: 'c' }, + ], + tokenAnalysisLinks: [ + link('ta-1', 'GEN 1:1:0'), + link('ta-2', 'GEN 1:2:0'), + link('ta-2', 'GEN 1:3:0'), + link('ta-2', 'GEN 1:4:0'), + link('ta-3', 'GEN 1:5:0'), + link('ta-3', 'GEN 1:6:0'), + ], + }; + + it('orders by usage count, most used first', () => { + const rows = buildCatalogRows(analysis, scope); + + expect(applyCatalogQuery(rows, makeQuery({ sort: 'usageCount' })).map((r) => r.analysisId)) // + .toEqual(['ta-2', 'ta-3', 'ta-1']); + }); + + it('collates surface forms by the source language rather than by code point', () => { + const greek: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'βίβλος' }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'ἀρχή' }, + ], + }; + // Code-point order would leave these as given: the accented alpha sits in the Greek Extended + // block, above beta. + expect('βίβλος' < 'ἀρχή').toBe(true); + const rows = buildCatalogRows(greek, scope); + + expect(applyCatalogQuery(rows, makeQuery({ sort: 'surfaceText' })).map((r) => r.analysisId)) // + .toEqual(['ta-2', 'ta-1']); + }); + + it('orders by usage count in the scope book, most used first', () => { + // ta-1 is the more used analysis overall, ta-2 the more used one in GEN. + const acrossBooks: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'a' }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'b' }, + ], + tokenAnalysisLinks: [ + link('ta-1', 'GEN 1:1:0'), + link('ta-1', 'JHN 1:1:0'), + link('ta-1', 'JHN 1:2:0'), + link('ta-2', 'GEN 1:2:0'), + link('ta-2', 'GEN 1:3:0'), + ], + }; + const rows = buildCatalogRows(acrossBooks, scope); + + expect( + applyCatalogQuery(rows, makeQuery({ sort: 'usageCountInBook' })).map((r) => r.analysisId), + ).toEqual(['ta-2', 'ta-1']); + }); + + // Swedish collates "ä" after "z"; English collates it near "a". Passing the Swedish collator + // proves the query's collator decides, not a default. + it('collates glosses with the query collator', () => { + const glossed: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'a', gloss: { en: 'äpple' } }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'b', gloss: { en: 'zebra' } }, + ], + }; + const rows = buildCatalogRows(glossed, scope); + const sortByGloss = (glossCollator: Intl.Collator) => + applyCatalogQuery(rows, makeQuery({ sort: 'gloss', glossCollator })).map((r) => r.analysisId); + + expect(sortByGloss(new Intl.Collator('sv'))).toEqual(['ta-2', 'ta-1']); + expect(sortByGloss(new Intl.Collator('en'))).toEqual(['ta-1', 'ta-2']); + }); + + // Collating alone would open the list with ta-2: a missing gloss is the empty string, which sorts + // ahead of every word. + it('sorts a row with no gloss in the scope language after every glossed one', () => { + const partlyGlossed: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'a', gloss: { en: 'zebra' } }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'b', gloss: { fr: 'parole' } }, + { ...FIXTURE_STAMPS, id: 'ta-3', surfaceText: 'c', gloss: { en: 'apple' } }, + ], + }; + const rows = buildCatalogRows(partlyGlossed, scope); + + expect(applyCatalogQuery(rows, makeQuery({ sort: 'gloss' })).map((r) => r.analysisId)) // + .toEqual(['ta-3', 'ta-1', 'ta-2']); + }); + + it('orders two rows that both lack a gloss by surface form', () => { + const unglossed: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'β' }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'α' }, + ], + }; + const rows = buildCatalogRows(unglossed, scope); + + expect(applyCatalogQuery(rows, makeQuery({ sort: 'gloss' })).map((r) => r.analysisId)) // + .toEqual(['ta-2', 'ta-1']); + }); + + // ta-1 is the more used analysis, so a count sort would put it first; ta-2 appears earlier. + it('orders by the first usage in document order', () => { + const spread: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'a' }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'b' }, + ], + tokenAnalysisLinks: [ + link('ta-1', 'JHN 1:1:0'), + link('ta-1', 'JHN 1:2:0'), + link('ta-1', 'JHN 1:3:0'), + link('ta-2', 'GEN 1:1:0'), + ], + }; + const rows = buildCatalogRows(spread, scope); + + expect(applyCatalogQuery(rows, makeQuery({ sort: 'firstUsage' })).map((r) => r.analysisId)) // + .toEqual(['ta-2', 'ta-1']); + }); + + it('puts rows with no usages last in first-usage order', () => { + const someUnused: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'a' }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'b' }, + { ...FIXTURE_STAMPS, id: 'ta-3', surfaceText: 'c' }, + { ...FIXTURE_STAMPS, id: 'ta-4', surfaceText: 'd' }, + ], + tokenAnalysisLinks: [ + link('ta-2', 'JHN 1:1:0'), + link('ta-2', 'JHN 1:2:0'), + link('ta-2', 'JHN 1:3:0'), + link('ta-3', 'GEN 1:1:0'), + ], + }; + const rows = buildCatalogRows(someUnused, scope); + + expect(applyCatalogQuery(rows, makeQuery({ sort: 'firstUsage' })).map((r) => r.analysisId)) // + .toEqual(['ta-3', 'ta-2', 'ta-1', 'ta-4']); + }); + + // Every row here is unused, so the count sort separates none of them. + it('orders rows the sort key cannot separate by surface form', () => { + const tied: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'γ' }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'β' }, + { ...FIXTURE_STAMPS, id: 'ta-3', surfaceText: 'α' }, + ], + }; + const rows = buildCatalogRows(tied, scope); + + expect(applyCatalogQuery(rows, makeQuery({ sort: 'usageCount' })).map((r) => r.analysisId)) // + .toEqual(['ta-3', 'ta-2', 'ta-1']); + }); + + it('orders two rows sharing a surface form by gloss', () => { + const homographs: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'α', gloss: { en: 'second' } }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'α', gloss: { en: 'first' } }, + ], + }; + const rows = buildCatalogRows(homographs, scope); + + expect(applyCatalogQuery(rows, makeQuery({ sort: 'usageCount' })).map((r) => r.analysisId)) // + .toEqual(['ta-2', 'ta-1']); + }); +}); + +/** Analyses with and without a morpheme breakdown. */ +const mixedBreakdowns: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { + ...FIXTURE_STAMPS, + id: 'ta-1', + surfaceText: 'a', + morphemes: [{ id: 'm-1', form: 'a', writingSystem: 'grc' }], + }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'b' }, + ], +}; + +/** Analyses tagged across every closed vocabulary, alongside an untagged analysis to filter out. */ +const tagged: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { + ...FIXTURE_STAMPS, + id: 'ta-1', + surfaceText: 'a', + pos: 'noun', + confidence: 'high', + features: { Number: 'Sg', Case: 'Nom' }, + }, + { + ...FIXTURE_STAMPS, + id: 'ta-2', + surfaceText: 'b', + pos: 'verb', + confidence: 'guess', + features: { Number: 'Pl' }, + }, + { ...FIXTURE_STAMPS, id: 'ta-3', surfaceText: 'c' }, + ], +}; + +describe('applyCatalogQuery filters', () => { + it('keeps only rows used in one of the selected books', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'a' }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'b' }, + { ...FIXTURE_STAMPS, id: 'ta-3', surfaceText: 'c' }, + ], + tokenAnalysisLinks: [ + link('ta-1', 'GEN 1:1:0'), + link('ta-2', 'JHN 1:1:0'), + link('ta-3', 'GEN 1:2:0'), + link('ta-3', 'MRK 1:1:0'), + ], + }; + const rows = buildCatalogRows(analysis, scope); + const query = makeQuery({ filters: { books: ['JHN', 'MRK'] } }); + + // The query's default sort puts the more-used ta-3 first. + expect(applyCatalogQuery(rows, query).map((r) => r.analysisId)).toEqual(['ta-3', 'ta-2']); + }); + + it('keeps only unused rows when filtering for zero usages', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'a' }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'b' }, + ], + tokenAnalysisLinks: [link('ta-1', 'GEN 1:1:0')], + }; + const rows = buildCatalogRows(analysis, scope); + const query = makeQuery({ filters: { zeroUsages: true } }); + + expect(applyCatalogQuery(rows, query).map((r) => r.analysisId)).toEqual(['ta-2']); + }); + + // ta-2 is glossed, just not in the scope's language, so an any-language check would drop it. + it('keeps a row glossed only in another language when filtering for a missing gloss', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'a', gloss: { en: 'word' } }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'b', gloss: { fr: 'parole' } }, + ], + }; + const rows = buildCatalogRows(analysis, scope); + const query = makeQuery({ filters: { missingGloss: true } }); + + expect(applyCatalogQuery(rows, query).map((r) => r.analysisId)).toEqual(['ta-2']); + }); + + // A blank gloss reaches storage only in a project file, never from an edit. + it('keeps a row whose stored gloss is only whitespace when filtering for a missing gloss', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'a', gloss: { en: 'word' } }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'b', gloss: { en: ' ' } }, + ], + }; + const rows = buildCatalogRows(analysis, scope); + const query = makeQuery({ filters: { missingGloss: true } }); + + expect(applyCatalogQuery(rows, query).map((r) => r.analysisId)).toEqual(['ta-2']); + }); + + it('keeps only rows with a morpheme breakdown when filtering for one', () => { + const rows = buildCatalogRows(mixedBreakdowns, scope); + const query = makeQuery({ filters: { morphemes: 'has' } }); + + expect(applyCatalogQuery(rows, query).map((r) => r.analysisId)).toEqual(['ta-1']); + }); + + it('keeps only rows without a morpheme breakdown when filtering against one', () => { + const rows = buildCatalogRows(mixedBreakdowns, scope); + const query = makeQuery({ filters: { morphemes: 'lacks' } }); + + expect(applyCatalogQuery(rows, query).map((r) => r.analysisId)).toEqual(['ta-2']); + }); + + it('keeps only rows carrying one of the selected parts of speech', () => { + const rows = buildCatalogRows(tagged, scope); + const query = makeQuery({ filters: { pos: ['noun'] } }); + + expect(applyCatalogQuery(rows, query).map((r) => r.analysisId)).toEqual(['ta-1']); + }); + + it('drops a row carrying no part of speech at all when filtering on one', () => { + const rows = buildCatalogRows(tagged, scope); + const query = makeQuery({ filters: { pos: ['noun', 'verb'] } }); + + expect(applyCatalogQuery(rows, query).map((r) => r.analysisId)).toEqual(['ta-1', 'ta-2']); + }); + + it('keeps only rows carrying no part of speech when that is the selected choice', () => { + const rows = buildCatalogRows(tagged, scope); + const query = makeQuery({ filters: { pos: [undefined] } }); + + expect(applyCatalogQuery(rows, query).map((r) => r.analysisId)).toEqual(['ta-3']); + }); + + it('keeps rows carrying no feature value alongside those carrying a selected one', () => { + const rows = buildCatalogRows(tagged, scope); + const query = makeQuery({ filters: { features: { Case: ['Nom', undefined] } } }); + + expect(applyCatalogQuery(rows, query).map((r) => r.analysisId)).toEqual([ + 'ta-1', + 'ta-2', + 'ta-3', + ]); + }); + + it('keeps only rows carrying one of the selected confidence levels', () => { + const rows = buildCatalogRows(tagged, scope); + const query = makeQuery({ filters: { confidence: ['guess'] } }); + + expect(applyCatalogQuery(rows, query).map((r) => r.analysisId)).toEqual(['ta-2']); + }); + + it('keeps only rows matching the selected value of a feature', () => { + const rows = buildCatalogRows(tagged, scope); + const query = makeQuery({ filters: { features: { Number: ['Sg'] } } }); + + expect(applyCatalogQuery(rows, query).map((r) => r.analysisId)).toEqual(['ta-1']); + }); + + // ta-1 is the only Sg row, and its Case is Nom rather than Acc, so nothing satisfies both. + it('requires a row to match every feature named in the selection', () => { + const rows = buildCatalogRows(tagged, scope); + const query = makeQuery({ filters: { features: { Number: ['Sg'], Case: ['Acc'] } } }); + + expect(applyCatalogQuery(rows, query)).toHaveLength(0); + }); + + it('treats an empty book selection as no filter rather than as an impossible one', () => { + const rows = buildCatalogRows(tagged, scope); + const query = makeQuery({ filters: { books: [] } }); + + expect(applyCatalogQuery(rows, query)).toHaveLength(rows.length); + }); + + it('treats an empty chosen-value selection as no filter', () => { + const rows = buildCatalogRows(tagged, scope); + const query = makeQuery({ filters: { pos: [], confidence: [], features: { Number: [] } } }); + + expect(applyCatalogQuery(rows, query)).toHaveLength(rows.length); + }); +}); + +describe('deriveFacets', () => { + it('offers a book facet once the rows span more than one book', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'a' }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'b' }, + ], + tokenAnalysisLinks: [link('ta-1', 'JHN 1:1:0'), link('ta-2', 'GEN 1:1:0')], + }; + + expect(deriveFacets(buildCatalogRows(analysis, scope)).books).toEqual(['GEN', 'JHN']); + }); + + it('hides the book facet when every row sits in one book', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'a' }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'b' }, + ], + tokenAnalysisLinks: [link('ta-1', 'GEN 1:1:0'), link('ta-2', 'GEN 1:2:0')], + }; + + expect(deriveFacets(buildCatalogRows(analysis, scope)).books).toBeUndefined(); + }); + + it('offers a part-of-speech facet once the rows disagree', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'a', pos: 'noun' }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'b', pos: 'verb' }, + ], + }; + + expect(deriveFacets(buildCatalogRows(analysis, scope)).pos).toEqual(['noun', 'verb']); + }); + + it('offers a confidence facet once the rows disagree', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'a', confidence: 'high' }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'b', confidence: 'guess' }, + ], + }; + + expect(deriveFacets(buildCatalogRows(analysis, scope)).confidence).toEqual(['high', 'guess']); + }); + + // Number varies across the rows; Case does not, so only Number earns a facet. + it('offers a facet per feature name the rows disagree on', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { + ...FIXTURE_STAMPS, + id: 'ta-1', + surfaceText: 'a', + features: { Case: 'Nom', Number: 'Sg' }, + }, + { + ...FIXTURE_STAMPS, + id: 'ta-2', + surfaceText: 'b', + features: { Case: 'Nom', Number: 'Pl' }, + }, + ], + }; + + expect(deriveFacets(buildCatalogRows(analysis, scope)).features).toEqual({ + Number: ['Pl', 'Sg'], + }); + }); + + // ta-3 carries no Number, which is a choice of the facet rather than an omission from it. + it('offers the untagged choice on a feature only some rows carry', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'a', features: { Number: 'Sg' } }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'b', features: { Number: 'Pl' } }, + { ...FIXTURE_STAMPS, id: 'ta-3', surfaceText: 'c' }, + ], + }; + + expect(deriveFacets(buildCatalogRows(analysis, scope)).features).toStrictEqual({ + Number: ['Pl', 'Sg', undefined], + }); + }); + + it('offers a part-of-speech facet where a single row is tagged and the rest are not', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'a', pos: 'noun' }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'b' }, + ], + }; + + expect(deriveFacets(buildCatalogRows(analysis, scope)).pos).toStrictEqual(['noun', undefined]); + }); + + it('hides the part-of-speech facet when every row carries the same one', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'a', pos: 'noun' }, + { ...FIXTURE_STAMPS, id: 'ta-2', surfaceText: 'b', pos: 'noun' }, + ], + }; + + expect(deriveFacets(buildCatalogRows(analysis, scope)).pos).toBeUndefined(); + }); + + // Nothing in the app writes pos, features, or confidence yet, so an analysis shaped the way the + // gloss and morpheme write paths shape it must not raise a dropdown with nothing behind it. + it('derives no facet from fields no write path populates', () => { + const analysis: TextAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [ + { ...FIXTURE_STAMPS, id: 'ta-1', surfaceText: 'ἀρχῇ', gloss: { en: 'beginning' } }, + { + ...FIXTURE_STAMPS, + id: 'ta-2', + surfaceText: 'ἦν', + gloss: { en: 'was' }, + morphemes: [{ id: 'm-1', form: 'εἰμί', writingSystem: 'grc', gloss: { en: 'to be' } }], + }, + ], + tokenAnalysisLinks: [link('ta-1', 'GEN 1:1:0'), link('ta-2', 'JHN 1:1:0')], + }; + + const facets = deriveFacets(buildCatalogRows(analysis, scope)); + expect(facets.pos).toBeUndefined(); + expect(facets.confidence).toBeUndefined(); + expect(facets.features).toBeUndefined(); + expect(facets.books).toEqual(['GEN', 'JHN']); + }); +}); diff --git a/src/__tests__/utils/search-fold.test.ts b/src/__tests__/utils/search-fold.test.ts new file mode 100644 index 00000000..f83edc43 --- /dev/null +++ b/src/__tests__/utils/search-fold.test.ts @@ -0,0 +1,75 @@ +/// + +import { foldForSearch } from '../../utils/search-fold'; + +describe('foldForSearch', () => { + it('folds Greek accents and the iota subscript away', () => { + expect(foldForSearch('ἀρχῇ')).toBe(foldForSearch('αρχη')); + }); + + it('folds Hebrew points away', () => { + expect(foldForSearch('שָׁלוֹם')).toBe(foldForSearch('שלום')); + }); + + it('strips a Latin diacritic rather than transliterating it', () => { + expect(foldForSearch('šālôm')).toBe('salom'); + }); + + it('lowercases so a query matches a capitalized form', () => { + expect(foldForSearch('Ἀρχή')).toBe(foldForSearch('αρχη')); + }); + + it('keeps a spacing combining mark, which spells a vowel rather than decorating one', () => { + const folded = foldForSearch('कि'); + expect(folded).toBe('कि'); + expect(folded).not.toBe(foldForSearch('क')); + }); + + // The virama that binds the conjunct is nonspacing. + it('merges a Devanagari conjunct with the cluster written without its joiner', () => { + expect(foldForSearch('क्ष')).toBe(foldForSearch('कष')); + }); + + // Thai writes this word's vowel and tone as nonspacing marks, leaving the consonant alone. + it('merges Thai words differing only in the marks that spell their vowels', () => { + expect(foldForSearch('ที่')).toBe(foldForSearch('ท')); + }); + + it('folds a Greek final sigma to the shape the letter takes mid-word', () => { + expect(foldForSearch('λόγος')).toBe(foldForSearch('λογοσ')); + }); + + it('folds a Hebrew final form to the shape the letter takes mid-word', () => { + expect(foldForSearch('שָׁלוֹם')).toBe(foldForSearch('שלומ')); + }); + + it('folds a final sigma reached by lowercasing a capitalized form', () => { + expect(foldForSearch('ΛΌΓΟΣ')).toBe(foldForSearch('λογοσ')); + }); + + // Arabic shapes its letters as it renders, so plain text needs no folding; text carrying the + // presentation forms an older encoding spelled out does. + it('folds Arabic presentation forms to the letters typed for them', () => { + expect(foldForSearch('ﺑﻴﺖ')).toBe(foldForSearch('بيت')); + }); + + it('folds a lunate sigma, which stands for the letter at either position', () => { + expect(foldForSearch('ϲοφία')).toBe(foldForSearch('σοφια')); + }); + + it('folds a long s, the word-medial shape an older Latin typography wrote', () => { + expect(foldForSearch('ſanctus')).toBe(foldForSearch('sanctus')); + }); + + it('keeps a stroked letter, which its script counts as a letter of its own', () => { + const folded = foldForSearch('łuk'); + expect(folded).toBe('łuk'); + expect(folded).not.toBe(foldForSearch('luk')); + }); + + it('keeps an eszett distinct from the two letters that spell its uppercase', () => { + const folded = foldForSearch('Straße'); + expect(folded).toBe('straße'); + expect(folded).not.toBe(foldForSearch('strasse')); + }); +}); diff --git a/src/store/analysisSlice.ts b/src/store/analysisSlice.ts index 4fce44e1..426ef2d7 100644 --- a/src/store/analysisSlice.ts +++ b/src/store/analysisSlice.ts @@ -12,6 +12,7 @@ import type { } from 'interlinearizer'; import { emptyAnalysis } from '../types/empty-factories'; import { analysesAreIdentical } from '../utils/analysis-identity'; +import { buildCatalogRows } from '../utils/analysis-query'; import { isEmptyMultiString } from '../utils/multi-string'; import { buildPoolIndex, @@ -1034,6 +1035,25 @@ export const selectPoolIndex = createSelector( buildPoolIndex, ); +/** + * Memoized selector building the Analysis Catalog's rows — one per distinct token analysis, with + * its usage counts and locations — against the book named as the second argument. Only a change to + * the analysis it reads or to the named book rebuilds the rows, so searching and sorting the result + * never does. + * + * Rows are cached per book code asked for rather than in a single slot, so two components reading + * different books do not thrash. Nothing evicts an entry, and the canon bounds how many there can + * be. + */ +export const selectCatalogRows = createSelector( + selectTokenAnalyses, + selectTokenAnalysisLinks, + selectAnalysisLanguage, + (_state: AnalysisState, currentBook: string) => currentBook, + (tokenAnalyses, tokenAnalysisLinks, analysisLanguage, currentBook) => + buildCatalogRows({ tokenAnalyses, tokenAnalysisLinks }, { analysisLanguage, currentBook }), +); + /** * Returns the merged analysis the renderer shows for a token: its approved decision when one * exists, otherwise the engine's suggestion derived live from the approved-analysis pool, or diff --git a/src/utils/analysis-query.ts b/src/utils/analysis-query.ts new file mode 100644 index 00000000..e523611d --- /dev/null +++ b/src/utils/analysis-query.ts @@ -0,0 +1,493 @@ +import { Canon } from '@sillsdev/scripture'; +import type { + Confidence, + MorphemeAnalysis, + TextAnalysis, + TokenAnalysis, + TokenAnalysisLink, +} from 'interlinearizer'; +import { bookOfRef } from './analysis-book'; +import { foldForSearch } from './search-fold'; +import { firstVerseNumber } from './verse-ref'; + +/** The values every row is derived relative to. */ +export interface CatalogScope { + /** BCP 47 tag whose gloss each row displays. */ + analysisLanguage: string; + /** Book code the per-book usage count is taken against. */ + currentBook: string; +} + +/** One distinct token analysis, with the usage data the catalog lists it by. */ +export interface CatalogRow { + analysisId: string; + surfaceText: string; + /** + * Gloss in the scope's analysis language; `''` when the analysis has none, blank counting as + * none. + */ + gloss: string; + morphemes: readonly MorphemeAnalysis[]; + pos?: string; + /** Morphosyntactic features, each feature name mapped to the analysis's value for it. */ + features?: Readonly>; + confidence?: Confidence; + /** Places in the whole draft where the analysis is applied. */ + usageCount: number; + /** Places in the scope's book where the analysis is applied. */ + usageCountInBook: number; + /** Every place the analysis is applied, in document order. */ + usages: readonly CatalogUsage[]; + /** Books the analysis is used in. */ + books: ReadonlySet; + /** Everything a search query is matched against, folded. */ + searchText: string; +} + +/** How the rows are ordered. */ +export type CatalogSort = + 'usageCount' | 'usageCountInBook' | 'surfaceText' | 'gloss' | 'firstUsage'; + +/** + * Which rows are kept. An absent filter is inactive, as is an empty selection. A selection of a + * field an analysis holds one value of accepts `undefined` as the choice for rows carrying no value + * for it, so each such field can be filtered for what it lacks as well as for what it carries. + */ +export interface CatalogFilters { + /** + * Keeps rows used in at least one of these books. A row nothing uses sits in no book, so an + * active selection drops every unused row. + */ + books?: readonly string[]; + /** Keeps rows carrying one of these parts of speech. */ + pos?: readonly (string | undefined)[]; + /** Keeps rows carrying one of these confidence levels. */ + confidence?: readonly (Confidence | undefined)[]; + /** Keeps rows matching every named feature, each against its own accepted values. */ + features?: Readonly>; + /** Keeps only rows nothing uses. Those sit in no book, so a book selection alongside keeps none. */ + zeroUsages?: boolean; + /** Keeps only rows with no gloss in the scope's analysis language. */ + missingGloss?: boolean; + /** Keeps rows according to whether they carry a morpheme breakdown. */ + morphemes?: 'has' | 'lacks'; +} + +/** How the caller narrows and orders the rows. */ +export interface CatalogQuery { + /** + * Matched as a plain substring against each row's folded text, ignoring surrounding whitespace; + * `''` matches everything, as does anything the fold empties — a query of combining marks alone + * leaves the list at its full width rather than emptying it. + */ + search: string; + sort: CatalogSort; + filters: CatalogFilters; + /** Collates surface forms, so ordering follows the source language rather than code points. */ + surfaceCollator: Intl.Collator; + /** Collates glosses, so ordering follows the analysis language rather than code points. */ + glossCollator: Intl.Collator; +} + +/** + * One place in the text where an analysis is applied, read off the token ref alone — the catalog + * never resolves the token itself. + */ +export interface CatalogUsage { + tokenRef: string; + book: string; + chapter: number; + verse: number; + /** Zero-based UTF-16 offset of the token within its segment's baseline text. */ + charStart: number; +} + +/** + * Reads the location a token ref names. A ref is a verse SID plus the token's character offset + * (`"GEN 1:1:0"`), so the whole location is recoverable from the string. The SID's verse portion is + * verbatim USJ, so a bridged verse resolves to the first verse it names. + * + * Every part is taken as the tokenizer wrote it rather than validated: a ref reaching here names a + * token of a tokenized book, never anything a user typed. + */ +function parseUsage(tokenRef: string): CatalogUsage { + const [chapterPart, versePart, charPart] = tokenRef.slice(tokenRef.indexOf(' ') + 1).split(':'); + return { + tokenRef, + book: bookOfRef(tokenRef), + chapter: Number(chapterPart), + /* v8 ignore next -- a sid whose verse portion starts with no digit cannot reach a token ref */ + verse: firstVerseNumber(versePart) ?? 0, + charStart: Number(charPart), + }; +} + +/** + * Orders two usages by document position, taking books in canonical rather than alphabetical order. + * + * Total over the refs a tokenized book produces, whose parts are all present and whose book code is + * canonical. A code outside the canon has no number to be placed by and would lead the list rather + * than trail it. + */ +function compareDocumentOrder(a: CatalogUsage, b: CatalogUsage): number { + return ( + Canon.bookIdToNumber(a.book) - Canon.bookIdToNumber(b.book) || + a.chapter - b.chapter || + a.verse - b.verse || + a.charStart - b.charStart + ); +} + +/** + * Files each analysis's usages under its id, each list in document order. + * + * Only an approved link is a usage: a rejected link is by definition not a place the analysis is + * applied. A token counts once however many approved links carry it to the same analysis, so a + * duplicate link leaves a row's count equal to the analysis's frequency in the suggestion pool, + * which counts the tokens an approval sits on rather than the approvals themselves. The two part + * company only over a token approved to two analyses at once: both rows count it, while the pool + * credits one. No write path builds that state, so it arrives only in imported or hand-edited data, + * and no later write repairs it. + */ +function groupUsagesByAnalysisId( + links: readonly TokenAnalysisLink[], +): ReadonlyMap { + const byId = links.reduce((acc, l) => { + if (l.status !== 'approved') return acc; + const usages = acc.get(l.analysisId) ?? new Map(); + usages.set(l.token.tokenRef, parseUsage(l.token.tokenRef)); + return acc.set(l.analysisId, usages); + }, new Map>()); + return new Map( + [...byId].map(([id, usages]) => [id, [...usages.values()].sort(compareDocumentOrder)]), + ); +} + +/** Separates the fields a match may not span, and so the one character no field may hold. */ +const FIELD_SEPARATOR = '\n'; + +/** + * Reads a line ending as the ordinary space it stands for between words, taking every line and + * paragraph break Unicode defines, so a field arriving from another format reads as a typed one + * does. + */ +function collapseLineEndings(text: string): string { + return text.replace(/\r\n?|[\n\v\f\u0085\u2028\u2029]/g, ' '); +} + +/** + * The folded text each analysis is searched by, held against the record it was folded from, so a + * write re-folds the one analysis it replaced. A record edited in place rather than replaced would + * keep the fold it arrived with: the store's own records are frozen and throw on such an edit, so + * the fragility is confined to records assembled outside it. + */ +const searchTextByAnalysis = new WeakMap(); + +/** + * Assembles the folded text a search matches an analysis by, drawing on its forms and on every + * gloss it carries whatever the language, so a gloss stays findable in a language the project does + * not declare. No match spans two of the assembled fields: a line ending inside a field contributes + * a space, so the separator between fields is the only one in the text. + */ +function buildSearchText(ta: TokenAnalysis): string { + const cached = searchTextByAnalysis.get(ta); + if (cached !== undefined) return cached; + + const morphemeText = (ta.morphemes ?? []).flatMap((m) => [ + m.form, + ...Object.values(m.gloss ?? {}), + ]); + const searchText = foldForSearch( + [ta.surfaceText, ...Object.values(ta.gloss ?? {}), ...morphemeText] + .map(collapseLineEndings) + .join(FIELD_SEPARATOR), + ); + searchTextByAnalysis.set(ta, searchText); + return searchText; +} + +/** + * Reads the gloss a row is listed and filtered by, a blank one reading as no gloss at all — the + * reading the analysis layer takes everywhere else. A write clears a blank gloss rather than + * storing it, so a stored blank is one a project file arrived carrying. + */ +function glossForScope(ta: TokenAnalysis, analysisLanguage: string): string { + const gloss = ta.gloss?.[analysisLanguage] ?? ''; + return gloss.trim() === '' ? '' : gloss; +} + +/** + * Derives one row per distinct token analysis. Pure in the analysis records: nothing here reads the + * tokenized text, so the catalog reports what was recorded rather than whether the text a usage + * points at still says the same thing. + */ +export function buildCatalogRows( + analysis: Pick, + scope: CatalogScope, +): readonly CatalogRow[] { + const usagesByAnalysisId = groupUsagesByAnalysisId(analysis.tokenAnalysisLinks); + + return analysis.tokenAnalyses.map((ta) => { + const usages = usagesByAnalysisId.get(ta.id) ?? []; + return { + analysisId: ta.id, + surfaceText: ta.surfaceText, + gloss: glossForScope(ta, scope.analysisLanguage), + morphemes: ta.morphemes ?? [], + pos: ta.pos, + features: ta.features, + confidence: ta.confidence, + usageCount: usages.length, + usageCountInBook: usages.filter((u) => u.book === scope.currentBook).length, + usages, + books: new Set(usages.map((u) => u.book)), + searchText: buildSearchText(ta), + }; + }); +} + +/** Orders two rows by where each is first applied, an unused analysis coming after every used one. */ +function compareFirstUsage(a: CatalogRow, b: CatalogRow): number { + const [firstA] = a.usages; + const [firstB] = b.usages; + if (!firstA) return firstB ? 1 : 0; + if (!firstB) return -1; + return compareDocumentOrder(firstA, firstB); +} + +/** + * Orders two rows by gloss, an analysis with none in the scope's language coming after every one + * that has one. + */ +function compareGloss(a: CatalogRow, b: CatalogRow, glossCollator: Intl.Collator): number { + if (!a.gloss) return b.gloss ? 1 : 0; + if (!b.gloss) return -1; + return glossCollator.compare(a.gloss, b.gloss); +} + +/** + * The choices the rows offer for each closed-vocabulary field, each field's list offered as a + * filter. + * + * For a field an analysis holds one value of, carrying no value is a choice of its own, listed as + * `undefined` after the values, so a field a single row is tagged with is still offered — that + * choice narrows the list to the tagged row, and its counterpart answers which analyses are still + * missing the field. A facet is absent only when the rows are all in one of these states, since a + * dropdown offering the state everything is already in filters nothing. + * + * Books are listed by value alone: an analysis in no book is one nothing uses, which + * {@link CatalogFilters} keeps on its own terms. A draft confined to a single book therefore offers + * no books facet even where unused rows sit alongside the used ones: the unused are reachable on + * those terms, the used only once a second book is in play. + * + * Assignment status is not a facet: only approved links count as usages, so every row agrees by + * construction. + */ +export interface CatalogFacets { + /** Canonical order. */ + books?: readonly string[]; + pos?: readonly (string | undefined)[]; + confidence?: readonly (Confidence | undefined)[]; + /** Distinct choices per feature name, each name judged on its own. */ + features?: Readonly>; +} + +/** + * Collects the distinct values the rows carry for one field holding several at a time, sorted, or + * `undefined` when fewer than two are present. + */ +function multiValueFacet( + rows: readonly CatalogRow[], + valuesOf: (row: CatalogRow) => Iterable, + compare?: (a: T, b: T) => number, +): readonly T[] | undefined { + const distinct = new Set(rows.flatMap((row) => [...valuesOf(row)])); + return distinct.size < 2 ? undefined : [...distinct].sort(compare); +} + +/** + * Assembles the choices for one field holding a value at a time — the distinct values sorted, then + * `undefined` where the field goes untagged — or `undefined` when fewer than two choices result, a + * lone choice being the state everything is already in. + */ +function valueChoices( + present: ReadonlySet, + anyUntagged: boolean, + compare?: (a: T, b: T) => number, +): readonly (T | undefined)[] | undefined { + const sorted = [...present].sort(compare); + const choices = anyUntagged ? [...sorted, undefined] : sorted; + return choices.length < 2 ? undefined : choices; +} + +/** + * Collects the choices the rows offer for one field holding a value at a time, or `undefined` when + * the rows are all in one state. + */ +function singleValueFacet( + rows: readonly CatalogRow[], + valueOf: (row: CatalogRow) => T | undefined, + compare?: (a: T, b: T) => number, +): readonly (T | undefined)[] | undefined { + const present = new Set(); + let anyUntagged = false; + rows.forEach((row) => { + const value = valueOf(row); + if (value === undefined) anyUntagged = true; + else present.add(value); + }); + return valueChoices(present, anyUntagged, compare); +} + +/** + * Collects a facet per feature name that earns one, or `undefined` when no name does. A row counts + * as untagged for every name its own feature map omits. + */ +function featureFacets(rows: readonly CatalogRow[]): CatalogFacets['features'] { + const tallies = new Map; taggedRows: number }>(); + rows.forEach((row) => { + Object.entries(row.features ?? {}).forEach(([name, value]) => { + const tally = tallies.get(name) ?? { values: new Set(), taggedRows: 0 }; + tally.values.add(value); + tally.taggedRows += 1; + tallies.set(name, tally); + }); + }); + const facets = [...tallies].reduce>( + (acc, [name, { values, taggedRows }]) => { + const choices = valueChoices(values, taggedRows < rows.length); + if (choices) acc[name] = choices; + return acc; + }, + {}, + ); + return Object.keys(facets).length === 0 ? undefined : facets; +} + +/** Confidence levels from strongest to weakest. */ +const CONFIDENCE_ORDER: readonly Confidence[] = ['high', 'medium', 'low', 'guess']; + +/** + * Derives the filter facets worth offering against these rows. + * + * @param rows Every row {@link buildCatalogRows} built, never what {@link applyCatalogQuery} narrowed + * them to: a facet judged against the rows its own selection kept collapses as soon as that + * selection is made, leaving nothing to widen it back by. + */ +export function deriveFacets(rows: readonly CatalogRow[]): CatalogFacets { + return { + books: multiValueFacet( + rows, + (row) => row.books, + (a, b) => Canon.bookIdToNumber(a) - Canon.bookIdToNumber(b), + ), + pos: singleValueFacet(rows, (row) => row.pos), + confidence: singleValueFacet( + rows, + (row) => row.confidence, + (a, b) => CONFIDENCE_ORDER.indexOf(a) - CONFIDENCE_ORDER.indexOf(b), + ), + features: featureFacets(rows), + }; +} + +/** + * Whether a chosen-value selection narrows anything. An empty selection is inactive rather than + * unsatisfiable, so clearing every choice restores the full list. + */ +function isActive(selected: readonly T[] | undefined): selected is readonly T[] { + return selected !== undefined && selected.length > 0; +} + +/** + * Whether a field holding one value at a time is in a state its selection accepts, carrying no + * value being the state the selection's `undefined` choice accepts. + */ +function passesValue( + selected: readonly (T | undefined)[] | undefined, + value: T | undefined, +): boolean { + return !isActive(selected) || selected.includes(value); +} + +/** Whether the row satisfies every feature named in the selection, each judged on its own values. */ +function passesFeatures(row: CatalogRow, selected: CatalogFilters['features']): boolean { + if (!selected) return true; + return Object.entries(selected).every(([name, values]) => + passesValue(values, row.features?.[name]), + ); +} + +/** Whether the row survives every active filter. */ +function passesFilters(row: CatalogRow, filters: CatalogFilters): boolean { + if (isActive(filters.books) && !filters.books.some((book) => row.books.has(book))) return false; + if (!passesValue(filters.pos, row.pos)) return false; + if (!passesValue(filters.confidence, row.confidence)) return false; + if (!passesFeatures(row, filters.features)) return false; + if (filters.zeroUsages && row.usageCount > 0) return false; + if (filters.missingGloss && row.gloss !== '') return false; + if (filters.morphemes) { + const hasMorphemes = row.morphemes.length > 0; + if (hasMorphemes !== (filters.morphemes === 'has')) return false; + } + return true; +} + +/** + * Orders two rows by the query's sort key: counts descending, so the most used lead; text + * ascending. + * + * A key added to {@link CatalogSort} fails to compile here until this orders it too. + */ +function compareBySortKey(a: CatalogRow, b: CatalogRow, query: CatalogQuery): number { + switch (query.sort) { + case 'usageCount': + return b.usageCount - a.usageCount; + case 'usageCountInBook': + return b.usageCountInBook - a.usageCountInBook; + case 'surfaceText': + return query.surfaceCollator.compare(a.surfaceText, b.surfaceText); + case 'gloss': + return compareGloss(a, b, query.glossCollator); + case 'firstUsage': + return compareFirstUsage(a, b); + // no default + } +} + +/** Orders two rows by their form ascending, falling back to their gloss. */ +function compareByText(a: CatalogRow, b: CatalogRow, query: CatalogQuery): number { + return ( + query.surfaceCollator.compare(a.surfaceText, b.surfaceText) || + compareGloss(a, b, query.glossCollator) + ); +} + +/** + * Orders two rows for the listing, falling back to their text where the sort key cannot separate + * them, so the order holds still as unrelated analyses are recorded. + */ +function compareBySort(a: CatalogRow, b: CatalogRow, query: CatalogQuery): number { + return compareBySortKey(a, b, query) || compareByText(a, b, query); +} + +/** + * Narrows rows to those the query's search and filters keep, in the order it asks for. + * + * Separate from {@link buildCatalogRows} so a keystroke re-runs only this pass: the analysis changes + * on every gloss blur, and rebuilding rows per keystroke would re-fold unchanged text. + */ +export function applyCatalogQuery( + rows: readonly CatalogRow[], + query: CatalogQuery, +): readonly CatalogRow[] { + // A line ending typed or pasted into the query reads as the ordinary space it stands for, rather + // than as the separator a match may not span, so it is collapsed here rather than inside + // foldForSearch, which also folds each row's search text, where the separators survive. Trimming + // follows the fold rather than preceding it: a spacing diacritic decomposes to a bare space, so + // it is whitespace only once folded. + const search = foldForSearch(collapseLineEndings(query.search)).trim(); + return rows + .filter((row) => row.searchText.includes(search) && passesFilters(row, query.filters)) + .toSorted((a, b) => compareBySort(a, b, query)); +} diff --git a/src/utils/search-fold.ts b/src/utils/search-fold.ts new file mode 100644 index 00000000..9430cd2f --- /dev/null +++ b/src/utils/search-fold.ts @@ -0,0 +1,58 @@ +/** + * @file The search fold, deliberately not merged with {@link normalizeSurfaceForm} in + * `analysis-identity`. The two look alike enough to invite it, and merging them would corrupt + * data rather than fail loudly: folding a diacritic into identity puts genuinely different words + * on one shared payload, silently dropping a distinction the user recorded. + */ + +/** + * Word-final letter shapes, each mapped to the shape its letter takes anywhere else. These are the + * letters no normalization unifies, their scripts encoding the two shapes as distinct letters + * rather than as variants of one. + */ +const FINAL_LETTER_FORMS: Readonly> = { + ς: 'σ', + ך: 'כ', + ם: 'מ', + ן: 'נ', + ף: 'פ', + ץ: 'צ', +}; + +/** Matches the letters {@link FINAL_LETTER_FORMS} folds. */ +const FINAL_LETTER_PATTERN = new RegExp(`[${Object.keys(FINAL_LETTER_FORMS).join('')}]`, 'gu'); + +/** + * Folds text down to the form a query is matched against: equal ignoring case, diacritics, and + * every distinction a compatibility decomposition erases — the shape a letter takes at its position + * in a word, but equally a ligature, a full-width or superscript digit, a circled number, a + * non-breaking space. A query typed on an ordinary keyboard therefore finds a fully pointed or + * accented form wherever in a word it sits. Never use it to decide whether two analyses are the + * same. + * + * Only the letters {@link FINAL_LETTER_FORMS} lists need folding by hand; every other positional + * shape decomposes to the plain letter typed for it whatever script writes it. + * + * Some distinctions deliberately survive the fold. A spacing combining mark spells a dependent + * vowel, so dropping it would merge words differing only in that vowel; nonspacing marks alone go. + * Where a script writes a vowel or a cluster joiner as a nonspacing mark, as Thai and Devanagari + * do, that costs real distinctions: words differing only in such a mark fold together, and a query + * for one finds the other. + * + * And a letter written with a stroke or bar keeps it, matching how the scripts that write one count + * it a letter of its own rather than a decorated one. So does a letter whose uppercase is spelled + * with two: `ß` stays itself rather than expanding to `ss`, so neither spelling of a German word + * finds the other. + */ +export function foldForSearch(text: string): string { + return ( + text + .normalize('NFKD') + .replace(/\p{Mn}/gu, '') + .normalize('NFC') + // Lowercasing a capitalized word is itself what puts its last letter into a final shape, so + // the fold of those shapes has to follow it. + .toLowerCase() + .replace(FINAL_LETTER_PATTERN, (letter) => FINAL_LETTER_FORMS[letter]) + ); +}