diff --git a/packages/cspell-dictionary/src/SpellingDictionary/SpellingDictionaryFromTrie.test.ts b/packages/cspell-dictionary/src/SpellingDictionary/SpellingDictionaryFromTrie.test.ts index 22b8ebb713e..b79cf95d003 100644 --- a/packages/cspell-dictionary/src/SpellingDictionary/SpellingDictionaryFromTrie.test.ts +++ b/packages/cspell-dictionary/src/SpellingDictionary/SpellingDictionaryFromTrie.test.ts @@ -14,36 +14,35 @@ import { const { outerWordForms } = __testing__; -// cspell:ignore guenstig günstig Bundesstaat Bundeßtaat -// cspell:ignore Goerresstraße, Goerreßtraße, Görresstraße, Görreßtraße Hoehenschnittpunkt Strasse Straße +// cspell:ignore Bundesstaat Goerresstraße Hoehenschnittpunkt Strasse Straße strassen describe('SpellingDictionaryFromTrie', () => { test.each` - word | repMap | expected - ${'hello'} | ${undefined} | ${['hello']} - ${'guenstig'} | ${[['ae', 'ä'], ['oe', 'ö'], ['ue', 'ü'], ['ss', 'ß']]} | ${['guenstig', 'günstig']} - ${'günstig'} | ${[['ae', 'ä'], ['oe', 'ö'], ['ue', 'ü'], ['ss', 'ß']]} | ${['günstig', N('günstig')]} - ${'Bundesstaat'} | ${[['ae', 'ä'], ['oe', 'ö'], ['ue', 'ü'], ['ss', 'ß']]} | ${['Bundesstaat', 'Bundeßtaat']} - ${'Goerresstraße'} | ${[['ae', 'ä'], ['oe', 'ö'], ['ue', 'ü'], ['ss', 'ß']]} | ${['Goerresstraße', 'Goerreßtraße', 'Görresstraße', 'Görreßtraße']} - ${'éa'} | ${[['a', 'b'], ['b', 'c']]} | ${['éa', N('éa'), 'éb', N('éb')]} - `('outerWordForms $word', ({ word, repMap, expected }) => { - const trie = buildITrieFromWords([]); - const dict = new SpellingDictionaryFromTrie(trie, 'test', { repMap }); - expect([...outerWordForms(word, dict.repMapper)]).toEqual([...new Set(expected)]); + word | expected + ${'hello'} | ${['hello']} + ${'éa'} | ${['éa', N('éa')]} + ${N('éa')} | ${[N('éa'), 'éa']} + `('outerWordForms $word', ({ word, expected }) => { + expect([...outerWordForms(word)]).toEqual(expected); }); - test('outerWordForms only maps the starting forms', () => { - const trie = buildITrieFromWords([]); - const repMap: [string, string][] = [ - ['a', 'æ'], - ['a', 'å'], - ]; - const dict = new SpellingDictionaryFromTrie(trie, 'test', { repMap }); - const word = 'é' + 'a'.repeat(12); - const startingForms = 2; - const maxFormsPerMap = 256; - const forms = [...outerWordForms(word, dict.repMapper)]; - expect(forms.length).toBeLessThanOrEqual(startingForms + startingForms * maxFormsPerMap); + test.each` + word | expected + ${'straße'} | ${true} + ${'strasse'} | ${true} + ${'Strasse'} | ${true} + ${'STRASSE'} | ${true} + ${'strassen'} | ${false} + `('has with repMap $word', ({ word, expected }) => { + const trie = buildITrieFromWords(['straße']); + const dict = new SpellingDictionaryFromTrie(trie, 'test', { repMap: [['ss', 'ß']] }); + expect(dict.has(word, { ignoreCase: false })).toBe(expected); + }); + + test('mapWord and remapWord are undefined without a repMap', () => { + const dict = new SpellingDictionaryFromTrie(buildITrieFromWords(['word']), 'test', {}); + expect(dict.mapWord).toBe(undefined); + expect(dict.remapWord).toBe(undefined); }); test('has with German Dictionary to make sure remap works as expected', async () => { diff --git a/packages/cspell-dictionary/src/SpellingDictionary/SpellingDictionaryFromTrie.ts b/packages/cspell-dictionary/src/SpellingDictionary/SpellingDictionaryFromTrie.ts index 6791c3efd44..d5ba5d4fd07 100644 --- a/packages/cspell-dictionary/src/SpellingDictionary/SpellingDictionaryFromTrie.ts +++ b/packages/cspell-dictionary/src/SpellingDictionary/SpellingDictionaryFromTrie.ts @@ -10,8 +10,6 @@ import { CompoundWordsMethod, decodeTrie, suggestionCollector } from 'cspell-tri import { clean } from '../util/clean.js'; import { measurePerf } from '../util/performance.js'; -import type { RepMapper } from '../util/repMap.js'; -import { createMapper, createRepMapper } from '../util/repMap.js'; import * as Defaults from './defaults.js'; import type { FindOptionsRO, @@ -39,7 +37,7 @@ export class SpellingDictionaryFromTrie implements SpellingDictionary { readonly unknownWords: Set = new Set(); readonly mapWord: MapWordSingleFn | undefined; readonly remapWord: MapWordMultipleFn | undefined; - readonly repMapper: RepMapper | undefined; + readonly trie: ITrie; readonly type = 'SpellingDictionaryFromTrie'; readonly isDictionaryCaseSensitive: boolean; readonly containsNoSuggestWords: boolean; @@ -51,17 +49,15 @@ export class SpellingDictionaryFromTrie implements SpellingDictionary { private weightMap: WeightMap | undefined; constructor( - readonly trie: ITrie, + trie: ITrie, readonly name: string, readonly options: SpellingDictionaryOptionsRO, readonly source = 'from trie', size?: number, ) { - const mapWord = createMapper(options.repMap, options.dictionaryInformation?.ignore); - const repMapper = createRepMapper(options.repMap, options.dictionaryInformation?.ignore); - this.mapWord = mapWord?.fn; - this.remapWord = repMapper?.fn; - this.repMapper = repMapper; + this.trie = trie.withReplacements(options.repMap, options.dictionaryInformation?.ignore); + this.mapWord = this.trie.mapWord; + this.remapWord = this.trie.remapWord; this.isDictionaryCaseSensitive = options.caseSensitive ?? true; this.containsNoSuggestWords = options.noSuggest || false; @@ -133,9 +129,7 @@ export class SpellingDictionaryFromTrie implements SpellingDictionary { ignoreCase: boolean, compoundSeparator: string | undefined, ): FindAnyFormResult | undefined { - const outerForms = outerWordForms(word, this.repMapper); - - for (const form of outerForms) { + for (const form of outerWordForms(word)) { const r = this._findAnyForm(form, useCompounds, ignoreCase, compoundSeparator); if (r) return r; } @@ -266,54 +260,17 @@ export function createSpellingDictionaryFromTrieFile( // eslint-disable-next-line no-control-regex const isAsciiRange = /^[\u0000-\u007F]*$/; -function* outerWordForms(word: string, repMapper: RepMapper | undefined): Iterable { - // Only generate the needed forms. - const sent = new Set(); - let w = word; - const ww = w; - yield w; +function* outerWordForms(word: string): Iterable { + yield word; // this function is called for every word lookup, so needs to be efficient. - // Check to see if it is a pure ascii word. - if (!isAsciiRange.test(w)) { - sent.add(w); - w = word.normalize('NFC'); - if (w !== ww) { - yield w; - sent.add(w); - } - w = word.normalize('NFD'); - if (w !== ww && !sent.has(w)) { - yield w; - sent.add(w); - } - } + if (isAsciiRange.test(word)) return; - if (!repMapper) return; - const mapWord = repMapper.fn; + const nfc = word.normalize('NFC'); + if (nfc !== word) yield nfc; - // nothing was added to the set, just do the map. - if (!sent.size) { - if (!repMapper.test.test(ww)) return; - for (const m of mapWord(ww)) { - if (m !== ww && !sent.has(m)) { - yield m; - sent.add(m); - } - } - return; - } - - const mapped = new Set(); - for (const f of sent) { - for (const m of mapWord(f)) { - if (!sent.has(m) && !mapped.has(m)) { - yield m; - mapped.add(m); - } - } - } - return; + const nfd = word.normalize('NFD'); + if (nfd !== word && nfd !== nfc) yield nfd; } export const __testing__: { diff --git a/packages/cspell-trie-lib/api/api.d.ts b/packages/cspell-trie-lib/api/api.d.ts index 9890cd6c0a5..25f56db3eae 100644 --- a/packages/cspell-trie-lib/api/api.d.ts +++ b/packages/cspell-trie-lib/api/api.d.ts @@ -1,4 +1,4 @@ -import { DictionaryDefinitionAugmented, SuggestionCostMapDef } from "@cspell/cspell-types"; +import { DictionaryDefinitionAugmented, ReplaceMap, SuggestionCostMapDef } from "@cspell/cspell-types"; interface BuildOptions { /** * Optimize the trie for size by merging duplicate sub-tries and using a String Table. @@ -468,7 +468,28 @@ interface ITrie { * @returns true if the word was found and is not forbidden. */ hasWord(word: string, caseSensitive: boolean): boolean; + /** + * Find a word. On a trie with replacements, the word's replacement forms are also looked up. + * @param word - the word to search for. + * @param options - how to look up the word. + * @returns the result for the word or the first replacement form found. + */ findWord(word: string, options?: FindWordOptionsRO): FindFullResult$1; + /** + * Get a trie with the same words that also finds words written with replacements. + * @param repMap - the replacements to make to a word when looking it up. + * @param ignoreCharset - characters to remove from a word when looking it up. + * @returns a trie with the replacements, or this trie if there is nothing to replace. + */ + withReplacements(repMap: ReplaceMap | undefined, ignoreCharset?: string): ITrie; + /** + * Replaces each match in a word with one replacement. `undefined` when there is nothing to replace. + */ + readonly mapWord: ((word: string) => string) | undefined; + /** + * Generates the replacement forms of a word, up to a limit. `undefined` when there is nothing to replace. + */ + readonly remapWord: ((word: string) => string[]) | undefined; /** * Determine if a word is in the forbidden word list. * @param word the word to lookup. diff --git a/packages/cspell-trie-lib/src/lib/ITrie.test.ts b/packages/cspell-trie-lib/src/lib/ITrie.test.ts index 9dbb29237ea..4b672363a35 100644 --- a/packages/cspell-trie-lib/src/lib/ITrie.test.ts +++ b/packages/cspell-trie-lib/src/lib/ITrie.test.ts @@ -12,6 +12,66 @@ import { clean } from './utils/clean.ts'; import { normalizeWordToLowercaseAndRemoveAccents } from './utils/normalizeWord.ts'; import { CompoundWordsMethod } from './walker/index.ts'; +describe('withReplacements', () => { + // cspell:ignore strasse straße xssyss + const repMap: [string, string][] = [['ss', 'ß']]; + const trie = ITrieClass.create(['straße', 'blue', 'bell']); + + test.each` + repMap | ignoreCharset + ${undefined} | ${undefined} + ${[]} | ${undefined} + `('nothing to replace $repMap', ({ repMap, ignoreCharset }) => { + const t = trie.withReplacements(repMap, ignoreCharset); + expect(t).toBe(trie); + expect(t.mapWord).toBe(undefined); + expect(t.remapWord).toBe(undefined); + }); + + test('does not change the original trie', () => { + trie.withReplacements(repMap); + expect(trie.findWord('strasse').found).toBe(false); + expect(trie.mapWord).toBe(undefined); + }); + + test.each` + word | options | found + ${'straße'} | ${{ caseSensitive: true }} | ${'straße'} + ${'strasse'} | ${{ caseSensitive: true }} | ${'straße'} + ${'strasse'} | ${{ caseSensitive: false }} | ${'straße'} + ${'STRASSE'} | ${{ caseSensitive: true }} | ${false} + ${'xssyss'} | ${{ caseSensitive: true }} | ${false} + ${'bluebell'} | ${{ caseSensitive: true }} | ${false} + ${'blue'} | ${{ useLegacyWordCompounds: 3 }} | ${'blue'} + ${'bluebell'} | ${{ useLegacyWordCompounds: 3 }} | ${'blue+bell'} + `('findWord $word $options', ({ word, options, found }) => { + expect(trie.withReplacements(repMap).findWord(word, options).found).toBe(found); + }); + + test('has and hasWord use the replacements', () => { + const t = trie.withReplacements(repMap); + expect(t.has('strasse')).toBe(true); + expect(t.hasWord('strasse', true)).toBe(true); + }); + + test('isForbiddenWord does not use the replacements', () => { + const t = ITrieClass.create(['!straße']).withReplacements(repMap); + expect(t.isForbiddenWord('straße')).toBe(true); + expect(t.isForbiddenWord('strasse')).toBe(false); + }); + + test('mapWord and remapWord', () => { + const t = trie.withReplacements(repMap); + expect(t.mapWord?.('strasse')).toBe('straße'); + expect(t.remapWord?.('strasse')).toEqual(['strasse', 'straße']); + }); + + test('ignoreCharset', () => { + const t = ITrieClass.create(['phone']).withReplacements(undefined, '\u0300-\u0308'); + expect(t.findWord('phone\u0301', { caseSensitive: true }).found).toBe('phone'); + }); +}); + describe('Validate Trie Class', () => { const NumSuggestions: SuggestionOptions = { numSuggestions: 10 }; const SEPARATE_WORDS: SuggestionOptions = { compoundMethod: CompoundWordsMethod.SEPARATE_WORDS }; diff --git a/packages/cspell-trie-lib/src/lib/ITrie.ts b/packages/cspell-trie-lib/src/lib/ITrie.ts index d6df83c8a86..90e26eefea1 100644 --- a/packages/cspell-trie-lib/src/lib/ITrie.ts +++ b/packages/cspell-trie-lib/src/lib/ITrie.ts @@ -1,4 +1,5 @@ import { opAppend, opFilter, opMap, pipe } from '@cspell/cspell-pipe/sync'; +import type { ReplaceMap } from '@cspell/cspell-types'; import type { WeightMap } from './distance/index.ts'; import { createFindOptions, findLegacyCompound, findWord, findWordNode, isForbiddenWord } from './ITrieNode/find.ts'; @@ -18,6 +19,8 @@ import type { TrieData } from './TrieData.ts'; import { clean } from './utils/clean.ts'; import { memorizeLastCall } from './utils/memorizeLastCall.ts'; import { mergeOptionalWithDefaults } from './utils/mergeOptionalWithDefaults.ts'; +import type { ReplaceMapper, RepMapper } from './utils/repMap.ts'; +import { createMapper, createRepMapper } from './utils/repMap.ts'; import { replaceAllFactory } from './utils/util.ts'; const defaultLegacyMinCompoundLength = 3; @@ -84,8 +87,32 @@ export interface ITrie { */ hasWord(word: string, caseSensitive: boolean): boolean; + /** + * Find a word. On a trie with replacements, the word's replacement forms are also looked up. + * @param word - the word to search for. + * @param options - how to look up the word. + * @returns the result for the word or the first replacement form found. + */ findWord(word: string, options?: FindWordOptionsRO): FindFullResult; + /** + * Get a trie with the same words that also finds words written with replacements. + * @param repMap - the replacements to make to a word when looking it up. + * @param ignoreCharset - characters to remove from a word when looking it up. + * @returns a trie with the replacements, or this trie if there is nothing to replace. + */ + withReplacements(repMap: ReplaceMap | undefined, ignoreCharset?: string): ITrie; + + /** + * Replaces each match in a word with one replacement. `undefined` when there is nothing to replace. + */ + readonly mapWord: ((word: string) => string) | undefined; + + /** + * Generates the replacement forms of a word, up to a limit. `undefined` when there is nothing to replace. + */ + readonly remapWord: ((word: string) => string[]) | undefined; + /** * Determine if a word is in the forbidden word list. * @param word the word to lookup. @@ -187,9 +214,15 @@ export class ITrieImpl implements ITrie { readonly hasCompoundWords: boolean; readonly hasNonStrictWords: boolean; readonly data: TrieData; + readonly mapWord: ((word: string) => string) | undefined; + readonly remapWord: ((word: string) => string[]) | undefined; + #repMapper: RepMapper | undefined; - constructor(data: TrieData) { + constructor(data: TrieData, mapper?: ReplaceMapper, repMapper?: RepMapper) { this.data = data; + this.mapWord = mapper?.fn; + this.remapWord = repMapper?.fn; + this.#repMapper = repMapper; this.root = data.getRoot(); this._info = mergeOptionalWithDefaults(data.info); this.hasForbiddenWords = data.hasForbiddenWords; @@ -278,6 +311,24 @@ export class ITrieImpl implements ITrie { } findWord(word: string, options?: FindWordOptionsRO): FindFullResult { + const result = this.#findWordAsWritten(word, options); + const repMapper = this.#repMapper; + if (result.found !== false || !repMapper || !repMapper.test.test(word)) return result; + for (const form of repMapper.fn(word)) { + if (form === word) continue; + const mapped = this.#findWordAsWritten(form, options); + if (mapped.found !== false) return mapped; + } + return result; + } + + withReplacements(repMap: ReplaceMap | undefined, ignoreCharset?: string): ITrie { + const mapper = createMapper(repMap, ignoreCharset); + const repMapper = createRepMapper(repMap, ignoreCharset); + return mapper || repMapper ? new ITrieImpl(this.data, mapper, repMapper) : this; + } + + #findWordAsWritten(word: string, options: FindWordOptionsRO | undefined): FindFullResult { if (options?.useLegacyWordCompounds) { const len = options.useLegacyWordCompounds !== true diff --git a/packages/cspell-dictionary/src/util/regexHelper.ts b/packages/cspell-trie-lib/src/lib/utils/regexHelper.ts similarity index 100% rename from packages/cspell-dictionary/src/util/regexHelper.ts rename to packages/cspell-trie-lib/src/lib/utils/regexHelper.ts diff --git a/packages/cspell-dictionary/src/util/repMap.test.ts b/packages/cspell-trie-lib/src/lib/utils/repMap.test.ts similarity index 99% rename from packages/cspell-dictionary/src/util/repMap.test.ts rename to packages/cspell-trie-lib/src/lib/utils/repMap.test.ts index d00f94235fb..3b8dc070b13 100644 --- a/packages/cspell-dictionary/src/util/repMap.test.ts +++ b/packages/cspell-trie-lib/src/lib/utils/repMap.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from 'vitest'; -import { __testing__, createMapper, createRepMapper } from './repMap.js'; +import { __testing__, createMapper, createRepMapper } from './repMap.ts'; const { createMapperRegExp, charsetToRepMap, createTrie, calcAllEdits, applyEdits } = __testing__; diff --git a/packages/cspell-dictionary/src/util/repMap.ts b/packages/cspell-trie-lib/src/lib/utils/repMap.ts similarity index 98% rename from packages/cspell-dictionary/src/util/repMap.ts rename to packages/cspell-trie-lib/src/lib/utils/repMap.ts index c5851cd5669..2bc0a1e2140 100644 --- a/packages/cspell-dictionary/src/util/repMap.ts +++ b/packages/cspell-trie-lib/src/lib/utils/repMap.ts @@ -1,8 +1,8 @@ import type { CharacterSet, ReplaceEntry, ReplaceMap } from '@cspell/cspell-types'; -import { expandCharacterSet } from 'cspell-trie-lib'; -import { escapeRegEx } from './regexHelper.js'; -import { isDefined } from './util.js'; +import { isDefined } from './isDefined.ts'; +import { escapeRegEx } from './regexHelper.ts'; +import { expandCharacterSet } from './text.ts'; export interface ReplaceMapper { test?: RegExp;