refactor: Move repMap from cspell-dictionary to cspell-trie-lib

packages/cspell-dictionary/src/SpellingDictionary/SpellingDictionaryFromTrie.test.ts

@@ -14,36 +14,35 @@ import {
 
 const { outerWordForms } = __testing__;
 
-// cspell:ignore guenstig günstig Bundesstaat Bundeßtaat
-// cspell:ignore Goerresstraße, Goerreßtraße, Görresstraße, Görreßtraße Hoehenschnittpunkt Strasse Straße
+// cspell:ignore Bundesstaat Goerresstraße Hoehenschnittpunkt Strasse Straße strassen
 
 describe('SpellingDictionaryFromTrie', () => {
     test.each`
-        word               | repMap                                                  | expected
-        ${'hello'}         | ${undefined}                                            | ${['hello']}
-        ${'guenstig'}      | ${[['ae', 'ä'], ['oe', 'ö'], ['ue', 'ü'], ['ss', 'ß']]} | ${['guenstig', 'günstig']}
-        ${'günstig'}       | ${[['ae', 'ä'], ['oe', 'ö'], ['ue', 'ü'], ['ss', 'ß']]} | ${['günstig', N('günstig')]}
-        ${'Bundesstaat'}   | ${[['ae', 'ä'], ['oe', 'ö'], ['ue', 'ü'], ['ss', 'ß']]} | ${['Bundesstaat', 'Bundeßtaat']}
-        ${'Goerresstraße'} | ${[['ae', 'ä'], ['oe', 'ö'], ['ue', 'ü'], ['ss', 'ß']]} | ${['Goerresstraße', 'Goerreßtraße', 'Görresstraße', 'Görreßtraße']}
-        ${'éa'}            | ${[['a', 'b'], ['b', 'c']]}                             | ${['éa', N('éa'), 'éb', N('éb')]}
-    `('outerWordForms $word', ({ word, repMap, expected }) => {
-        const trie = buildITrieFromWords([]);
-        const dict = new SpellingDictionaryFromTrie(trie, 'test', { repMap });
-        expect([...outerWordForms(word, dict.repMapper)]).toEqual([...new Set(expected)]);
+        word       | expected
+        ${'hello'} | ${['hello']}
+        ${'éa'}    | ${['éa', N('éa')]}
+        ${N('éa')} | ${[N('éa'), 'éa']}
+    `('outerWordForms $word', ({ word, expected }) => {
+        expect([...outerWordForms(word)]).toEqual(expected);
     });
 
-    test('outerWordForms only maps the starting forms', () => {
-        const trie = buildITrieFromWords([]);
-        const repMap: [string, string][] = [
-            ['a', 'æ'],
-            ['a', 'å'],
-        ];
-        const dict = new SpellingDictionaryFromTrie(trie, 'test', { repMap });
-        const word = 'é' + 'a'.repeat(12);
-        const startingForms = 2;
-        const maxFormsPerMap = 256;
-        const forms = [...outerWordForms(word, dict.repMapper)];
-        expect(forms.length).toBeLessThanOrEqual(startingForms + startingForms * maxFormsPerMap);
+    test.each`
+        word          | expected
+        ${'straße'}   | ${true}
+        ${'strasse'}  | ${true}
+        ${'Strasse'}  | ${true}
+        ${'STRASSE'}  | ${true}
+        ${'strassen'} | ${false}
+    `('has with repMap $word', ({ word, expected }) => {
+        const trie = buildITrieFromWords(['straße']);
+        const dict = new SpellingDictionaryFromTrie(trie, 'test', { repMap: [['ss', 'ß']] });
+        expect(dict.has(word, { ignoreCase: false })).toBe(expected);
+    });
+
+    test('mapWord and remapWord are undefined without a repMap', () => {
+        const dict = new SpellingDictionaryFromTrie(buildITrieFromWords(['word']), 'test', {});
+        expect(dict.mapWord).toBe(undefined);
+        expect(dict.remapWord).toBe(undefined);
     });
 
     test('has with German Dictionary to make sure remap works as expected', async () => {

packages/cspell-dictionary/src/SpellingDictionary/SpellingDictionaryFromTrie.ts

@@ -10,8 +10,6 @@ import { CompoundWordsMethod, decodeTrie, suggestionCollector } from 'cspell-tri
 
 import { clean } from '../util/clean.js';
 import { measurePerf } from '../util/performance.js';
-import type { RepMapper } from '../util/repMap.js';
-import { createMapper, createRepMapper } from '../util/repMap.js';
 import * as Defaults from './defaults.js';
 import type {
     FindOptionsRO,
@@ -39,7 +37,7 @@ export class SpellingDictionaryFromTrie implements SpellingDictionary {
     readonly unknownWords: Set<string> = new Set<string>();
     readonly mapWord: MapWordSingleFn | undefined;
     readonly remapWord: MapWordMultipleFn | undefined;
-    readonly repMapper: RepMapper | undefined;
+    readonly trie: ITrie;
     readonly type = 'SpellingDictionaryFromTrie';
     readonly isDictionaryCaseSensitive: boolean;
     readonly containsNoSuggestWords: boolean;
@@ -51,17 +49,15 @@ export class SpellingDictionaryFromTrie implements SpellingDictionary {
     private weightMap: WeightMap | undefined;
 
     constructor(
-        readonly trie: ITrie,
+        trie: ITrie,
         readonly name: string,
         readonly options: SpellingDictionaryOptionsRO,
         readonly source = 'from trie',
         size?: number,
     ) {
-        const mapWord = createMapper(options.repMap, options.dictionaryInformation?.ignore);
-        const repMapper = createRepMapper(options.repMap, options.dictionaryInformation?.ignore);
-        this.mapWord = mapWord?.fn;
-        this.remapWord = repMapper?.fn;
-        this.repMapper = repMapper;
+        this.trie = trie.withReplacements(options.repMap, options.dictionaryInformation?.ignore);
+        this.mapWord = this.trie.mapWord;
+        this.remapWord = this.trie.remapWord;
 
         this.isDictionaryCaseSensitive = options.caseSensitive ?? true;
         this.containsNoSuggestWords = options.noSuggest || false;
@@ -133,9 +129,7 @@ export class SpellingDictionaryFromTrie implements SpellingDictionary {
         ignoreCase: boolean,
         compoundSeparator: string | undefined,
     ): FindAnyFormResult | undefined {
-        const outerForms = outerWordForms(word, this.repMapper);
-
-        for (const form of outerForms) {
+        for (const form of outerWordForms(word)) {
             const r = this._findAnyForm(form, useCompounds, ignoreCase, compoundSeparator);
             if (r) return r;
         }
@@ -266,54 +260,17 @@ export function createSpellingDictionaryFromTrieFile(
 // eslint-disable-next-line no-control-regex
 const isAsciiRange = /^[\u0000-\u007F]*$/;
 
-function* outerWordForms(word: string, repMapper: RepMapper | undefined): Iterable<string> {
-    // Only generate the needed forms.
-    const sent = new Set<string>();
-    let w = word;
-    const ww = w;
-    yield w;
+function* outerWordForms(word: string): Iterable<string> {
+    yield word;
 
     // this function is called for every word lookup, so needs to be efficient.
-    // Check to see if it is a pure ascii word.
-    if (!isAsciiRange.test(w)) {
-        sent.add(w);
-        w = word.normalize('NFC');
-        if (w !== ww) {
-            yield w;
-            sent.add(w);
-        }
-        w = word.normalize('NFD');
-        if (w !== ww && !sent.has(w)) {
-            yield w;
-            sent.add(w);
-        }
-    }
+    if (isAsciiRange.test(word)) return;
 
-    if (!repMapper) return;
-    const mapWord = repMapper.fn;
+    const nfc = word.normalize('NFC');
+    if (nfc !== word) yield nfc;
 
-    // nothing was added to the set, just do the map.
-    if (!sent.size) {
-        if (!repMapper.test.test(ww)) return;
-        for (const m of mapWord(ww)) {
-            if (m !== ww && !sent.has(m)) {
-                yield m;
-                sent.add(m);
-            }
-        }
-        return;
-    }
-
-    const mapped = new Set<string>();
-    for (const f of sent) {
-        for (const m of mapWord(f)) {
-            if (!sent.has(m) && !mapped.has(m)) {
-                yield m;
-                mapped.add(m);
-            }
-        }
-    }
-    return;
+    const nfd = word.normalize('NFD');
+    if (nfd !== word && nfd !== nfc) yield nfd;
 }
 
 export const __testing__: {

packages/cspell-trie-lib/src/lib/ITrie.test.ts

@@ -12,6 +12,66 @@ import { clean } from './utils/clean.ts';
 import { normalizeWordToLowercaseAndRemoveAccents } from './utils/normalizeWord.ts';
 import { CompoundWordsMethod } from './walker/index.ts';
 
+describe('withReplacements', () => {
+    // cspell:ignore strasse straße xssyss
+    const repMap: [string, string][] = [['ss', 'ß']];
+    const trie = ITrieClass.create(['straße', 'blue', 'bell']);
+
+    test.each`
+        repMap       | ignoreCharset
+        ${undefined} | ${undefined}
+        ${[]}        | ${undefined}
+    `('nothing to replace $repMap', ({ repMap, ignoreCharset }) => {
+        const t = trie.withReplacements(repMap, ignoreCharset);
+        expect(t).toBe(trie);
+        expect(t.mapWord).toBe(undefined);
+        expect(t.remapWord).toBe(undefined);
+    });
+
+    test('does not change the original trie', () => {
+        trie.withReplacements(repMap);
+        expect(trie.findWord('strasse').found).toBe(false);
+        expect(trie.mapWord).toBe(undefined);
+    });
+
+    test.each`
+        word          | options                          | found
+        ${'straße'}   | ${{ caseSensitive: true }}       | ${'straße'}
+        ${'strasse'}  | ${{ caseSensitive: true }}       | ${'straße'}
+        ${'strasse'}  | ${{ caseSensitive: false }}      | ${'straße'}
+        ${'STRASSE'}  | ${{ caseSensitive: true }}       | ${false}
+        ${'xssyss'}   | ${{ caseSensitive: true }}       | ${false}
+        ${'bluebell'} | ${{ caseSensitive: true }}       | ${false}
+        ${'blue'}     | ${{ useLegacyWordCompounds: 3 }} | ${'blue'}
+        ${'bluebell'} | ${{ useLegacyWordCompounds: 3 }} | ${'blue+bell'}
+    `('findWord $word $options', ({ word, options, found }) => {
+        expect(trie.withReplacements(repMap).findWord(word, options).found).toBe(found);
+    });
+
+    test('has and hasWord use the replacements', () => {
+        const t = trie.withReplacements(repMap);
+        expect(t.has('strasse')).toBe(true);
+        expect(t.hasWord('strasse', true)).toBe(true);
+    });
+
+    test('isForbiddenWord does not use the replacements', () => {
+        const t = ITrieClass.create(['!straße']).withReplacements(repMap);
+        expect(t.isForbiddenWord('straße')).toBe(true);
+        expect(t.isForbiddenWord('strasse')).toBe(false);
+    });
+
+    test('mapWord and remapWord', () => {
+        const t = trie.withReplacements(repMap);
+        expect(t.mapWord?.('strasse')).toBe('straße');
+        expect(t.remapWord?.('strasse')).toEqual(['strasse', 'straße']);
+    });
+
+    test('ignoreCharset', () => {
+        const t = ITrieClass.create(['phone']).withReplacements(undefined, '\u0300-\u0308');
+        expect(t.findWord('phone\u0301', { caseSensitive: true }).found).toBe('phone');
+    });
+});
+
 describe('Validate Trie Class', () => {
     const NumSuggestions: SuggestionOptions = { numSuggestions: 10 };
     const SEPARATE_WORDS: SuggestionOptions = { compoundMethod: CompoundWordsMethod.SEPARATE_WORDS };

packages/cspell-trie-lib/src/lib/ITrie.ts

@@ -1,4 +1,5 @@
 import { opAppend, opFilter, opMap, pipe } from '@cspell/cspell-pipe/sync';
+import type { ReplaceMap } from '@cspell/cspell-types';
 
 import type { WeightMap } from './distance/index.ts';
 import { createFindOptions, findLegacyCompound, findWord, findWordNode, isForbiddenWord } from './ITrieNode/find.ts';
@@ -18,6 +19,8 @@ import type { TrieData } from './TrieData.ts';
 import { clean } from './utils/clean.ts';
 import { memorizeLastCall } from './utils/memorizeLastCall.ts';
 import { mergeOptionalWithDefaults } from './utils/mergeOptionalWithDefaults.ts';
+import type { ReplaceMapper, RepMapper } from './utils/repMap.ts';
+import { createMapper, createRepMapper } from './utils/repMap.ts';
 import { replaceAllFactory } from './utils/util.ts';
 
 const defaultLegacyMinCompoundLength = 3;
@@ -84,8 +87,32 @@ export interface ITrie {
      */
     hasWord(word: string, caseSensitive: boolean): boolean;
 
+    /**
+     * Find a word. On a trie with replacements, the word's replacement forms are also looked up.
+     * @param word - the word to search for.
+     * @param options - how to look up the word.
+     * @returns the result for the word or the first replacement form found.
+     */
     findWord(word: string, options?: FindWordOptionsRO): FindFullResult;
 
+    /**
+     * Get a trie with the same words that also finds words written with replacements.
+     * @param repMap - the replacements to make to a word when looking it up.
+     * @param ignoreCharset - characters to remove from a word when looking it up.
+     * @returns a trie with the replacements, or this trie if there is nothing to replace.
+     */
+    withReplacements(repMap: ReplaceMap | undefined, ignoreCharset?: string): ITrie;
+
+    /**
+     * Replaces each match in a word with one replacement. `undefined` when there is nothing to replace.
+     */
+    readonly mapWord: ((word: string) => string) | undefined;
+
+    /**
+     * Generates the replacement forms of a word, up to a limit. `undefined` when there is nothing to replace.
+     */
+    readonly remapWord: ((word: string) => string[]) | undefined;
+
     /**
      * Determine if a word is in the forbidden word list.
      * @param word the word to lookup.
@@ -187,9 +214,15 @@ export class ITrieImpl implements ITrie {
     readonly hasCompoundWords: boolean;
     readonly hasNonStrictWords: boolean;
     readonly data: TrieData;
+    readonly mapWord: ((word: string) => string) | undefined;
+    readonly remapWord: ((word: string) => string[]) | undefined;
+    #repMapper: RepMapper | undefined;
 
-    constructor(data: TrieData) {
+    constructor(data: TrieData, mapper?: ReplaceMapper, repMapper?: RepMapper) {
         this.data = data;
+        this.mapWord = mapper?.fn;
+        this.remapWord = repMapper?.fn;
+        this.#repMapper = repMapper;
         this.root = data.getRoot();
         this._info = mergeOptionalWithDefaults(data.info);
         this.hasForbiddenWords = data.hasForbiddenWords;
@@ -278,6 +311,24 @@ export class ITrieImpl implements ITrie {
     }
 
     findWord(word: string, options?: FindWordOptionsRO): FindFullResult {
+        const result = this.#findWordAsWritten(word, options);
+        const repMapper = this.#repMapper;
+        if (result.found !== false || !repMapper || !repMapper.test.test(word)) return result;
+        for (const form of repMapper.fn(word)) {
+            if (form === word) continue;
+            const mapped = this.#findWordAsWritten(form, options);
+            if (mapped.found !== false) return mapped;
+        }
+        return result;
+    }
+
+    withReplacements(repMap: ReplaceMap | undefined, ignoreCharset?: string): ITrie {
+        const mapper = createMapper(repMap, ignoreCharset);
+        const repMapper = createRepMapper(repMap, ignoreCharset);
+        return mapper || repMapper ? new ITrieImpl(this.data, mapper, repMapper) : this;
+    }
+
+    #findWordAsWritten(word: string, options: FindWordOptionsRO | undefined): FindFullResult {
         if (options?.useLegacyWordCompounds) {
             const len =
                 options.useLegacyWordCompounds !== true

packages/cspell-dictionary/src/util/repMap.test.ts → packages/cspell-trie-lib/src/lib/utils/repMap.test.ts

@@ -1,6 +1,6 @@
 import { describe, expect, test } from 'vitest';
 
-import { __testing__, createMapper, createRepMapper } from './repMap.js';
+import { __testing__, createMapper, createRepMapper } from './repMap.ts';
 
 const { createMapperRegExp, charsetToRepMap, createTrie, calcAllEdits, applyEdits } = __testing__;
 

packages/cspell-dictionary/src/util/repMap.ts → packages/cspell-trie-lib/src/lib/utils/repMap.ts

@@ -1,8 +1,8 @@
 import type { CharacterSet, ReplaceEntry, ReplaceMap } from '@cspell/cspell-types';
-import { expandCharacterSet } from 'cspell-trie-lib';
 
-import { escapeRegEx } from './regexHelper.js';
-import { isDefined } from './util.js';
+import { isDefined } from './isDefined.ts';
+import { escapeRegEx } from './regexHelper.ts';
+import { expandCharacterSet } from './text.ts';
 
 export interface ReplaceMapper {
     test?: RegExp;