Add repeating character preprocessor

chatterbot/preprocessors.py

@@ -3,7 +3,7 @@
 """
 from chatterbot.conversation import Statement
 from unicodedata import normalize
-from re import sub as re_sub
+from re import sub as re_sub, compile as re_compile
 from html import unescape
 
 
@@ -45,3 +45,37 @@ def convert_to_ascii(statement: Statement) -> Statement:
 
     statement.text = str(text)
     return statement
+
+
+# Matches a single letter that is immediately repeated three or more times.
+# No correctly spelled English word contains a run of that length, so a run
+# of three or more is always an intentional elongation and can be reduced
+# without altering text that was already spelled correctly. Digits,
+# punctuation, and whitespace are excluded from the pattern so that values
+# such as "1000000" or "!!!" are left unchanged.
+_REPEATING_CHARACTER_PATTERN = re_compile(r'([^\W\d_])\1{2,}')
+
+
+def normalize_repeating_characters(statement: Statement) -> Statement:
+    """
+    Reduce runs of three or more repeated letters down to a single letter.
+
+    Elongated words are common in conversational text (for example
+    "I am sooooo happy"). Reducing the repeated characters maps these
+    variations onto the word being elongated ("I am so happy") which helps
+    the chat bot match input against statements it has been trained on.
+
+    Only runs of three or more characters are reduced, so letter pairs that
+    occur naturally (such as the "oo" in "cool") are left untouched, as are
+    repeated digits and punctuation ("1000000" and "!!!").
+
+    Note that a word which genuinely contains a doubled letter is reduced
+    past its correct spelling when it is elongated, so "gooood" becomes
+    "god" rather than "good". Distinguishing the two cases requires a
+    dictionary lookup, which is intentionally outside the scope of a
+    preprocessor; a project that needs that distinction can register its own
+    preprocessor with access to a word list.
+    """
+    statement.text = _REPEATING_CHARACTER_PATTERN.sub(r'\1', statement.text)
+
+    return statement

docs/preprocessors.rst

@@ -28,6 +28,8 @@ ChatterBot comes with several built-in preprocessors.
 
 .. autofunction:: chatterbot.preprocessors.convert_to_ascii
 
+.. autofunction:: chatterbot.preprocessors.normalize_repeating_characters
+
 
 Creating new preprocessors
 ==========================

tests/test_preprocessors.py

@@ -78,3 +78,78 @@ def test_convert_to_ascii(self):
         normal_text = 'Kluft skrams infor pa federal electoral groe'
 
         self.assertEqual(cleaned.text, normal_text)
+
+
+class NormalizeRepeatingCharactersPreprocessorTestCase(ChatBotTestCase):
+    """
+    Make sure that ChatterBot's repeating-character preprocessor works as expected.
+    """
+
+    def test_elongated_word_is_reduced(self):
+        statement = Statement(text='I am sooooo happy')
+        cleaned = preprocessors.normalize_repeating_characters(statement)
+
+        self.assertEqual(cleaned.text, 'I am so happy')
+
+    def test_elongated_word_matches_its_unelongated_form(self):
+        elongated = preprocessors.normalize_repeating_characters(
+            Statement(text='I am sooooo happy')
+        )
+        plain = preprocessors.normalize_repeating_characters(
+            Statement(text='I am so happy')
+        )
+
+        self.assertEqual(elongated.text, plain.text)
+
+    def test_multiple_elongated_words(self):
+        statement = Statement(text='Yesss that was greaaaat')
+        cleaned = preprocessors.normalize_repeating_characters(statement)
+
+        self.assertEqual(cleaned.text, 'Yes that was great')
+
+    def test_case_of_first_character_in_run_is_kept(self):
+        statement = Statement(text='HEYYY there')
+        cleaned = preprocessors.normalize_repeating_characters(statement)
+
+        self.assertEqual(cleaned.text, 'HEY there')
+
+    def test_natural_double_letters_preserved(self):
+        statement = Statement(text='That book looks really cool')
+        cleaned = preprocessors.normalize_repeating_characters(statement)
+
+        self.assertEqual(cleaned.text, 'That book looks really cool')
+
+    def test_non_ascii_letters_are_reduced(self):
+        statement = Statement(text=u'Das ist schööön')
+        cleaned = preprocessors.normalize_repeating_characters(statement)
+
+        self.assertEqual(cleaned.text, u'Das ist schön')
+
+    def test_repeating_digits_preserved(self):
+        statement = Statement(text='I have 1000000 dollars')
+        cleaned = preprocessors.normalize_repeating_characters(statement)
+
+        self.assertEqual(cleaned.text, 'I have 1000000 dollars')
+
+    def test_repeating_punctuation_preserved(self):
+        statement = Statement(text='Wow!!!')
+        cleaned = preprocessors.normalize_repeating_characters(statement)
+
+        self.assertEqual(cleaned.text, 'Wow!!!')
+
+    def test_repeating_whitespace_preserved(self):
+        statement = Statement(text='Hello    there')
+        cleaned = preprocessors.normalize_repeating_characters(statement)
+
+        self.assertEqual(cleaned.text, 'Hello    there')
+
+    def test_elongated_word_containing_a_doubled_letter(self):
+        """
+        A word that genuinely contains a doubled letter is reduced past its
+        correct spelling, since telling the two cases apart would require a
+        dictionary lookup.
+        """
+        statement = Statement(text='That is gooood')
+        cleaned = preprocessors.normalize_repeating_characters(statement)
+
+        self.assertEqual(cleaned.text, 'That is god')