KIT-4469 fixed added more unit tests

Author: cwehmeier

ai_data_preprocessing_queue/Steps/remove_signature.py

@@ -1,4 +1,5 @@
 import re
+from typing import Any
 
 
 def remove_newline(text: str) -> str:
@@ -40,7 +41,7 @@ def remove_greetings_and_following_text(text: str) -> str:
 THANK_SUFFIXES = [
     r"(?:in advance(?: for (?:your|the) (?:help|support|understanding|assistance))?)",
     r"(?:for (?:your|the) (?:help|support|understanding|assistance))",
-    r"(?:schon mal )?(?:im voraus)?(?: für (?:ihre|ihr|eure|die|den) (?:hilfe|support|verständnis))?",
+    r"(?:schon mal\s+)?(?:im voraus\s+)?für\s+(?:ihre|ihr|eure|die|den)\s+(?:hilfe|support|verständnis)",
     r"vorab",
     r"kindly?"
 ]
@@ -68,11 +69,11 @@ def remove_single_greeting_words(text: str, pattern: str) -> str:
     return re.sub(pattern, " ", text, flags=re.IGNORECASE | re.UNICODE)
 
 
-def step(text: str) -> str:
-    if not text:
-        return text
+def step(item: Any, item_state: dict[str, Any], global_state: dict[str, Any] | None, preprocessor_data: str) -> Any:
+    if not item:
+        return item
     try:
-        text_greetings_removed = remove_greetings_and_following_text(text)
+        text_greetings_removed = remove_greetings_and_following_text(item)
         thankyou_removed = remove_thanking_expressions(text_greetings_removed)
         single_greetings_removed = remove_single_greeting_words(thankyou_removed, single_greetings_pattern)
 

tests/test_remove_signature.py

@@ -2,6 +2,7 @@
 
 from parameterized import parameterized
 
+from ai_data_preprocessing_queue.Pipeline import Pipeline
 from ai_data_preprocessing_queue.Steps.remove_signature import (
     remove_greetings_and_following_text, remove_newline)
 
@@ -87,6 +88,46 @@ def test_remove_newline(self, name: str, input_text: str, expected: str) -> None
     def test_remove_greetings_and_following_text(self, name: str, input_text: str, expected: str) -> None:
         self.assertEqual(remove_greetings_and_following_text(input_text), expected)
 
+    @parameterized.expand([  # type: ignore[misc]
+        (
+            "remove_signature_basic",
+            "We're sending the final draft for review. Best regards, Alice Johnson\nProject Lead",
+            "We're sending the final draft for review.",
+        ),
+        (
+            "thanking_at_start",
+            "Thank you very much for your support. "
+            "I will prepare the contract and send it tomorrow.\n\nBest regards, Bob Brown",
+            "I will prepare the contract and send it tomorrow.",
+        ),
+        (
+            "thanking_in_middle",
+            "Thank you very much for your support. "
+            "I appreciate your support on this migration. Thanks a lot, I will share the logs shortly.",
+            "I appreciate your support on this migration. I will share the logs shortly.",
+        ),
+        (
+            "single_greeting_word_german",
+            "The deliverables are ready. Grüße",
+            "The deliverables are ready.",
+        ),
+        (
+            "german_empty_result",
+            "Vielen Dank für Ihre Hilfe. Mit freundlichen Grüßen, Lena Meyer "
+            "Und hier kommt noch mehr Text.",
+            "",
+        ),
+        (
+            "no_change",
+            "Please schedule the kickoff meeting for next Tuesday morning at 10:00.",
+            "Please schedule the kickoff meeting for next Tuesday morning at 10:00.",
+        ),
+    ])
+    def test_remove_signature_parameterized(self, name: str, input_text: str, expected: str) -> None:
+        pipeline = Pipeline({"remove_signature": None})
+        value = pipeline.consume(input_text)
+        self.assertEqual(expected, value)
+
 
 if __name__ == "__main__":
     unittest.main()