KIT-4745 fixed linting issues

Author: cwehmeier

ai_data_preprocessing_queue/Steps/language_detect.py

@@ -1,11 +1,11 @@
-"""
-Detects one of the following languages and writes the language to local state.
+"""Detects one of the following languages and writes the language to local state.
 
 af, ar, bg, bn, ca, cs, cy, da, de, el, en, es, et, fa, fi, fr, gu, he,
 hi, hr, hu, id, it, ja, kn, ko, lt, lv, mk, ml, mr, ne, nl, no, pa, pl,
 pt, ro, ru, sk, sl, so, sq, sv, sw, ta, te, th, tl, tr, uk, ur, vi,
 zh-cn, zh-tw
 """
+
 from typing import Any
 
 from langdetect import detect

ai_data_preprocessing_queue/Steps/remove_numbers.py

@@ -3,5 +3,4 @@
 
 
 def step(item: Any, item_state: dict[str, Any], global_state: dict[str, Any] | None, preprocessor_data: str) -> Any:
-    item = re.sub(r"""\d""", " ", item)
-    return item
+    return re.sub(r"""\d""", " ", item)

ai_data_preprocessing_queue/Steps/remove_punctuation.py

@@ -3,5 +3,4 @@
 
 
 def step(item: Any, item_state: dict[str, Any], global_state: dict[str, Any] | None, preprocessor_data: str) -> Any:
-    item = re.sub(r"[^\w\s]", " ", item)
-    return item
+    return re.sub(r"[^\w\s]", " ", item)

ai_data_preprocessing_queue/Steps/remove_signature.py

@@ -6,15 +6,30 @@ def remove_newline(text: str) -> str:
     """Remove excessive newlines or spaces from the text."""
     pattern = re.compile(r"\s{2,}|[\n\r]{3,}")
     result = pattern.sub(" ", text)
-    result = re.sub(r"\s+", " ", result).strip()
-
-    return result
-
-
-GreetingExpressions = ["sincerely", "best regards", "happy holidays", "kind regards", "warm regards", "cheers",
-                       "regards", "mit freundlichen grüßen", "freundliche grüße", "beste grüße", "viele grüße",
-                       "herzliche grüße", "liebe grüße", "mit freundlichen grüssen", "freundliche grüsse",
-                       "beste grüsse", "viele grüsse", "herzliche grüsse", "liebe grüsse"]
+    return re.sub(r"\s+", " ", result).strip()
+
+
+GreetingExpressions = [
+    "sincerely",
+    "best regards",
+    "happy holidays",
+    "kind regards",
+    "warm regards",
+    "cheers",
+    "regards",
+    "mit freundlichen grüßen",
+    "freundliche grüße",
+    "beste grüße",
+    "viele grüße",
+    "herzliche grüße",
+    "liebe grüße",
+    "mit freundlichen grüssen",
+    "freundliche grüsse",
+    "beste grüsse",
+    "viele grüsse",
+    "herzliche grüsse",
+    "liebe grüsse",
+]
 greetings_regex = r"(" + "|".join(GreetingExpressions) + r")\s*,?\s*"
 
 
@@ -26,15 +41,15 @@ def remove_greetings_and_following_text(text: str) -> str:
 # thank you expressions should be removed after greetings and following signature text,
 # as they often appear at the beginning of a message
 THANK_EXPRESSIONS = [
-    r"thank you(?: very much)?",   # thank you, thank you very much
-    r"thankyou(?: very much)?",   # thankyou, thankyou very much
-    r"thanks(?: a lot| again)?",   # thanks, thanks a lot, thanks again
-    r"many thanks",                # many thanks
-    r"a thousand thanks",          # a thousand thanks
-    r"danke(?: schön)?",           # danke, danke schön, danke und
-    r"vielen dank",                # vielen dank
-    r"dankeschön",                 # dankeschön
-    r"besten dank"                 # besten dank
+    r"thank you(?: very much)?",  # thank you, thank you very much
+    r"thankyou(?: very much)?",  # thankyou, thankyou very much
+    r"thanks(?: a lot| again)?",  # thanks, thanks a lot, thanks again
+    r"many thanks",  # many thanks
+    r"a thousand thanks",  # a thousand thanks
+    r"danke(?: schön)?",  # danke, danke schön, danke und
+    r"vielen dank",  # vielen dank
+    r"dankeschön",  # dankeschön
+    r"besten dank",  # besten dank
 ]
 
 # Suffixes which could follow thank you expressions
@@ -43,15 +58,13 @@ def remove_greetings_and_following_text(text: str) -> str:
     r"(?:for (?:your|the) (?:help|support|understanding|assistance))",
     r"(?:schon mal\s+)?(?:im voraus\s+)?für\s+(?:ihre|ihr|eure|die|den)\s+(?:hilfe|support|verständnis)",
     r"vorab",
-    r"kindly?"
+    r"kindly?",
 ]
 
 # Combine them into a final regex pattern and compile
 thank_expressions = r"|".join(THANK_EXPRESSIONS)
 suffixes = r"(?:\s+(?:" + r"|".join(THANK_SUFFIXES) + r"))?"
-final_pattern = (
-    r"\b(?:" + thank_expressions + r")" + suffixes + r"\s*(?:,|\.|!|;)?\s*"
-)
+final_pattern = r"\b(?:" + thank_expressions + r")" + suffixes + r"\s*(?:,|\.|!|;)?\s*"
 thanking_regex = re.compile(final_pattern, flags=re.IGNORECASE | re.UNICODE)
 
 

ai_data_preprocessing_queue/Steps/spellcheck.py

@@ -30,9 +30,12 @@ def step(item: Any, item_state: dict[str, Any], global_state: dict[str, Any] | N
         all_words_to_check: Any = reduce(lambda x, y: cast(str, x) + cast(str, y), items)
 
         for w in all_words_to_check:
-            if len(item_word) < 4 and _levenshtein(item_word, w) == 1:
-                item = item.replace(item_word, w)
-            elif len(item_word) >= 4 and 1 <= _levenshtein(item_word, w) <= 2:
+            if (
+                len(item_word) < 4  # noqa: PLR2004
+                and _levenshtein(item_word, w) == 1
+                or len(item_word) >= 4  # noqa: PLR2004
+                and 1 <= _levenshtein(item_word, w) <= 2  # noqa: PLR2004
+            ):
                 item = item.replace(item_word, w)
 
     return item

ai_data_preprocessing_queue/Steps/text_only.py

@@ -4,5 +4,4 @@
 
 def step(item: Any, item_state: dict[str, Any], global_state: dict[str, Any] | None, preprocessor_data: str) -> Any:
     item = re.sub(r"[^\w\s]", " ", item)
-    item = re.sub(r"""\d""", " ", item)
-    return item
+    return re.sub(r"""\d""", " ", item)

ai_data_preprocessing_queue/Steps/token_replacement.py

@@ -15,7 +15,7 @@ def step(item: Any, item_state: dict[str, Any], global_state: dict[str, Any] | N
 
         # also replace dots at end of word
         if not line[0].endswith("."):
-            regex = regex + "\\b"
+            regex += "\\b"
 
         pattern = re.compile(regex)
         item = pattern.sub(line[1], item)
@@ -38,18 +38,16 @@ def _get_data_from_store_or_reload(global_state: dict[str, Any] | None, preproce
 
 def _prepare_pre_processor_data(preprocessor_data: str) -> list[list[str]]:
     lines: list[list[str]] = [
-        [s.strip() for i, s in enumerate(line.split(",")) if (i == 2 and re.compile(r"^[0-9\s]+$").match(s)) or i < 2]
+        [s.strip() for i, s in enumerate(line.split(",")) if (i == 2 and re.compile(r"^[0-9\s]+$").match(s)) or i < 2]  # noqa: PLR2004
         for line in preprocessor_data.splitlines()
-        if line.count(",") == 2
+        if line.count(",") == 2  # noqa: PLR2004
     ]
-    lines = [line for line in lines if len(line) == 3]
+    lines = [line for line in lines if len(line) == 3]  # noqa: PLR2004
 
     i: int = 0
     while i < len(lines):
         lines[i][2] = int(lines[i][2])  # type: ignore
         i += 1
 
-    # sort
-    lines = sorted(lines, key=lambda f: 0 - f[2])  # type: ignore
-
-    return lines
+    # sort and return
+    return sorted(lines, key=lambda f: 0 - f[2])  # type: ignore

ai_data_preprocessing_queue/__init__.py

@@ -1 +1 @@
-from .Pipeline import Pipeline
+from .Pipeline import Pipeline as Pipeline

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ai-data-preprocessing-queue"
-version = "1.7.1"
+version = "1.7.2"
 description = "A collection of different text processing steps that can be enabled or disabled dynamically."
 authors = ["KI-Team"]
 license = "MIT"

requirements-dev.txt

@@ -5,4 +5,5 @@ mypy
 parameterized
 pytest
 pytest-cov
+ruff