Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[tool.poetry]
name = "ai-data-preprocessing-queue"
version = "1.7.0"
version = "1.7.1"
description = "A collection of different text processing steps that can be enabled or disabled dynamically."
authors = ["KI-Team"]
license = "MIT"
Expand Down
2 changes: 1 addition & 1 deletion requirements.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
langdetect~=1.0.9
nltk>=3.9.0, <4.0
pandas>=2.0.0, <3.0
pandas>=3.0.0, <4.0
numpy>=2.0.0, <3.0
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@

setuptools.setup(
name="ai-data-preprocessing-queue",
version="1.7.0",
version="1.7.1",
description="Can be used to pre process data before ai processing",
long_description=LONG_DESCRIPTION,
long_description_content_type="text/markdown",
Expand Down
2 changes: 1 addition & 1 deletion tests/test_pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ def test_regex_replacement(self) -> None:
self.assertEqual("test password test", value)

def test_token_replacement_do_not_crash_for_no_data(self) -> None:
pipeline = Pipeline({"token_replacement": None})
pipeline = Pipeline({"token_replacement": None}) # noqa: S105
value = pipeline.consume("test text")
self.assertEqual("test text", value)

Expand Down
12 changes: 7 additions & 5 deletions tests/test_remove_signature.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,15 @@
import unittest
from unittest.mock import MagicMock, patch

from parameterized import parameterized
from unittest.mock import MagicMock, patch

from ai_data_preprocessing_queue.Pipeline import Pipeline
from ai_data_preprocessing_queue.Steps.remove_signature import (
step, remove_greetings_and_following_text, remove_newline)
remove_greetings_and_following_text, remove_newline, step)


class TestRemoveSignature(unittest.TestCase):
@parameterized.expand([ # type: ignore[misc]
@parameterized.expand([ # type: ignore[untyped-decorator]
(
"multiple_newlines",
"Could you please review the attached document?\n\n\nI need your feedback by Friday.",
Expand Down Expand Up @@ -38,7 +39,7 @@ class TestRemoveSignature(unittest.TestCase):
def test_remove_newline(self, name: str, input_text: str, expected: str) -> None:
self.assertEqual(remove_newline(input_text), expected)

@parameterized.expand([ # type: ignore[misc]
@parameterized.expand([ # type: ignore[untyped-decorator]
(
"english_signature_basic",
"Here's the project update. Sincerely, John Smith\nProject Manager",
Expand Down Expand Up @@ -88,7 +89,7 @@ def test_remove_newline(self, name: str, input_text: str, expected: str) -> None
def test_remove_greetings_and_following_text(self, name: str, input_text: str, expected: str) -> None:
self.assertEqual(remove_greetings_and_following_text(input_text), expected)

@parameterized.expand([ # type: ignore[misc]
@parameterized.expand([ # type: ignore[untyped-decorator]
(
"remove_signature_basic",
"We're sending the final draft for review. Best regards, Alice Johnson\nProject Lead",
Expand Down Expand Up @@ -151,3 +152,4 @@ def test_remove_signature_step_error(self, _: MagicMock) -> None:

if __name__ == "__main__":
unittest.main()
unittest.main()