Merge pull request #423 from mabel-dev/ver0.6.30

0.6.30

Author: joocer

mabel/data/writers/internals/blob_writer.py

@@ -1,7 +1,9 @@
 import io
 import json
 import threading
-from typing import List, Optional, Union
+from typing import List
+from typing import Optional
+from typing import Union
 
 import orjson
 import orso
@@ -61,7 +63,12 @@ def arrow_append(self, record: dict = {}):
         self.records_in_buffer += 1
         self.wal.append(record)  # type:ignore
         # if this write would exceed the blob size, close it
-        if self.wal.nbytes() > self.maximum_blob_size:
+        # check every 1000 records to reduce overhead.
+        if (
+            self.records_in_buffer > 0
+            and self.records_in_buffer % 1000 == 0
+            and self.wal.nbytes() > self.maximum_blob_size
+        ):
             self.commit()
             self.open_buffer()
 
@@ -170,7 +177,9 @@ def commit(self):
                     if self.sort_by:
                         # Convert list of strings to PyArrow format
                         sort_spec = self.sort_by
-                        if isinstance(self.sort_by, list) and all(isinstance(item, str) for item in self.sort_by):
+                        if isinstance(self.sort_by, list) and all(
+                            isinstance(item, str) for item in self.sort_by
+                        ):
                             # Convert list of strings to list of tuples with default ascending order
                             sort_spec = [(col, "ascending") for col in self.sort_by]
                         pytable = pytable.sort_by(sort_spec)

mabel/version.py

@@ -1,6 +1,6 @@
 # Store the version here so:
 # 1) we don't load dependencies by storing it in __init__.py
 # 2) we can import it in setup.py for the same reason
-__version__ = "0.6.29"
+__version__ = "0.6.30"
 
 # nodoc - don't add to the documentation wiki

tests/NFT/currency.py

@@ -27,8 +27,8 @@
 import requests  # type:ignore
 import logging
 import operator
-import pkg_resources  # type:ignore
-from pkg_resources import parse_version  # type:ignore
+from importlib.metadata import distributions  # type:ignore
+from packaging.version import Version  # type:ignore
 
 logger = logging.getLogger("measures")
 logger.setLevel(10)
@@ -85,16 +85,16 @@ def get_latest_version(package_name):
 
 def compare_versions(version_a, version_b):
     # default to equals
-    operator = COMPARATORS["="]
+    operator_func = COMPARATORS["="]
 
     # find if it's a different operator
     find_operator = [c for c in COMPARATORS if version_a.startswith(c)]
     if len(find_operator):
         s = find_operator[0]
-        operator = COMPARATORS[s]
+        operator_func = COMPARATORS[s]
         version_a = version_a.lstrip(s)
 
-    return operator(parse_version(version_b), parse_version(version_a))
+    return operator_func(Version(version_b), Version(version_a))
 
 
 def get_package_summary(package=None, installed_version=None, vuln_details={}):
@@ -157,11 +157,11 @@ def test(self):
         results = []
 
         known_vulns = get_known_vulns()
-        for package in pkg_resources.working_set:
+        for dist in distributions():
             package_result = get_package_summary(
-                package=package.project_name,
-                installed_version=package.version,
-                vuln_details=known_vulns.get(package.project_name),
+                package=dist.metadata["Name"],
+                installed_version=dist.version,
+                vuln_details=known_vulns.get(dist.metadata["Name"]),
             )
 
             if package_result["state"] == "VULNERABLE":

tests/test_utils_dates.py

@@ -8,7 +8,7 @@
 import numpy
 import pandas
 
-from mabel.utils import dates
+#from mabel.utils import dates
 
 # fmt:off
 DATE_TESTS = [
@@ -40,6 +40,7 @@
 
 @pytest.mark.parametrize("string, expect", DATE_TESTS)
 def test_date_parser(string, expect):
+    from opteryx.utils import dates
     assert (
         dates.parse_iso(string) == expect
     ), f"in:{string}  res:{dates.parse_iso(string)} exp:{expect}"