diff --git a/mabel/data/writers/internals/blob_writer.py b/mabel/data/writers/internals/blob_writer.py index feb1d49..57278f2 100644 --- a/mabel/data/writers/internals/blob_writer.py +++ b/mabel/data/writers/internals/blob_writer.py @@ -1,7 +1,9 @@ import io import json import threading -from typing import List, Optional, Union +from typing import List +from typing import Optional +from typing import Union import orjson import orso @@ -61,7 +63,12 @@ def arrow_append(self, record: dict = {}): self.records_in_buffer += 1 self.wal.append(record) # type:ignore # if this write would exceed the blob size, close it - if self.wal.nbytes() > self.maximum_blob_size: + # check every 1000 records to reduce overhead. + if ( + self.records_in_buffer > 0 + and self.records_in_buffer % 1000 == 0 + and self.wal.nbytes() > self.maximum_blob_size + ): self.commit() self.open_buffer() @@ -170,7 +177,9 @@ def commit(self): if self.sort_by: # Convert list of strings to PyArrow format sort_spec = self.sort_by - if isinstance(self.sort_by, list) and all(isinstance(item, str) for item in self.sort_by): + if isinstance(self.sort_by, list) and all( + isinstance(item, str) for item in self.sort_by + ): # Convert list of strings to list of tuples with default ascending order sort_spec = [(col, "ascending") for col in self.sort_by] pytable = pytable.sort_by(sort_spec) diff --git a/mabel/version.py b/mabel/version.py index 9e24bee..92bcba0 100644 --- a/mabel/version.py +++ b/mabel/version.py @@ -1,6 +1,6 @@ # Store the version here so: # 1) we don't load dependencies by storing it in __init__.py # 2) we can import it in setup.py for the same reason -__version__ = "0.6.29" +__version__ = "0.6.30" # nodoc - don't add to the documentation wiki diff --git a/tests/NFT/currency.py b/tests/NFT/currency.py index a00e021..4107d29 100644 --- a/tests/NFT/currency.py +++ b/tests/NFT/currency.py @@ -27,8 +27,8 @@ import requests # type:ignore import logging import operator -import pkg_resources # type:ignore -from pkg_resources import parse_version # type:ignore +from importlib.metadata import distributions # type:ignore +from packaging.version import Version # type:ignore logger = logging.getLogger("measures") logger.setLevel(10) @@ -85,16 +85,16 @@ def get_latest_version(package_name): def compare_versions(version_a, version_b): # default to equals - operator = COMPARATORS["="] + operator_func = COMPARATORS["="] # find if it's a different operator find_operator = [c for c in COMPARATORS if version_a.startswith(c)] if len(find_operator): s = find_operator[0] - operator = COMPARATORS[s] + operator_func = COMPARATORS[s] version_a = version_a.lstrip(s) - return operator(parse_version(version_b), parse_version(version_a)) + return operator_func(Version(version_b), Version(version_a)) def get_package_summary(package=None, installed_version=None, vuln_details={}): @@ -157,11 +157,11 @@ def test(self): results = [] known_vulns = get_known_vulns() - for package in pkg_resources.working_set: + for dist in distributions(): package_result = get_package_summary( - package=package.project_name, - installed_version=package.version, - vuln_details=known_vulns.get(package.project_name), + package=dist.metadata["Name"], + installed_version=dist.version, + vuln_details=known_vulns.get(dist.metadata["Name"]), ) if package_result["state"] == "VULNERABLE": diff --git a/tests/test_utils_dates.py b/tests/test_utils_dates.py index 8a539b3..8811b67 100644 --- a/tests/test_utils_dates.py +++ b/tests/test_utils_dates.py @@ -8,7 +8,7 @@ import numpy import pandas -from mabel.utils import dates +#from mabel.utils import dates # fmt:off DATE_TESTS = [ @@ -40,6 +40,7 @@ @pytest.mark.parametrize("string, expect", DATE_TESTS) def test_date_parser(string, expect): + from opteryx.utils import dates assert ( dates.parse_iso(string) == expect ), f"in:{string} res:{dates.parse_iso(string)} exp:{expect}"