Merge branch 'main' into issue-4950

Author: grantfitzsimmons

.env

@@ -43,6 +43,9 @@ ASSET_SERVER_URL=http://host.docker.internal/web_asset_store.xml
 # Make sure to set the `ASSET_SERVER_KEY` to a unique value
 ASSET_SERVER_KEY=your_asset_server_access_key
 
+# Information to connect to a Redis database
+# Specify will use this database as a process broker and storage for temporary
+# values
 REDIS_HOST=redis
 REDIS_PORT=6379
 REDIS_DB_INDEX=0

.github/workflows/test.yml

@@ -73,6 +73,11 @@ jobs:
         options:
           --health-cmd="mariadb-admin ping" --health-interval=5s
           --health-timeout=2s --health-retries=3
+      redis:
+        image: redis:latest
+        ports:
+          - 6379
+
 
     steps:
       - uses: actions/checkout@v4
@@ -128,6 +133,8 @@ jobs:
           echo "MIGRATOR_PASSWORD = 'MasterPassword'" >> specifyweb/settings/local_specify_settings.py
           echo "APP_USER_NAME = 'MasterUser'" >> specifyweb/settings/local_specify_settings.py
           echo "APP_USER_PASSWORD = 'MasterPassword'" >> specifyweb/settings/local_specify_settings.py
+          echo "REDIS_HOST = '127.0.0.1'" >> specifyweb/settings/local_specify_settings.py
+          echo "REDIS_PORT = ${{ job.services.redis.ports[6379] }}" >> specifyweb/settings/local_specify_settings.py
       
       - name: Need these files to be present
         run:

specifyweb/backend/businessrules/rules/attachment_rules.py

@@ -30,8 +30,8 @@ def attachment_jointable_save(sender, obj):
 
     attachee = get_attachee(obj)
     obj.attachment.tableid = attachee.specify_model.tableId
-    scopetype, scope = Scoping(attachee)()
-    obj.attachment.scopetype, obj.attachment.scopeid = scopetype, scope.id
+    scopetype, scope = Scoping.from_instance(attachee)
+    obj.attachment.scopetype, obj.attachment.scopeid = scopetype.value, scope.id
     obj.attachment.save()
 
 

specifyweb/backend/redis_cache/connect.py

@@ -0,0 +1,116 @@
+from redis import Redis
+from django.conf import settings
+
+class RedisConnection:
+    def __init__(self,
+                 host=getattr(settings, "REDIS_HOST", None),
+                 port=getattr(settings, "REDIS_PORT", None),
+                 db_index=getattr(settings, "REDIS_DB_INDEX", 0),
+                 decode_responses=True):
+        if None in (host, port, db_index):
+            raise ValueError(
+                "Redis is not correctly configured", host, port, db_index)
+        self.host = host
+        self.port = port
+        self.db_index = db_index
+        self.decode_responses = decode_responses
+        self.connection = Redis(
+            host=self.host,
+            port=self.port,
+            db=self.db_index,
+            decode_responses=self.decode_responses
+        )
+
+    def delete(self, key: str):
+        return self.connection.delete(key)
+
+
+class RedisDataType:
+    def __init__(self, established: RedisConnection) -> None:
+        self._established = established
+
+    @property
+    def connection(self):
+        return self._established.connection
+
+    def delete(self, key: str):
+        return self._established.delete(key)
+
+class RedisList(RedisDataType):
+    """
+    See https://redis.io/docs/latest/develop/data-types/lists/
+    """
+
+    def left_push(self, key: str, value) -> int:
+        return self.connection.lpush(key, value)
+
+    def right_push(self, key: str, value) -> int:
+        return self.connection.rpush(key, value)
+
+    def right_pop(self, key: str) -> str | bytes | None:
+        return self.connection.rpop(key)
+
+    def left_pop(self, key: str) -> str | bytes | None:
+        return self.connection.lpop(key)
+
+    def length(self, key: str) -> int:
+        return self.connection.llen(key)
+
+    def range(self, key: str, start_index: int, end_index: int) -> list[str] | list[bytes]:
+        return self.connection.lrange(key, start_index, end_index)
+
+    def trim(self, key: str, start_index: int, end_index: int) -> list[str] | list[bytes]:
+        return self.connection.ltrim(key, start_index, end_index)
+    
+    def blocking_left_pop(self, key: str, timeout: int) -> str | bytes | None:
+        response = self.connection.blpop(key, timeout=timeout)
+        if response is None:
+            return None
+        _filled_list_key, item = response
+        return item
+
+class RedisSet(RedisDataType):
+    """
+    See https://redis.io/docs/latest/develop/data-types/sets/
+    """
+    def add(self, key: str, *values: str) -> int:
+        return self.connection.sadd(key, *values)
+
+    def is_member(self, key: str, value: str) -> bool:
+        is_member = int(self.connection.sismember(key, value))
+        return is_member == 1
+
+    def remove(self, key: str, value: str):
+        return self.connection.srem(key, value)
+
+    def size(self, key: str) -> int:
+        return self.connection.scard(key)
+
+    def members(self, key: str) -> set[str]:
+        return self.connection.smembers(key)
+
+    def union(self, *keys: str) -> set[str]:
+        return self.connection.sunion(*keys)
+
+    def intersection(self, *keys: str) -> set[str]:
+        return self.connection.sinter(*keys)
+
+    def difference(self, *keys: str) -> set[str]:
+        return self.connection.sdiff(*keys)
+
+class RedisString(RedisDataType):
+    """
+    See https://redis.io/docs/latest/develop/data-types/strings/
+    """
+
+    def set(self, key, value, time_to_live=None, override_existing=True):
+        flags = {
+            "ex": time_to_live,
+            "nx": not override_existing
+        }
+        self.connection.set(key, value, **flags)
+
+    def get(self, key, delete_key=False) -> str | bytes | None:
+        if delete_key:
+            return self.connection.getdel(key)
+        return self.connection.get(key)

specifyweb/backend/redis_cache/rqueue.py

@@ -0,0 +1,61 @@
+import json
+
+from typing import Callable, Generator, Iterable, cast
+
+from specifyweb.backend.redis_cache.connect import RedisConnection, RedisList, RedisSet
+
+type Serialized = str | bytes | bytearray
+
+type Serializer[T] = Callable[[T], str]
+type Deserializer[T] = Callable[[Serialized], T]
+
+def default_serializer(obj) -> str:
+    return str(obj)
+
+def default_deserializer(serialized: Serialized):
+    return serialized
+
+class RedisQueue[T]:
+    def __init__(self, connection: RedisConnection, key: str,
+                 serializer: Serializer[T] | None = None,
+                 deserializer: Deserializer[T] | None = None):
+        self.connection = RedisList(connection)
+        self.key = key
+        self.serializer = serializer or cast(Serializer[T], default_serializer)
+        self.deserializer = deserializer or cast(Deserializer[T], default_deserializer)
+
+    def key_name(self, *name_parts: str | None):
+        key_name = "_".join([self.key, *(part for part in name_parts if part is not None)])
+        return key_name
+
+    def push(self, *objs: T, sub_key: str | None = None) -> int:
+        key_name = self.key_name(sub_key)
+        return self.connection.right_push(key_name, *self._serialize_objs(*objs))
+
+    def pop(self, sub_key: str | None = None) -> T | None:
+        key_name = self.key_name(sub_key)
+        popped = self.connection.left_pop(key_name)
+        if popped is None:
+            return None
+        return self.deserializer(popped)
+    
+    def wait_and_pop(self, timeout: int = 0, sub_key: str | None = None) -> T:
+        key_name = self.key_name(sub_key)
+        popped = self.connection.blocking_left_pop(key_name, timeout)
+        if popped is None:
+            raise TimeoutError("No items in queue after timeout")
+        return self.deserializer(popped)
+
+    def peek(self, sub_key: str | None = None) -> T | None:
+        key_name = self.key_name(sub_key)
+        top_value = self._deserialize_objs(*self.connection.range(key_name, 0, 0))
+        if len(top_value) == 0:
+            return None
+        return top_value[0]
+
+    def _serialize_objs(self, *objs: T) -> Generator[str, None, None]:
+        return (self.serializer(obj) for obj in objs)
+    
+    def _deserialize_objs(self, *serialized: Serialized):
+        return tuple(self.deserializer(obj) for obj in serialized)
+

specifyweb/backend/redis_cache/store.py

@@ -1,4 +1,4 @@
-from .utils import _set_string, _get_string
+from .utils import _set_string, _get_string, _delete_key, _add_to_set, _set_elements, _redis_type
 
 
 def set_string(key: str, value: str, time_to_live=None, override_existing=True):
@@ -15,3 +15,18 @@ def get_string(key: str, delete_key=False) -> str:
 
 def get_bytes(key: str, delete_key=False) -> bytes:
     return _get_string(key, delete_key=delete_key, decode_responses=False)
+
+
+def add_to_set(key: str, *elements: str):
+    return _add_to_set(key, *elements)
+
+def set_members(key: str):
+    return _set_elements(key)
+
+
+def delete_key(key: str):
+    return _delete_key(key)
+
+
+def redis_type(key: str):
+    return _redis_type(key)

specifyweb/backend/redis_cache/utils.py

@@ -1,4 +1,4 @@
-from typing import overload
+from typing import overload, Literal
 
 from redis import Redis
 from django.conf import settings
@@ -12,6 +12,9 @@ def redis_connection(decode_responses=True):
         raise ValueError("Redis is not correctly configured", redis_host, redis_port)
     return Redis(host=redis_host, port=redis_port, db=redis_db_index, decode_responses=decode_responses)
 
+def _delete_key(key: str):
+    host = redis_connection()
+    host.delete(key)
 
 def _set_string(key: str, value: str, time_to_live=None, override_existing=True, decode_responses=True):
     host = redis_connection(decode_responses=decode_responses)
@@ -37,3 +40,20 @@ def _get_string(key: str, delete_key: bool=False, decode_responses=True) -> str
         return host.getdel(key)
 
     return host.get(key)
+
+def _add_to_set(key: str, *elements: str):
+    if len(elements) <= 0:
+        return 0
+    host = redis_connection(decode_responses=True)
+    return host.sadd(key, *elements)
+
+def _set_elements(key: str) -> set:
+    host = redis_connection(decode_responses=True)
+    return host.smembers(key)
+
+# https://redis.io/docs/latest/commands/type/
+Redis_Type = Literal["none", "string", "list", "set", "hash", "stream", "vectorset"]
+
+def _redis_type(key: str) -> Redis_Type:
+    host = redis_connection(decode_responses=True)
+    return host.type(key)

specifyweb/backend/stored_queries/execution.py

@@ -51,7 +51,6 @@ class QuerySort:
     def by_id(sort_id: QUREYFIELD_SORT_T):
         return QuerySort.SORT_TYPES[sort_id]
 
-
 def DefaultQueryFormatterProps():
     return ObjectFormatterProps(
         format_agent_type=False,

specifyweb/backend/workbench/upload/parse.py

@@ -44,7 +44,12 @@ class ParseSucess(NamedTuple):
 ParseResult = ParseSucess | ParseFailure
 
 
-def parse_field(table_name: str, field_name: str, raw_value: str, formatter: ScopedFormatter | None = None) -> ParseResult:
+def parse_field(
+        table_name: str,
+        field_name: str,
+        raw_value: str,
+        formatter: ScopedFormatter | None = None
+        ) -> ParseResult:
     table = datamodel.get_table_strict(table_name)
     field = table.get_field_strict(field_name)
 
@@ -170,7 +175,11 @@ def parse_date(table: Table, field_name: str, dateformat: str, value: str) -> Pa
     return ParseFailure('badDateFormat', {'value': value, 'format': dateformat})
 
 
-def parse_formatted(uiformatter: ScopedFormatter, table: Table, field: Field | Relationship, value: str) -> ParseResult:
+def parse_formatted(
+        uiformatter: ScopedFormatter,
+        table: Table,
+        field: Field | Relationship,
+        value: str) -> ParseResult:
     try:
         canonicalized = uiformatter(table, value)
     except FormatMismatch as e: