Expand Settings to handle different Django Architectures

django_lightweight_queue/app_settings.py

@@ -1,52 +1,58 @@
-from typing import Dict, Union, Mapping, TypeVar, Callable, Optional, Sequence
+from typing import Any, Dict, List, Union, Callable, Optional, Sequence
 
-from django.conf import settings
+from django.conf import settings as django_settings
 
 from . import constants
 from .types import Logger, QueueName
 
-T = TypeVar('T')
 
-
-def setting(suffix: str, default: T) -> T:
-    attr_name = '{}{}'.format(constants.SETTING_NAME_PREFIX, suffix)
-    return getattr(settings, attr_name, default)
-
-
-WORKERS = setting('WORKERS', {})  # type: Dict[QueueName, int]
-BACKEND = setting(
-    'BACKEND',
-    'django_lightweight_queue.backends.synchronous.SynchronousBackend',
-)  # type: str
-
-LOGGER_FACTORY = setting(
-    'LOGGER_FACTORY',
-    'logging.getLogger',
-)  # type: Union[str, Callable[[str], Logger]]
-
-# Allow per-queue overrides of the backend.
-BACKEND_OVERRIDES = setting('BACKEND_OVERRIDES', {})  # type: Mapping[QueueName, str]
-
-MIDDLEWARE = setting('MIDDLEWARE', (
-    'django_lightweight_queue.middleware.logging.LoggingMiddleware',
-    'django_lightweight_queue.middleware.transaction.TransactionMiddleware',
-))  # type: Sequence[str]
-
-# Apps to ignore when looking for tasks. Apps must be specified as the dotted
-# name used in `INSTALLED_APPS`. This is expected to be useful when you need to
-# have a file called `tasks.py` within an app, but don't want
-# django-lightweight-queue to import that file.
-# Note: this _doesn't_ prevent tasks being registered from these apps.
-IGNORE_APPS = setting('IGNORE_APPS', ())  # type: Sequence[str]
-
-# Backend-specific settings
-REDIS_HOST = setting('REDIS_HOST', '127.0.0.1')  # type: str
-REDIS_PORT = setting('REDIS_PORT', 6379)  # type: int
-REDIS_PASSWORD = setting('REDIS_PASSWORD', None)  # type: Optional[str]
-REDIS_PREFIX = setting('REDIS_PREFIX', '')  # type: str
-
-ENABLE_PROMETHEUS = setting('ENABLE_PROMETHEUS', False)  # type: bool
-# Workers will export metrics on this port, and ports following it
-PROMETHEUS_START_PORT = setting('PROMETHEUS_START_PORT', 9300)  # type: int
-
-ATOMIC_JOBS = setting('ATOMIC_JOBS', True)  # type: bool
+class Settings:
+    _uses_short_names: bool = True  # used later in checking for values
+
+
+class Defaults(Settings):
+    WORKERS: Dict[QueueName, int] = {}
+    BACKEND: str = 'django_lightweight_queue.backends.synchronous.SynchronousBackend'
+    LOGGER_FACTORY: Union[str, Callable[[str], Logger]] = 'logging.getLogger'
+    # Allow per-queue overrides of the backend.
+    BACKEND_OVERRIDES: Dict[QueueName, str] = {}
+    MIDDLEWARE: Sequence[str] = ('django_lightweight_queue.middleware.logging.LoggingMiddleware',)
+    # Apps to ignore when looking for tasks. Apps must be specified as the dotted
+    # name used in `INSTALLED_APPS`. This is expected to be useful when you need to
+    # have a file called `tasks.py` within an app, but don't want
+    # django-lightweight-queue to import that file.
+    # Note: this _doesn't_ prevent tasks being registered from these apps.
+    IGNORE_APPS: Sequence[str] = ()
+    REDIS_HOST: str = '127.0.0.1'
+    REDIS_PORT: int = 6379
+    REDIS_PASSWORD: Optional[str] = None
+    REDIS_PREFIX: str = ""
+    ENABLE_PROMETHEUS: bool = False
+    # Workers will export metrics on this port, and ports following it
+    PROMETHEUS_START_PORT: int = 9300
+    ATOMIC_JOBS: bool = True
+
+
+class AppSettings:
+    def __init__(self, layers: List[Settings]) -> None:
+        self._layers = layers
+
+    def add_layer(self, layer: Settings) -> None:  # to be called by `load_extra_config`
+        self._layers.append(layer)
+
+    def __getattr__(self, name: str) -> Any:
+        # reverse so that later layers override earlier ones
+        for layer in reversed(self._layers):
+            # check to see if the layer is internal or external
+            use_short_names = getattr(layer, "_uses_short_names", False)
+            attr_name = (
+                name if use_short_names else
+                '{}{}'.format(constants.SETTING_NAME_PREFIX, name)
+            )
+            if hasattr(layer, attr_name):
+                return getattr(layer, attr_name)
+
+        raise AttributeError(f"Sorry, '{name}' is not a valid setting.")
+
+
+app_settings = AppSettings(layers=[Defaults(), django_settings])

django_lightweight_queue/backends/redis.py

@@ -3,11 +3,11 @@
 
 import redis
 
-from .. import app_settings
 from ..job import Job
 from .base import BackendWithPauseResume
 from ..types import QueueName, WorkerNumber
 from ..utils import block_for_time
+from ..app_settings import app_settings
 
 
 class RedisBackend(BackendWithPauseResume):

django_lightweight_queue/backends/reliable_redis.py

@@ -3,11 +3,11 @@
 
 import redis
 
-from .. import app_settings
 from ..job import Job
 from .base import BackendWithDeduplicate, BackendWithPauseResume
 from ..types import QueueName, WorkerNumber
 from ..utils import block_for_time, get_worker_numbers
+from ..app_settings import app_settings
 from ..progress_logger import ProgressLogger, NULL_PROGRESS_LOGGER
 
 # Work around https://github.com/python/mypy/issues/9914. Name needs to match

django_lightweight_queue/exposition.py

@@ -6,8 +6,8 @@
 
 from prometheus_client.exposition import MetricsHandler
 
-from . import app_settings
 from .types import QueueName, WorkerNumber
+from .app_settings import app_settings
 
 
 def get_config_response(

django_lightweight_queue/management/commands/queue_configuration.py

@@ -2,8 +2,8 @@
 
 from django.core.management.base import BaseCommand, CommandParser
 
-from ... import app_settings
 from ...utils import get_backend, get_queue_counts, load_extra_config
+from ...app_settings import app_settings
 from ...cron_scheduler import get_cron_config
 
 

django_lightweight_queue/runner.py

@@ -4,10 +4,10 @@
 import subprocess
 from typing import Dict, Tuple, Callable, Optional
 
-from . import app_settings
 from .types import Logger, QueueName, WorkerNumber
 from .utils import get_backend, set_process_title
 from .exposition import metrics_http_server
+from .app_settings import app_settings
 from .machine_types import Machine
 from .cron_scheduler import (
     CronScheduler,

django_lightweight_queue/task.py

@@ -12,10 +12,10 @@
     Optional,
 )
 
-from . import app_settings
 from .job import Job
 from .types import QueueName
 from .utils import get_backend, contribute_implied_queue_name
+from .app_settings import app_settings
 
 TCallable = TypeVar('TCallable', bound=Callable[..., Any])
 

django_lightweight_queue/utils.py

@@ -21,8 +21,9 @@
 from django.core.exceptions import MiddlewareNotUsed
 from django.utils.module_loading import module_has_submodule
 
-from . import constants, app_settings
+from . import constants
 from .types import Logger, QueueName, WorkerNumber
+from .app_settings import Settings, app_settings
 
 if TYPE_CHECKING:
     from .backends.base import BaseBackend
@@ -33,6 +34,11 @@
 
 
 def load_extra_config(file_path: str) -> None:
+
+    class Overrides(Settings):
+        """Container for holding internally overridden settings."""
+        pass
+
     extra_settings = imp.load_source('extra_settings', file_path)
 
     def get_setting_names(module: object) -> Set[str]:
@@ -53,9 +59,14 @@ def with_prefix(names: Iterable[str]) -> Set[str]:
         warnings.warn("Ignoring unexpected setting(s) '{}'.".format(unexpected_str))
 
     override_names = extra_names - unexpected_names
+
+    overrides = Overrides()
+
     for name in override_names:
         short_name = name[len(constants.SETTING_NAME_PREFIX):]
-        setattr(app_settings, short_name, getattr(extra_settings, name))
+        setattr(overrides, short_name, getattr(extra_settings, name))
+
+    app_settings.add_layer(overrides)
 
 
 @lru_cache()

django_lightweight_queue/worker.py

@@ -12,9 +12,9 @@
 
 from django.db import connections, transaction
 
-from . import app_settings
 from .types import QueueName, WorkerNumber
 from .utils import get_logger, get_backend, set_process_title
+from .app_settings import app_settings
 from .backends.base import BaseBackend
 
 if app_settings.ENABLE_PROMETHEUS:

tests/test_pause_resume.py

@@ -52,7 +52,7 @@ def setUp(self) -> None:
     # Can't use override_settings due to the copying of the settings values into
     # module values at startup.
     @mock.patch(
-        'django_lightweight_queue.app_settings.BACKEND',
+        'django_lightweight_queue.app_settings.Defaults.BACKEND',
         new='django_lightweight_queue.backends.redis.RedisBackend',
     )
     def test_pause_resume(self) -> None:

tests/test_reliable_redis_backend.py

@@ -50,7 +50,7 @@ def mock_workers(self, workers: Mapping[str, int]) -> Iterator[None]:
             'django_lightweight_queue.utils._accepting_implied_queues',
             new=False,
         ), unittest.mock.patch.dict(
-            'django_lightweight_queue.app_settings.WORKERS',
+            'django_lightweight_queue.app_settings.app_settings.WORKERS',
             workers,
         ):
             yield

tests/test_task.py

@@ -30,7 +30,7 @@ def mock_workers(self, workers: Mapping[str, int]) -> Iterator[None]:
             'django_lightweight_queue.utils._accepting_implied_queues',
             new=False,
         ), unittest.mock.patch.dict(
-            'django_lightweight_queue.app_settings.WORKERS',
+            'django_lightweight_queue.app_settings.Defaults.WORKERS',
             workers,
         ):
             yield
@@ -54,7 +54,7 @@ def mocked_get_path(path: str) -> Any:
             return get_path(path)
 
         patch = mock.patch(
-            'django_lightweight_queue.app_settings.BACKEND',
+            'django_lightweight_queue.app_settings.Defaults.BACKEND',
             new='test-backend',
         )
         patch.start()