From 2c490827a3f830f1e8836e5dad91b50b681710a8 Mon Sep 17 00:00:00 2001 From: ndpvt-web Date: Tue, 10 Mar 2026 04:32:30 +0000 Subject: [PATCH 1/2] fix: use lazy FTP connection initialization to avoid connecting on import Resolves #242. FTP connections are now deferred until data is actually requested, preventing import failures when the FTP server is unavailable. Changes: - Added lazy wrapper classes for all online_data database modules - SINAN, SIM, CNES, SIA, SIH, PNI, CIHA, SINASC, and IBGE modules now use lazy initialization - FTP connections only occur when methods are called, not at import time - Maintains backward compatibility with existing API Co-Authored-By: AI Assistant (Claude) Signed-off-by: ndpvt-web --- pysus/online_data/CIHA.py | 25 ++++++++++++++++++++++++- pysus/online_data/CNES.py | 28 +++++++++++++++++++++++++++- pysus/online_data/IBGE.py | 25 ++++++++++++++++++++++++- pysus/online_data/PNI.py | 25 ++++++++++++++++++++++++- pysus/online_data/SIA.py | 25 ++++++++++++++++++++++++- pysus/online_data/SIH.py | 25 ++++++++++++++++++++++++- pysus/online_data/SIM.py | 25 ++++++++++++++++++++++++- pysus/online_data/SINAN.py | 29 ++++++++++++++++++++++++++++- pysus/online_data/SINASC.py | 25 ++++++++++++++++++++++++- 9 files changed, 223 insertions(+), 9 deletions(-) diff --git a/pysus/online_data/CIHA.py b/pysus/online_data/CIHA.py index 9be4ecc..d3f2524 100644 --- a/pysus/online_data/CIHA.py +++ b/pysus/online_data/CIHA.py @@ -13,7 +13,30 @@ from pysus.ftp.databases.ciha import CIHA from pysus.ftp.utils import parse_UFs -ciha = CIHA().load() + +class _LazyCIHA: + """Lazy wrapper for CIHA database to defer FTP connection until needed.""" + + def __init__(self): + self._instance = None + + def _ensure_loaded(self): + """Ensure the CIHA database is loaded.""" + if self._instance is None: + self._instance = CIHA().load() + return self._instance + + def get_files(self, *args, **kwargs): + return self._ensure_loaded().get_files(*args, **kwargs) + + def describe(self, *args, **kwargs): + return self._ensure_loaded().describe(*args, **kwargs) + + def download(self, *args, **kwargs): + return self._ensure_loaded().download(*args, **kwargs) + + +ciha = _LazyCIHA() def get_available_years( diff --git a/pysus/online_data/CNES.py b/pysus/online_data/CNES.py index a3b1188..3b8b961 100644 --- a/pysus/online_data/CNES.py +++ b/pysus/online_data/CNES.py @@ -5,7 +5,33 @@ from pysus.ftp.databases.cnes import CNES from pysus.ftp.utils import parse_UFs -cnes = CNES().load() + +class _LazyCNES: + """Lazy wrapper for CNES database to defer FTP connection until needed.""" + + def __init__(self): + self._instance = None + + def _ensure_loaded(self): + """Ensure the CNES database is loaded.""" + if self._instance is None: + self._instance = CNES().load() + return self._instance + + def load(self, *args, **kwargs): + return self._ensure_loaded().load(*args, **kwargs) + + def get_files(self, *args, **kwargs): + return self._ensure_loaded().get_files(*args, **kwargs) + + def describe(self, *args, **kwargs): + return self._ensure_loaded().describe(*args, **kwargs) + + def download(self, *args, **kwargs): + return self._ensure_loaded().download(*args, **kwargs) + + +cnes = _LazyCNES() group_dict = { diff --git a/pysus/online_data/IBGE.py b/pysus/online_data/IBGE.py index 33fba90..fbd677d 100644 --- a/pysus/online_data/IBGE.py +++ b/pysus/online_data/IBGE.py @@ -20,7 +20,30 @@ APIBASE = "https://servicodados.ibge.gov.br/api/v3/" -ibge = IBGEDATASUS().load() + +class _LazyIBGEDATASUS: + """Lazy wrapper for IBGEDATASUS database to defer FTP connection until needed.""" + + def __init__(self): + self._instance = None + + def _ensure_loaded(self): + """Ensure the IBGEDATASUS database is loaded.""" + if self._instance is None: + self._instance = IBGEDATASUS().load() + return self._instance + + def get_files(self, *args, **kwargs): + return self._ensure_loaded().get_files(*args, **kwargs) + + def describe(self, *args, **kwargs): + return self._ensure_loaded().describe(*args, **kwargs) + + def download(self, *args, **kwargs): + return self._ensure_loaded().download(*args, **kwargs) + + +ibge = _LazyIBGEDATASUS() def get_sidra_table( diff --git a/pysus/online_data/PNI.py b/pysus/online_data/PNI.py index 2df41c1..933a9f0 100644 --- a/pysus/online_data/PNI.py +++ b/pysus/online_data/PNI.py @@ -8,7 +8,30 @@ from pysus.ftp.databases.pni import PNI from pysus.ftp.utils import parse_UFs -pni = PNI().load() + +class _LazyPNI: + """Lazy wrapper for PNI database to defer FTP connection until needed.""" + + def __init__(self): + self._instance = None + + def _ensure_loaded(self): + """Ensure the PNI database is loaded.""" + if self._instance is None: + self._instance = PNI().load() + return self._instance + + def get_files(self, *args, **kwargs): + return self._ensure_loaded().get_files(*args, **kwargs) + + def describe(self, *args, **kwargs): + return self._ensure_loaded().describe(*args, **kwargs) + + def download(self, *args, **kwargs): + return self._ensure_loaded().download(*args, **kwargs) + + +pni = _LazyPNI() def get_available_years(group, states): diff --git a/pysus/online_data/SIA.py b/pysus/online_data/SIA.py index 19ff22a..20f2caf 100644 --- a/pysus/online_data/SIA.py +++ b/pysus/online_data/SIA.py @@ -14,7 +14,30 @@ from pysus.ftp.databases.sia import SIA from pysus.ftp.utils import parse_UFs -sia = SIA().load() + +class _LazySIA: + """Lazy wrapper for SIA database to defer FTP connection until needed.""" + + def __init__(self): + self._instance = None + + def _ensure_loaded(self): + """Ensure the SIA database is loaded.""" + if self._instance is None: + self._instance = SIA().load() + return self._instance + + def get_files(self, *args, **kwargs): + return self._ensure_loaded().get_files(*args, **kwargs) + + def describe(self, *args, **kwargs): + return self._ensure_loaded().describe(*args, **kwargs) + + def download(self, *args, **kwargs): + return self._ensure_loaded().download(*args, **kwargs) + + +sia = _LazySIA() group_dict: Dict[str, Tuple[str, int, int]] = { diff --git a/pysus/online_data/SIH.py b/pysus/online_data/SIH.py index 67749f5..6986ac9 100644 --- a/pysus/online_data/SIH.py +++ b/pysus/online_data/SIH.py @@ -11,7 +11,30 @@ from pysus.ftp.databases.sih import SIH from pysus.ftp.utils import parse_UFs -sih = SIH().load() + +class _LazySIH: + """Lazy wrapper for SIH database to defer FTP connection until needed.""" + + def __init__(self): + self._instance = None + + def _ensure_loaded(self): + """Ensure the SIH database is loaded.""" + if self._instance is None: + self._instance = SIH().load() + return self._instance + + def get_files(self, *args, **kwargs): + return self._ensure_loaded().get_files(*args, **kwargs) + + def describe(self, *args, **kwargs): + return self._ensure_loaded().describe(*args, **kwargs) + + def download(self, *args, **kwargs): + return self._ensure_loaded().download(*args, **kwargs) + + +sih = _LazySIH() def get_available_years( diff --git a/pysus/online_data/SIM.py b/pysus/online_data/SIM.py index c021111..1e1fba4 100644 --- a/pysus/online_data/SIM.py +++ b/pysus/online_data/SIM.py @@ -15,7 +15,30 @@ from pysus.ftp.databases.sim import SIM from pysus.ftp.utils import parse_UFs -sim = SIM().load() + +class _LazySIM: + """Lazy wrapper for SIM database to defer FTP connection until needed.""" + + def __init__(self): + self._instance = None + + def _ensure_loaded(self): + """Ensure the SIM database is loaded.""" + if self._instance is None: + self._instance = SIM().load() + return self._instance + + def get_files(self, *args, **kwargs): + return self._ensure_loaded().get_files(*args, **kwargs) + + def describe(self, *args, **kwargs): + return self._ensure_loaded().describe(*args, **kwargs) + + def download(self, *args, **kwargs): + return self._ensure_loaded().download(*args, **kwargs) + + +sim = _LazySIM() def get_available_years( diff --git a/pysus/online_data/SINAN.py b/pysus/online_data/SINAN.py index fe5692d..16677bc 100644 --- a/pysus/online_data/SINAN.py +++ b/pysus/online_data/SINAN.py @@ -5,7 +5,34 @@ from pysus.ftp import CACHEPATH from pysus.ftp.databases.sinan import SINAN -sinan = SINAN().load() + +class _LazySINAN: + """Lazy wrapper for SINAN database to defer FTP connection until needed.""" + + def __init__(self): + self._instance = None + + def _ensure_loaded(self): + """Ensure the SINAN database is loaded.""" + if self._instance is None: + self._instance = SINAN().load() + return self._instance + + @property + def diseases(self): + return self._ensure_loaded().diseases + + def get_files(self, *args, **kwargs): + return self._ensure_loaded().get_files(*args, **kwargs) + + def describe(self, *args, **kwargs): + return self._ensure_loaded().describe(*args, **kwargs) + + def download(self, *args, **kwargs): + return self._ensure_loaded().download(*args, **kwargs) + + +sinan = _LazySINAN() def list_diseases() -> dict: diff --git a/pysus/online_data/SINASC.py b/pysus/online_data/SINASC.py index 2469d88..80523dc 100644 --- a/pysus/online_data/SINASC.py +++ b/pysus/online_data/SINASC.py @@ -11,7 +11,30 @@ from pysus.ftp.databases.sinasc import SINASC from pysus.ftp.utils import parse_UFs -sinasc = SINASC().load() + +class _LazySINASC: + """Lazy wrapper for SINASC database to defer FTP connection until needed.""" + + def __init__(self): + self._instance = None + + def _ensure_loaded(self): + """Ensure the SINASC database is loaded.""" + if self._instance is None: + self._instance = SINASC().load() + return self._instance + + def get_files(self, *args, **kwargs): + return self._ensure_loaded().get_files(*args, **kwargs) + + def describe(self, *args, **kwargs): + return self._ensure_loaded().describe(*args, **kwargs) + + def download(self, *args, **kwargs): + return self._ensure_loaded().download(*args, **kwargs) + + +sinasc = _LazySINASC() def get_available_years(group: str, states: Union[str, list[str]]) -> list: From cc98e092ad5ba631084e800859171ccdc4b31842 Mon Sep 17 00:00:00 2001 From: ndpvt-web Date: Tue, 10 Mar 2026 10:59:38 +0000 Subject: [PATCH 2/2] refactor: extract _LazyDatabase base class to eliminate duplication - Move all duplicated lazy wrapper logic into a shared _LazyDatabase base class in pysus/online_data/_lazy.py - Use __getattr__ to transparently proxy all attribute access to the underlying database instance, eliminating per-class method stubs - Add FTP connection error handling: _ensure_loaded now catches exceptions and raises ConnectionError with a descriptive message - Replace 9 individual _Lazy* classes with simple one-liner instantiations: e.g. ciha = _LazyDatabase(CIHA) Addresses review feedback from @fccoelho requesting base class extraction and FTP error handling. Co-Authored-By: Claude Opus 4.6 --- pysus/online_data/CIHA.py | 26 ++-------------------- pysus/online_data/CNES.py | 29 ++---------------------- pysus/online_data/IBGE.py | 26 ++-------------------- pysus/online_data/PNI.py | 26 ++-------------------- pysus/online_data/SIA.py | 26 ++-------------------- pysus/online_data/SIH.py | 26 ++-------------------- pysus/online_data/SIM.py | 26 ++-------------------- pysus/online_data/SINAN.py | 30 ++----------------------- pysus/online_data/SINASC.py | 26 ++-------------------- pysus/online_data/_lazy.py | 44 +++++++++++++++++++++++++++++++++++++ 10 files changed, 62 insertions(+), 223 deletions(-) create mode 100644 pysus/online_data/_lazy.py diff --git a/pysus/online_data/CIHA.py b/pysus/online_data/CIHA.py index d3f2524..a8de94d 100644 --- a/pysus/online_data/CIHA.py +++ b/pysus/online_data/CIHA.py @@ -12,31 +12,9 @@ from pysus.ftp import CACHEPATH from pysus.ftp.databases.ciha import CIHA from pysus.ftp.utils import parse_UFs +from pysus.online_data._lazy import _LazyDatabase - -class _LazyCIHA: - """Lazy wrapper for CIHA database to defer FTP connection until needed.""" - - def __init__(self): - self._instance = None - - def _ensure_loaded(self): - """Ensure the CIHA database is loaded.""" - if self._instance is None: - self._instance = CIHA().load() - return self._instance - - def get_files(self, *args, **kwargs): - return self._ensure_loaded().get_files(*args, **kwargs) - - def describe(self, *args, **kwargs): - return self._ensure_loaded().describe(*args, **kwargs) - - def download(self, *args, **kwargs): - return self._ensure_loaded().download(*args, **kwargs) - - -ciha = _LazyCIHA() +ciha = _LazyDatabase(CIHA) def get_available_years( diff --git a/pysus/online_data/CNES.py b/pysus/online_data/CNES.py index 3b8b961..85546f8 100644 --- a/pysus/online_data/CNES.py +++ b/pysus/online_data/CNES.py @@ -4,34 +4,9 @@ from pysus.ftp import CACHEPATH from pysus.ftp.databases.cnes import CNES from pysus.ftp.utils import parse_UFs +from pysus.online_data._lazy import _LazyDatabase - -class _LazyCNES: - """Lazy wrapper for CNES database to defer FTP connection until needed.""" - - def __init__(self): - self._instance = None - - def _ensure_loaded(self): - """Ensure the CNES database is loaded.""" - if self._instance is None: - self._instance = CNES().load() - return self._instance - - def load(self, *args, **kwargs): - return self._ensure_loaded().load(*args, **kwargs) - - def get_files(self, *args, **kwargs): - return self._ensure_loaded().get_files(*args, **kwargs) - - def describe(self, *args, **kwargs): - return self._ensure_loaded().describe(*args, **kwargs) - - def download(self, *args, **kwargs): - return self._ensure_loaded().download(*args, **kwargs) - - -cnes = _LazyCNES() +cnes = _LazyDatabase(CNES) group_dict = { diff --git a/pysus/online_data/IBGE.py b/pysus/online_data/IBGE.py index fbd677d..9ebe81f 100644 --- a/pysus/online_data/IBGE.py +++ b/pysus/online_data/IBGE.py @@ -14,36 +14,14 @@ import urllib3 from pysus.data.local import ParquetSet from pysus.ftp.databases.ibge_datasus import IBGEDATASUS +from pysus.online_data._lazy import _LazyDatabase # requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS = 'ALL:@SECLEVEL=1' APIBASE = "https://servicodados.ibge.gov.br/api/v3/" - -class _LazyIBGEDATASUS: - """Lazy wrapper for IBGEDATASUS database to defer FTP connection until needed.""" - - def __init__(self): - self._instance = None - - def _ensure_loaded(self): - """Ensure the IBGEDATASUS database is loaded.""" - if self._instance is None: - self._instance = IBGEDATASUS().load() - return self._instance - - def get_files(self, *args, **kwargs): - return self._ensure_loaded().get_files(*args, **kwargs) - - def describe(self, *args, **kwargs): - return self._ensure_loaded().describe(*args, **kwargs) - - def download(self, *args, **kwargs): - return self._ensure_loaded().download(*args, **kwargs) - - -ibge = _LazyIBGEDATASUS() +ibge = _LazyDatabase(IBGEDATASUS) def get_sidra_table( diff --git a/pysus/online_data/PNI.py b/pysus/online_data/PNI.py index 933a9f0..a4fa13f 100644 --- a/pysus/online_data/PNI.py +++ b/pysus/online_data/PNI.py @@ -7,31 +7,9 @@ from pysus.ftp import CACHEPATH from pysus.ftp.databases.pni import PNI from pysus.ftp.utils import parse_UFs +from pysus.online_data._lazy import _LazyDatabase - -class _LazyPNI: - """Lazy wrapper for PNI database to defer FTP connection until needed.""" - - def __init__(self): - self._instance = None - - def _ensure_loaded(self): - """Ensure the PNI database is loaded.""" - if self._instance is None: - self._instance = PNI().load() - return self._instance - - def get_files(self, *args, **kwargs): - return self._ensure_loaded().get_files(*args, **kwargs) - - def describe(self, *args, **kwargs): - return self._ensure_loaded().describe(*args, **kwargs) - - def download(self, *args, **kwargs): - return self._ensure_loaded().download(*args, **kwargs) - - -pni = _LazyPNI() +pni = _LazyDatabase(PNI) def get_available_years(group, states): diff --git a/pysus/online_data/SIA.py b/pysus/online_data/SIA.py index 20f2caf..6825f6f 100644 --- a/pysus/online_data/SIA.py +++ b/pysus/online_data/SIA.py @@ -13,31 +13,9 @@ from pysus.ftp import CACHEPATH from pysus.ftp.databases.sia import SIA from pysus.ftp.utils import parse_UFs +from pysus.online_data._lazy import _LazyDatabase - -class _LazySIA: - """Lazy wrapper for SIA database to defer FTP connection until needed.""" - - def __init__(self): - self._instance = None - - def _ensure_loaded(self): - """Ensure the SIA database is loaded.""" - if self._instance is None: - self._instance = SIA().load() - return self._instance - - def get_files(self, *args, **kwargs): - return self._ensure_loaded().get_files(*args, **kwargs) - - def describe(self, *args, **kwargs): - return self._ensure_loaded().describe(*args, **kwargs) - - def download(self, *args, **kwargs): - return self._ensure_loaded().download(*args, **kwargs) - - -sia = _LazySIA() +sia = _LazyDatabase(SIA) group_dict: Dict[str, Tuple[str, int, int]] = { diff --git a/pysus/online_data/SIH.py b/pysus/online_data/SIH.py index 6986ac9..e9afffa 100644 --- a/pysus/online_data/SIH.py +++ b/pysus/online_data/SIH.py @@ -10,31 +10,9 @@ from pysus.ftp import CACHEPATH from pysus.ftp.databases.sih import SIH from pysus.ftp.utils import parse_UFs +from pysus.online_data._lazy import _LazyDatabase - -class _LazySIH: - """Lazy wrapper for SIH database to defer FTP connection until needed.""" - - def __init__(self): - self._instance = None - - def _ensure_loaded(self): - """Ensure the SIH database is loaded.""" - if self._instance is None: - self._instance = SIH().load() - return self._instance - - def get_files(self, *args, **kwargs): - return self._ensure_loaded().get_files(*args, **kwargs) - - def describe(self, *args, **kwargs): - return self._ensure_loaded().describe(*args, **kwargs) - - def download(self, *args, **kwargs): - return self._ensure_loaded().download(*args, **kwargs) - - -sih = _LazySIH() +sih = _LazyDatabase(SIH) def get_available_years( diff --git a/pysus/online_data/SIM.py b/pysus/online_data/SIM.py index 1e1fba4..f5d565e 100644 --- a/pysus/online_data/SIM.py +++ b/pysus/online_data/SIM.py @@ -14,31 +14,9 @@ from pysus.ftp import CACHEPATH from pysus.ftp.databases.sim import SIM from pysus.ftp.utils import parse_UFs +from pysus.online_data._lazy import _LazyDatabase - -class _LazySIM: - """Lazy wrapper for SIM database to defer FTP connection until needed.""" - - def __init__(self): - self._instance = None - - def _ensure_loaded(self): - """Ensure the SIM database is loaded.""" - if self._instance is None: - self._instance = SIM().load() - return self._instance - - def get_files(self, *args, **kwargs): - return self._ensure_loaded().get_files(*args, **kwargs) - - def describe(self, *args, **kwargs): - return self._ensure_loaded().describe(*args, **kwargs) - - def download(self, *args, **kwargs): - return self._ensure_loaded().download(*args, **kwargs) - - -sim = _LazySIM() +sim = _LazyDatabase(SIM) def get_available_years( diff --git a/pysus/online_data/SINAN.py b/pysus/online_data/SINAN.py index 16677bc..5a7c0b1 100644 --- a/pysus/online_data/SINAN.py +++ b/pysus/online_data/SINAN.py @@ -4,35 +4,9 @@ import pandas as pd from pysus.ftp import CACHEPATH from pysus.ftp.databases.sinan import SINAN +from pysus.online_data._lazy import _LazyDatabase - -class _LazySINAN: - """Lazy wrapper for SINAN database to defer FTP connection until needed.""" - - def __init__(self): - self._instance = None - - def _ensure_loaded(self): - """Ensure the SINAN database is loaded.""" - if self._instance is None: - self._instance = SINAN().load() - return self._instance - - @property - def diseases(self): - return self._ensure_loaded().diseases - - def get_files(self, *args, **kwargs): - return self._ensure_loaded().get_files(*args, **kwargs) - - def describe(self, *args, **kwargs): - return self._ensure_loaded().describe(*args, **kwargs) - - def download(self, *args, **kwargs): - return self._ensure_loaded().download(*args, **kwargs) - - -sinan = _LazySINAN() +sinan = _LazyDatabase(SINAN) def list_diseases() -> dict: diff --git a/pysus/online_data/SINASC.py b/pysus/online_data/SINASC.py index 80523dc..66f535c 100644 --- a/pysus/online_data/SINASC.py +++ b/pysus/online_data/SINASC.py @@ -10,31 +10,9 @@ from pysus.ftp import CACHEPATH from pysus.ftp.databases.sinasc import SINASC from pysus.ftp.utils import parse_UFs +from pysus.online_data._lazy import _LazyDatabase - -class _LazySINASC: - """Lazy wrapper for SINASC database to defer FTP connection until needed.""" - - def __init__(self): - self._instance = None - - def _ensure_loaded(self): - """Ensure the SINASC database is loaded.""" - if self._instance is None: - self._instance = SINASC().load() - return self._instance - - def get_files(self, *args, **kwargs): - return self._ensure_loaded().get_files(*args, **kwargs) - - def describe(self, *args, **kwargs): - return self._ensure_loaded().describe(*args, **kwargs) - - def download(self, *args, **kwargs): - return self._ensure_loaded().download(*args, **kwargs) - - -sinasc = _LazySINASC() +sinasc = _LazyDatabase(SINASC) def get_available_years(group: str, states: Union[str, list[str]]) -> list: diff --git a/pysus/online_data/_lazy.py b/pysus/online_data/_lazy.py new file mode 100644 index 0000000..970d442 --- /dev/null +++ b/pysus/online_data/_lazy.py @@ -0,0 +1,44 @@ +""" +Lazy database wrapper to defer FTP connections until first use. + +This avoids connecting to the FTP server on import, which can cause +hangs and failures when the server is unavailable. +""" + +from loguru import logger + + +class _LazyDatabase: + """Base lazy wrapper that defers FTP connection until the database is + actually accessed. All attribute access is transparently proxied to + the underlying database instance. + + Subclasses only need to override ``_ensure_loaded`` if custom + initialisation logic is required; the default implementation calls + ``db_class().load()`` with error handling for FTP failures. + """ + + def __init__(self, db_class): + # Use object.__setattr__ to avoid triggering __getattr__ + object.__setattr__(self, "_db_class", db_class) + object.__setattr__(self, "_instance", None) + + def _ensure_loaded(self): + if self._instance is None: + try: + instance = self._db_class().load() + except Exception as exc: + logger.error( + "Failed to connect to FTP server for " + f"{self._db_class.__name__}: {exc}" + ) + raise ConnectionError( + f"Could not load {self._db_class.__name__} database. " + "The FTP server may be unavailable. " + f"Original error: {exc}" + ) from exc + object.__setattr__(self, "_instance", instance) + return self._instance + + def __getattr__(self, name): + return getattr(self._ensure_loaded(), name)