From 2c775d3c1f71c1729c85f5359957bf9dd1f5935a Mon Sep 17 00:00:00 2001 From: Med-Yassine-B Date: Sat, 20 Jun 2026 12:46:09 +0100 Subject: [PATCH 01/44] test(#40): asserting error type by printed error --- tests/test_exchangerate_client.py | 38 +++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/tests/test_exchangerate_client.py b/tests/test_exchangerate_client.py index faf0864..0b74d7b 100644 --- a/tests/test_exchangerate_client.py +++ b/tests/test_exchangerate_client.py @@ -3,7 +3,7 @@ from argus.clients.exchangerate_client import get_rates, check_error -def test_check_currency_timeout(monkeypatch): +def test_check_currency_timeout(monkeypatch,capsys): def test_get_resp(url, timeout): raise req.exceptions.Timeout() @@ -12,8 +12,10 @@ def test_get_resp(url, timeout): data = get_rates("EUR", "USD") assert data is None + captured=capsys.readouterr() + assert "API hat zu lange gebraucht." in captured.out -def test_check_currency_connection_error(monkeypatch): +def test_check_currency_connection_error(monkeypatch,capsys): def test_get_resp(url, timeout): raise req.exceptions.ConnectionError() @@ -22,8 +24,11 @@ def test_get_resp(url, timeout): data = get_rates("EUR", "USD") assert data is None + captured=capsys.readouterr() + assert "Keine Verbindung zur API." in captured.out -def test_check_currency_request_exception(monkeypatch): + +def test_check_currency_request_exception(monkeypatch,capsys): def test_get_resp(url, timeout): raise req.exceptions.RequestException("Testfehler") @@ -32,8 +37,11 @@ def test_get_resp(url, timeout): data = get_rates("EUR", "USD") assert data is None + captured=capsys.readouterr() + assert "Request fehlgeschlagen:" in captured.out + -def test_check_currency_value_error(monkeypatch): +def test_check_currency_value_error(monkeypatch,capsys): test_resp = Mock() test_resp.raise_for_status.return_value = None test_resp.json.side_effect = ValueError("Ungültige JSON-Antwort") @@ -46,12 +54,15 @@ def test_get_resp(url, timeout): data = get_rates("EUR", "USD") assert data is None + captured=capsys.readouterr() + assert "Fehler beim Verarbeiten der API-Antwort." in captured.out + -def test_check_currency_key_error(monkeypatch): +def test_check_currency_key_error(monkeypatch,capsys): test_resp = Mock() test_resp.raise_for_status.return_value = None test_resp.json.return_value = { - "result": "", + "result": "success",#not passing "success" bypases the "conversion_rate" checking "error_type": "", # "conversion_rate" fehlt absichtlich } @@ -64,6 +75,9 @@ def test_get_resp(url, timeout): data = get_rates("EUR", "USD") assert data is None + captured=capsys.readouterr() + assert "Unerwartete API-Antwortstruktur." in captured.out + def test_check_currency_valid(monkeypatch): test_resp = Mock() @@ -83,7 +97,7 @@ def test_get_resp(url, timeout): assert data == {"result": "success", "error_type": "", "conversion_rate": 1.2} -def test_check_currency_invalid(monkeypatch): +def test_check_currency_invalid(monkeypatch,capsys): test_resp = Mock() test_resp.raise_for_status.return_value = None test_resp.json.return_value = { @@ -100,6 +114,9 @@ def test_get_resp(url, timeout): data = get_rates("EUR", "USD") assert data is None + captured=capsys.readouterr() + assert "Invalid request! Please try again later." in captured.out + def test_check_error(capsys): check_error("unsupported-code") @@ -123,3 +140,10 @@ def test_check_error(capsys): captured.out == "Request limit reached! Please try again later or upgrade to exchangerate-api.com.\n" ) + + check_error("Some unknown Error") + captured = capsys.readouterr() + assert ( + captured.out + == "" + ) From 8181a29a4ace7e0ddca7724a5812fbf8e121059a Mon Sep 17 00:00:00 2001 From: Med-Yassine-B Date: Sat, 20 Jun 2026 12:59:37 +0100 Subject: [PATCH 02/44] fix(#40): Fixing the error where ValueErros never thrown --- src/argus/clients/exchangerate_client.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/argus/clients/exchangerate_client.py b/src/argus/clients/exchangerate_client.py index 8ea5e89..15d32fa 100644 --- a/src/argus/clients/exchangerate_client.py +++ b/src/argus/clients/exchangerate_client.py @@ -24,6 +24,17 @@ def get_rates(curr1: str, curr2: str): resp.raise_for_status() payload = resp.json() + + if payload["result"] == "success": + data["result"] = "success" + data["conversion_rate"] = payload["conversion_rate"] + return data + else: + data["result"] = "error" + data["error_type"] = payload.get("error_type") + check_error(data["error_type"]) + return None + except req.exceptions.Timeout: print("API hat zu lange gebraucht.") return None @@ -41,15 +52,6 @@ def get_rates(curr1: str, curr2: str): print("Unerwartete API-Antwortstruktur.") return None - if payload.get("result") == "success": - data["result"] = "success" - data["conversion_rate"] = payload.get("conversion_rate") - return data - else: - data["result"] = "error" - data["error_type"] = payload.get("error_type") - check_error(data["error_type"]) - return None def check_error(err_type: str) -> None: From 62ec4c8bd416d80559347cac1d5f3b0fe18a0d76 Mon Sep 17 00:00:00 2001 From: Med-Yassine-B Date: Sat, 20 Jun 2026 15:49:24 +0100 Subject: [PATCH 03/44] test(#40): chenged return False to a valid assert --- tests/test_timeseries_service.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_timeseries_service.py b/tests/test_timeseries_service.py index 7dd3c9f..b0c8b79 100644 --- a/tests/test_timeseries_service.py +++ b/tests/test_timeseries_service.py @@ -23,8 +23,9 @@ def test_get_a_full_timeseries(): "max_rate": [1.1055831909179688], } result = prepare_trend_analysis(test_curr, test_start, test_end, test_interval) - if result is None: - return False + + assert not ( result is None) + result_df, result_dict = result result_df["date"] = result_df["date"].astype("str") result_dict["min_date"] = [str(result_dict["min_date"][0])] From 2599718ec442477f76d6d8d5eaff38a21594f2e8 Mon Sep 17 00:00:00 2001 From: Med-Yassine-B Date: Sat, 20 Jun 2026 16:06:08 +0100 Subject: [PATCH 04/44] test(#40): covered all operators testing --- tests/test_validation_domain.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_validation_domain.py b/tests/test_validation_domain.py index a5bd41f..0166741 100644 --- a/tests/test_validation_domain.py +++ b/tests/test_validation_domain.py @@ -7,9 +7,13 @@ def test_op_is_valid(): - data = is_valid_op("+") - assert data is True + assert is_valid_op("+") is True + assert is_valid_op("-") is True + assert is_valid_op("*") is True + assert is_valid_op("/") is True + assert is_valid_op("%") is True + assert is_valid_op("**") is True def test_op_is_not_valid(): From 2d6196b30061c5eead0a782c1e5a67556becabd8 Mon Sep 17 00:00:00 2001 From: Med-Yassine-B Date: Sat, 20 Jun 2026 16:34:57 +0100 Subject: [PATCH 05/44] test(#40): fixed test_error_raise to raise the error instead of returning it --- src/argus/clients/exchangerate_client.py | 2 -- tests/test_exchangerate_client.py | 32 +++++++++++------------- tests/test_timeseries_service.py | 2 +- tests/test_yfinance_client.py | 2 +- 4 files changed, 17 insertions(+), 21 deletions(-) diff --git a/src/argus/clients/exchangerate_client.py b/src/argus/clients/exchangerate_client.py index 15d32fa..d899718 100644 --- a/src/argus/clients/exchangerate_client.py +++ b/src/argus/clients/exchangerate_client.py @@ -24,7 +24,6 @@ def get_rates(curr1: str, curr2: str): resp.raise_for_status() payload = resp.json() - if payload["result"] == "success": data["result"] = "success" data["conversion_rate"] = payload["conversion_rate"] @@ -53,7 +52,6 @@ def get_rates(curr1: str, curr2: str): return None - def check_error(err_type: str) -> None: """ Check the error type returned by the API and print an appropriate message. diff --git a/tests/test_exchangerate_client.py b/tests/test_exchangerate_client.py index 0b74d7b..1f15512 100644 --- a/tests/test_exchangerate_client.py +++ b/tests/test_exchangerate_client.py @@ -3,7 +3,7 @@ from argus.clients.exchangerate_client import get_rates, check_error -def test_check_currency_timeout(monkeypatch,capsys): +def test_check_currency_timeout(monkeypatch, capsys): def test_get_resp(url, timeout): raise req.exceptions.Timeout() @@ -12,10 +12,11 @@ def test_get_resp(url, timeout): data = get_rates("EUR", "USD") assert data is None - captured=capsys.readouterr() + captured = capsys.readouterr() assert "API hat zu lange gebraucht." in captured.out -def test_check_currency_connection_error(monkeypatch,capsys): + +def test_check_currency_connection_error(monkeypatch, capsys): def test_get_resp(url, timeout): raise req.exceptions.ConnectionError() @@ -24,11 +25,11 @@ def test_get_resp(url, timeout): data = get_rates("EUR", "USD") assert data is None - captured=capsys.readouterr() + captured = capsys.readouterr() assert "Keine Verbindung zur API." in captured.out -def test_check_currency_request_exception(monkeypatch,capsys): +def test_check_currency_request_exception(monkeypatch, capsys): def test_get_resp(url, timeout): raise req.exceptions.RequestException("Testfehler") @@ -37,11 +38,11 @@ def test_get_resp(url, timeout): data = get_rates("EUR", "USD") assert data is None - captured=capsys.readouterr() + captured = capsys.readouterr() assert "Request fehlgeschlagen:" in captured.out -def test_check_currency_value_error(monkeypatch,capsys): +def test_check_currency_value_error(monkeypatch, capsys): test_resp = Mock() test_resp.raise_for_status.return_value = None test_resp.json.side_effect = ValueError("Ungültige JSON-Antwort") @@ -54,15 +55,15 @@ def test_get_resp(url, timeout): data = get_rates("EUR", "USD") assert data is None - captured=capsys.readouterr() + captured = capsys.readouterr() assert "Fehler beim Verarbeiten der API-Antwort." in captured.out -def test_check_currency_key_error(monkeypatch,capsys): +def test_check_currency_key_error(monkeypatch, capsys): test_resp = Mock() test_resp.raise_for_status.return_value = None test_resp.json.return_value = { - "result": "success",#not passing "success" bypases the "conversion_rate" checking + "result": "success", # not passing "success" bypases the "conversion_rate" checking "error_type": "", # "conversion_rate" fehlt absichtlich } @@ -75,7 +76,7 @@ def test_get_resp(url, timeout): data = get_rates("EUR", "USD") assert data is None - captured=capsys.readouterr() + captured = capsys.readouterr() assert "Unerwartete API-Antwortstruktur." in captured.out @@ -97,7 +98,7 @@ def test_get_resp(url, timeout): assert data == {"result": "success", "error_type": "", "conversion_rate": 1.2} -def test_check_currency_invalid(monkeypatch,capsys): +def test_check_currency_invalid(monkeypatch, capsys): test_resp = Mock() test_resp.raise_for_status.return_value = None test_resp.json.return_value = { @@ -114,7 +115,7 @@ def test_get_resp(url, timeout): data = get_rates("EUR", "USD") assert data is None - captured=capsys.readouterr() + captured = capsys.readouterr() assert "Invalid request! Please try again later." in captured.out @@ -143,7 +144,4 @@ def test_check_error(capsys): check_error("Some unknown Error") captured = capsys.readouterr() - assert ( - captured.out - == "" - ) + assert captured.out == "" diff --git a/tests/test_timeseries_service.py b/tests/test_timeseries_service.py index b0c8b79..cd5c97a 100644 --- a/tests/test_timeseries_service.py +++ b/tests/test_timeseries_service.py @@ -24,7 +24,7 @@ def test_get_a_full_timeseries(): } result = prepare_trend_analysis(test_curr, test_start, test_end, test_interval) - assert not ( result is None) + assert result is not None result_df, result_dict = result result_df["date"] = result_df["date"].astype("str") diff --git a/tests/test_yfinance_client.py b/tests/test_yfinance_client.py index faf15fc..6201b19 100644 --- a/tests/test_yfinance_client.py +++ b/tests/test_yfinance_client.py @@ -72,7 +72,7 @@ def test_error_raise(monkeypatch): def fake_yfinance_download( tickers=test_curr, start=test_start, end=test_end, interval=test_interval ): - return Exception("fake yfinance error") + raise Exception("fake yfinance error") monkeypatch.setattr("yfinance.download", fake_yfinance_download) From 8d6cbe03454acd219be1ebf28205addd5f86a7c3 Mon Sep 17 00:00:00 2001 From: KartavyaDikshit <90633769+KartavyaDikshit@users.noreply.github.com> Date: Sun, 21 Jun 2026 10:22:12 +0200 Subject: [PATCH 06/44] docs(43): research first forecasting approach * chore: auto-format to pass CI checks * chore(43): delete create_pr.py Add research for the first forecasting approach for ARGUS market time-series data. Remove out-of-scope PR automation script before merging. --------- Co-authored-by: Lev Gusiev <89646710+BytecodeBrewer@users.noreply.github.com> --- docs/forecast_research.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 docs/forecast_research.md diff --git a/docs/forecast_research.md b/docs/forecast_research.md new file mode 100644 index 0000000..5db8c1b --- /dev/null +++ b/docs/forecast_research.md @@ -0,0 +1,38 @@ +# Research: First Forecasting Approach for Market Time Series + +## 1. Realistic First Prediction Task for ARGUS +A realistic first prediction task for ARGUS is **next-day exchange-rate movement** or **trend direction**. Predicting the exact next value (point forecast) is generally much harder and often less useful for trading/signal workflows than predicting the direction of the movement (up/down). A directional classification task serves as a simple, actionable signal for basic workflows. + +## 2. Baseline Methods to Implement First +Before jumping into complex models, the following baselines should be implemented to evaluate the added value of any machine learning model: +- **Naive last-value forecast**: The prediction for the next period is exactly the value from the current period. This is surprisingly hard to beat in random walk-like financial time series. +- **Moving average forecast**: A simple rolling average to predict the next value or determine trend direction. +- **Simple linear regression**: To capture basic linear trends over a given historical window. + +## 3. Libraries: NumPy, pandas, or scikit-learn? +The first implementation should use **pandas** and **scikit-learn**: +- **pandas**: Excellent for time-series manipulation, rolling windows, lagging features, and handling missing data. +- **scikit-learn**: Offers robust implementations of simple models (e.g., Linear Regression, Logistic Regression for direction) and provides standardized metrics and cross-validation tools designed for time series (e.g., `TimeSeriesSplit`). + +## 4. Evaluation Metrics +For the initial approaches, we should focus on: +- **Directional accuracy**: The percentage of times the model correctly predicts the direction of the price movement (up vs down). This is often more relevant than magnitude errors. +- **MAE (Mean Absolute Error)**: If point forecasting is used, MAE is more robust to outliers than RMSE and provides a linear penalty for errors. +- **RMSE (Root Mean Squared Error)**: Useful to penalize larger errors more heavily, but should be secondary to directional accuracy for basic signal generation. + +## 5. Why is LSTM not the first implementation step? +LSTMs are highly complex, require a large amount of well-structured data to train effectively without overfitting, and are notoriously difficult to tune. For financial time series, which suffer from low signal-to-noise ratios, an LSTM is likely to overfit the training data or collapse to predicting the last known value. Starting with an LSTM obscures whether the underlying data has any predictive power and sets a high barrier for debugging and infrastructure. + +## 6. Prerequisites for an LSTM Ticket +Before considering LSTMs or other deep learning approaches, the following must be established: +- A reliable data ingestion and preprocessing pipeline. +- Established baseline performance metrics (e.g., a naive model and a linear regression model) to compare against. +- Sufficient historical data size. +- A robust backtesting and cross-validation framework to ensure the LSTM isn't just memorizing data or overfitting. +- Hardware/infrastructure to support longer training times and hyperparameter tuning. + +## 7. Recommended First Implementation Approach +**Recommendation**: Start with **directional trend prediction** (predicting whether the next value is higher or lower than the current value) using a simple **Logistic Regression** model via **scikit-learn**. +- Use **pandas** to create basic lagged features (e.g., previous returns, moving averages). +- Evaluate using **directional accuracy**. +- Compare performance strictly against a **naive momentum** (predicting the trend continues) or **majority-class** baseline. From cc3f3bd889cdb47a51ae183ceb5631d9b9b414dd Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Sun, 21 Jun 2026 11:20:38 +0200 Subject: [PATCH 07/44] docs(#64): update contributing --- CONTRIBUTING.md | 235 +++++++++++++++++------------------------------- 1 file changed, 81 insertions(+), 154 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0b53865..fba5e68 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,33 +4,9 @@ Thank you for your interest in contributing to ARGUS. ARGUS is a Python-based market analytics project focused on clean data workflows, reliable code, useful metrics and future AI-assisted monitoring. -This project is still growing, so contributions should help the project become more stable, understandable and useful step by step. +The project is still growing, so contributions should be small, focused and easy to review. You do not need to be an expert to contribute, but your changes should be understandable, reliable and related to the current project direction. -> [!IMPORTANT] -> ARGUS values reliability, clear communication and long-term skill building. -> Contributions should improve the project without creating unnecessary complexity. - ---- - -## Project Mindset - -ARGUS is not only about adding features quickly. - -The project is built around: - -- clean Python code -- understandable architecture -- reliable tests -- useful documentation -- careful data handling -- practical analytics -- continuous learning - -Good contributions should make the project easier to use, test, maintain or extend. - ---- - -## What You Can Contribute +Good starting points are issues labeled `good first issue`. These issues are usually smaller, easier to review and better suited for getting familiar with the project. Helpful contributions include: @@ -38,17 +14,14 @@ Helpful contributions include: - tests - documentation improvements - small refactorings -- validation improvements - analytics metrics -- chart improvements - data-source clients -- CI/CD improvements -- issue clarification -- architecture notes -- examples and usage instructions +- UI or chart improvements +- CI/CD and tooling improvements +- architecture or research notes -> [!NOTE] -> Large features should usually start with an issue or short discussion before implementation. +> [!IMPORTANT] +> Please keep changes focused and avoid adding unnecessary complexity. --- @@ -76,46 +49,21 @@ Bad examples: --- -## Development Setup - -Clone the repository: - -```bash -git clone https://github.com/BytecodeBrewer/argus.git -cd argus -``` - -Create a virtual environment: - -```bash -python -m venv .venv -``` - -Activate it. - -On Windows PowerShell: - -```powershell -.venv\Scripts\Activate.ps1 -``` +## Branch Workflow -On macOS/Linux: +For issue-based work, create your branch from the related GitHub issue when possible. -```bash -source .venv/bin/activate -``` +GitHub may suggest a branch name based on the issue title. You can shorten it if the generated name is too long. -Install the project with development dependencies: +Good branch names are focused and describe the task: -```bash -pip install -e ".[dev]" +```text +43-research-forecasting-approach +33-add-yfinance-client +40-improve-test-coverage ``` ---- - -## Branch Workflow - -Create a new branch for your work: +If you create the branch manually, use: ```bash git checkout -b @@ -124,24 +72,30 @@ git checkout -b Example: ```bash -git checkout -b 12-add-volatility-metric +git checkout -b 43-research-forecasting-approach ``` -Use focused branch names that describe the work. - --- ## Commit Expectations Commits should be small, understandable and related to the current task. +ARGUS uses a conventional commit style with an issue reference: + +```text +type(#issue): short description +``` + Good commit messages: ```text -Add rolling volatility metric -Fix currency validation edge case -Update README setup instructions -Add tests for trend metrics +docs(#43): research first forecasting approach +feat(#33): add yfinance historical data client +test(#40): add tests for conversion service +fix(#33): handle empty historical data response +refactor(#34): split metric helpers +ci(#10): update commit message workflow ``` Avoid unclear messages: @@ -155,21 +109,25 @@ final ``` > [!TIP] -> A good commit tells future readers what changed and why it belongs to the task. +> A good commit tells future readers what changed and which issue it belongs to. --- -## Testing +## Checks -Before opening a pull request, run the test suite: +Before opening a pull request, run the project checks: ```bash pytest +ruff check . +ruff format --check . ``` -A pull request should not be opened as ready for review if tests are failing without explanation. +These checks verify that tests pass, code style is valid and formatting is consistent. + +A pull request should not be marked as ready for review if checks are failing without explanation. -If a test fails and you do not know why, mention it clearly in the pull request. +If a check fails and you are unsure why, mention it clearly in the pull request. > [!IMPORTANT] > CI checks must pass before a pull request can be merged. @@ -178,65 +136,23 @@ If a test fails and you do not know why, mention it clearly in the pull request. ## Pull Request Expectations -A good pull request should include: +Pull requests should target `develop` unless the maintainer explicitly says otherwise. -- a clear title -- a short explanation of what changed -- a link to the related issue if available -- notes about tests -- screenshots for UI changes if useful -- a short explanation of any trade-offs +Do not open feature, research or documentation pull requests directly against `main`. +The `main` branch is reserved for stable/release-ready changes. -Pull requests should be focused and reviewable. +Please use the pull request template and fill it out clearly. -Before opening a pull request, run: +The template helps reviewers understand: -```bash -pytest -ruff check . -ruff format --check . -``` +- what changed +- which issue is related +- whether tests were run +- whether documentation or screenshots are needed +- if there are any notes or trade-offs ---- - -## Reliability Expectations - -Contributors are expected to work reliably. - -This means: - -- do not submit random or unfinished code without context -- do not ignore failing tests -- do not introduce secrets, API keys or local machine paths -- do not rewrite unrelated parts of the project without discussion -- communicate if you are unsure -- keep changes understandable for future contributors -- respect the existing architecture unless there is a clear reason to change it - -Reliability does not mean knowing everything already. - -It means being honest, careful and consistent. - ---- - -## Learning Mindset - -ARGUS welcomes contributors who want to improve their technical skills. - -You do not need to be an expert to contribute. - -Helpful behavior includes: - -- asking clear questions -- explaining your reasoning -- being open to review feedback -- improving your code after feedback -- learning from tests, errors and architecture discussions -- documenting what you learned when it helps others - -> [!NOTE] -> This project values skill growth. -> A thoughtful small contribution is better than a large unclear one. +Do not bypass the pull request template or replace it with an unrelated auto-generated description. +It makes reviewing harder and may delay the merge. --- @@ -278,7 +194,7 @@ Do not commit: Use a local `.env` file for secrets. ```env -api_key=your_api_key_here +EXCHANGE_RATE_API_KEY=your_api_key_here ``` > [!WARNING] @@ -299,12 +215,40 @@ Useful documentation includes: - data-source assumptions - troubleshooting notes -Repository-level files such as `README.md`, `CONTRIBUTING.md`, `CODE_OF_CONDUCT.md` and `LICENSE` belong in the repository root. - Technical notes, research and deeper explanations belong in `docs/`. --- +## Contribution Expectations + +Contributors are expected to keep changes focused, understandable and related to the issue or task. + +Please: + +- keep pull requests small and reviewable +- follow the pull request template +- explain your changes clearly +- communicate if you are unsure +- ask questions when something is unclear +- be open to review feedback +- improve your contribution step by step after feedback +- avoid unrelated rewrites +- avoid committing secrets, API keys or local machine paths +- respect the existing architecture unless there is a clear reason to change it +- do not add scripts that automatically run `git add`, `git commit`, `git push` or create pull requests unless this was discussed first + +A contribution may be declined or delayed if it: + +- does not fit the current roadmap +- adds too much complexity too early +- breaks existing functionality +- lacks necessary checks or documentation +- duplicates existing work +- bypasses the repository workflow +- does not follow the project’s quality expectations + +--- + ## Communication Please communicate respectfully and constructively. @@ -323,20 +267,3 @@ When receiving feedback: - improve the contribution step by step All contributors are expected to follow the project’s Code of Conduct. - ---- - -## Maintainer Notes - -The maintainer may ask for changes before merging a pull request. - -A contribution may be declined if it: - -- does not fit the current roadmap -- adds too much complexity too early -- breaks existing functionality -- lacks necessary tests -- duplicates existing work -- does not follow the project’s quality expectations - -This helps keep ARGUS stable, learnable and maintainable. \ No newline at end of file From 13627c6ead11665673fe97b7b350bae0d7e4aebf Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Sun, 21 Jun 2026 11:26:56 +0200 Subject: [PATCH 08/44] docs(#64): shorten contributing --- CONTRIBUTING.md | 90 ++++++++++++++----------------------------------- 1 file changed, 26 insertions(+), 64 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fba5e68..3f87759 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -49,6 +49,30 @@ Bad examples: --- +## Contribution Expectations + +Contributors are expected to keep changes focused, understandable and related to the issue or task. + +Please: + +- explain your changes clearly +- be open to review feedback +- improve your contribution step by step after feedback +- avoid unrelated rewrites +- respect the existing architecture unless there is a clear reason to change it +- do not add scripts that automatically run `git add`, `git commit`, `git push` or create pull requests unless this was discussed first + +A contribution may be declined or delayed if it: + +- does not fit the current roadmap +- adds too much complexity too early +- breaks existing functionality +- lacks necessary checks or documentation +- duplicates existing work +- bypasses the repository workflow + +--- + ## Branch Workflow For issue-based work, create your branch from the related GitHub issue when possible. @@ -181,25 +205,14 @@ For analytics code: ## Secrets and API Keys -Never commit secrets. - -Do not commit: +Never commit secrets, API keys, tokens, passwords, `.env` files or local config files with private data. -- API keys -- tokens -- passwords -- `.env` files -- local config files with private data - -Use a local `.env` file for secrets. +Use a local `.env` file for secrets: ```env EXCHANGE_RATE_API_KEY=your_api_key_here ``` -> [!WARNING] -> If you accidentally commit a secret, revoke it immediately and inform the maintainer. - --- ## Documentation @@ -216,54 +229,3 @@ Useful documentation includes: - troubleshooting notes Technical notes, research and deeper explanations belong in `docs/`. - ---- - -## Contribution Expectations - -Contributors are expected to keep changes focused, understandable and related to the issue or task. - -Please: - -- keep pull requests small and reviewable -- follow the pull request template -- explain your changes clearly -- communicate if you are unsure -- ask questions when something is unclear -- be open to review feedback -- improve your contribution step by step after feedback -- avoid unrelated rewrites -- avoid committing secrets, API keys or local machine paths -- respect the existing architecture unless there is a clear reason to change it -- do not add scripts that automatically run `git add`, `git commit`, `git push` or create pull requests unless this was discussed first - -A contribution may be declined or delayed if it: - -- does not fit the current roadmap -- adds too much complexity too early -- breaks existing functionality -- lacks necessary checks or documentation -- duplicates existing work -- bypasses the repository workflow -- does not follow the project’s quality expectations - ---- - -## Communication - -Please communicate respectfully and constructively. - -When giving feedback: - -- focus on the code or idea, not the person -- explain the reason behind suggestions -- be specific -- stay open to alternatives - -When receiving feedback: - -- assume good intent -- ask questions if something is unclear -- improve the contribution step by step - -All contributors are expected to follow the project’s Code of Conduct. From 635625892ea5539bf8b4e216d891a9102d8ecdec Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Thu, 25 Jun 2026 15:05:49 +0200 Subject: [PATCH 09/44] docs(#37): add use case + compare db --- docs/research-databases-and-storage.md | 97 ++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 docs/research-databases-and-storage.md diff --git a/docs/research-databases-and-storage.md b/docs/research-databases-and-storage.md new file mode 100644 index 0000000..b27ac9e --- /dev/null +++ b/docs/research-databases-and-storage.md @@ -0,0 +1,97 @@ +# ARGUS Storage Research + +## Goal + +Research what ARGUS should store and which database/storage approach fits the project. + +ARGUS is moving from live API requests and in-memory analytics toward real data workflows. +The first storage decision should support local market analytics, SQL practice and future dashboard features without adding unnecessary infrastructure too early. + +--- + +## Storage Use Cases + +ARGUS should eventually store different kinds of data, but not all of them need to be implemented at once. + +Relevant storage use cases are: + +* historical exchange rates +* cleaned historical market data +* source information +* instruments that ARGUS can analyze +* later watchlists +* later generated reports +* later macroeconomic data +* later paper-trading history + +The first implementation should focus on historical market data and the basic entities needed to query it. + +--- + +## Storage Candidates + +| Option | Best Use Case | Strengths | Limitations | Fit for ARGUS | +|---|---|---|---|---| +| SQLite | Small local app storage | Very simple, serverless, good for settings and watchlists | Less analytics-focused | Good later for app-state, not first choice | +| DuckDB | Local analytical storage | SQL-based, local, strong for analytical queries, good with Python/notebooks | Not a server database | Best first choice | +| PostgreSQL | Server/product database | Strong relational database, good for web apps, users, reports and cloud setups | More setup and infrastructure | Very good later | +| Parquet | Export/archive format | Efficient columnar format for analytical data | Not a database by itself | Useful later, not first storage layer | + +--- + +## Local, Server and Cloud Options + +| Option | Meaning | Fit Now | Fit Later | +|---|---|---:|---:| +| Local storage | Database or files run locally in the project | High | High | +| Server database | Database runs as a separate service, e.g. PostgreSQL | Medium | High | +| Cloud storage/database | Managed storage or database in the cloud | Low | High | + +ARGUS should start with local storage. + +Reason: + +* simpler setup +* easier learning curve +* good fit for a Python analytics project +* no cloud or server infrastructure required yet +* enough for historical data, metrics and dashboard development + +Server and cloud storage should come later when ARGUS has stronger product features such as reports, user state, paper-trading history or deployment needs. + +--- + +## Recommended Decision + +DuckDB should be the first storage technology for ARGUS. + +Why: + +* ARGUS currently needs local analytical storage +* DuckDB is designed for analytical SQL workflows +* it does not require a database server +* it works well with Python and notebook-based exploration +* it fits historical time-series and market-data analysis +* it keeps the first implementation manageable + +PostgreSQL should be introduced later when ARGUS moves toward a more product-like architecture. + +SQLGate should also be kept for that later PostgreSQL phase, not for the first DuckDB phase. + +--- + +## First Data Model Direction + +The first data model should support FX data now and broader market data later. + +ARGUS should not use a narrow `date | value` table as the main market-data model. + +That would work for simple exchange rates, but it would become limiting once ARGUS adds stocks, ETFs, indices or broader market APIs. + +The first model should focus on three tables: + +```text +data_sources +instruments +price_bars +``` From 2321e1b62c0453d8d8416be36e5d68fce593590a Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Thu, 25 Jun 2026 15:20:43 +0200 Subject: [PATCH 10/44] docs(#37): add plan and data model --- docs/research-databases-and-storage.md | 336 +++++++++++++++++++++++-- 1 file changed, 319 insertions(+), 17 deletions(-) diff --git a/docs/research-databases-and-storage.md b/docs/research-databases-and-storage.md index b27ac9e..d91259b 100644 --- a/docs/research-databases-and-storage.md +++ b/docs/research-databases-and-storage.md @@ -30,12 +30,91 @@ The first implementation should focus on historical market data and the basic en ## Storage Candidates -| Option | Best Use Case | Strengths | Limitations | Fit for ARGUS | -|---|---|---|---|---| -| SQLite | Small local app storage | Very simple, serverless, good for settings and watchlists | Less analytics-focused | Good later for app-state, not first choice | -| DuckDB | Local analytical storage | SQL-based, local, strong for analytical queries, good with Python/notebooks | Not a server database | Best first choice | -| PostgreSQL | Server/product database | Strong relational database, good for web apps, users, reports and cloud setups | More setup and infrastructure | Very good later | -| Parquet | Export/archive format | Efficient columnar format for analytical data | Not a database by itself | Useful later, not first storage layer | +ARGUS should compare storage options based on the current project phase. + +The project currently needs local analytical storage, not a full server or cloud database. + +### DuckDB + +DuckDB is a local analytical database. + +It is a strong fit for ARGUS because it supports SQL-based analytics without requiring a database server. + +Useful for: + +* historical market data +* local time-series analysis +* SQL practice +* Python-based analytics +* notebook-based exploration +* dashboard data preparation + +Limitations: + +* not a server database +* not directly supported by SQLGate +* less suitable for multi-user product features later + +Fit for ARGUS: + +DuckDB is the best first storage choice because ARGUS currently needs local analytics, not server infrastructure. + +--- + +### SQLite + +SQLite is a simple local database. + +It is strong for small app storage and simple persistence. + +Useful for: + +* settings +* small app-state data +* simple local tables +* later watchlists +* lightweight metadata + +Limitations: + +* less analytics-focused than DuckDB +* not ideal as the main storage layer for historical market data +* better for app-state than analytical time-series queries + +Fit for ARGUS: + +SQLite is useful later if ARGUS needs simple app-state storage, but it should not be the first storage choice for market analytics. + +--- + +### PostgreSQL + +PostgreSQL is a server-based relational database. + +It is a strong long-term option when ARGUS becomes more product-like. + +Useful for: + +* server-based storage +* user-facing features +* report history +* watchlists +* paper-trading history +* richer metadata +* cloud-ready architecture +* SQLGate usage later + +Limitations: + +* more setup than needed right now +* requires server or Docker setup +* adds infrastructure complexity too early + +Fit for ARGUS: + +PostgreSQL should be introduced later when ARGUS moves toward a server-based or cloud-ready architecture. + +It should not be selected first only because SQLGate is available. --- @@ -43,8 +122,8 @@ The first implementation should focus on historical market data and the basic en | Option | Meaning | Fit Now | Fit Later | |---|---|---:|---:| -| Local storage | Database or files run locally in the project | High | High | -| Server database | Database runs as a separate service, e.g. PostgreSQL | Medium | High | +| Local storage | Database runs locally inside or next to the project | High | High | +| Server database | Database runs as a separate service, for example PostgreSQL | Medium | High | | Cloud storage/database | Managed storage or database in the cloud | Low | High | ARGUS should start with local storage. @@ -61,22 +140,69 @@ Server and cloud storage should come later when ARGUS has stronger product featu --- -## Recommended Decision +## Recommended First Storage Approach DuckDB should be the first storage technology for ARGUS. -Why: +Reason: -* ARGUS currently needs local analytical storage -* DuckDB is designed for analytical SQL workflows -* it does not require a database server +* ARGUS currently needs local analytical storage, not a full server database +* DuckDB fits historical time-series analysis well +* it supports SQL-based analytics without requiring a database server * it works well with Python and notebook-based exploration -* it fits historical time-series and market-data analysis -* it keeps the first implementation manageable +* it keeps the first storage implementation manageable +* it can later be replaced or complemented by PostgreSQL if ARGUS becomes more product-like + +The first storage implementation should focus on: + +* historical market data +* cleaned OHLCV-ready price data +* source information +* instruments that ARGUS can analyze + +PostgreSQL and SQLGate become more relevant later. + +For the first DuckDB phase, the goal is to build a clean local analytics workflow. + +--- + +## Developer Interaction Workflow + +ARGUS should use a practical developer workflow for DuckDB. + +The goal is to make the database easy to inspect, explore and validate before logic is moved into production code. + +### Notebook Exploration + +Notebooks should be the main exploration layer. -PostgreSQL should be introduced later when ARGUS moves toward a more product-like architecture. +They are useful for: -SQLGate should also be kept for that later PostgreSQL phase, not for the first DuckDB phase. +* opening the DuckDB database +* testing SQL queries +* validating imported data +* comparing SQL results with pandas calculations +* exploring metric logic +* documenting research assumptions + +This workflow is especially useful before turning queries into reusable project code. + +Notebook exploration should be preferred over a GUI database tool in the first phase. + +### DuckDB CLI + +The DuckDB CLI should be used for quick database inspection. + +It is useful for: + +* checking available tables +* running small SQL queries +* validating stored records +* debugging the local database file + +The CLI is not the main research environment, but it is useful as a fast inspection tool. + +A GUI tool such as DBeaver can be tested if needed, but it should stay optional. --- @@ -95,3 +221,179 @@ data_sources instruments price_bars ``` + +### data_sources + +Stores where data came from. + +Recommended first fields: + +```text +id +name +provider_kind +requires_api_key +created_at +updated_at +``` + +Example: + +| name | provider_kind | requires_api_key | +|---|---|---:| +| Frankfurter | fx_rates | false | +| yfinance | market_prices | false | +| FRED | macro_data | true | + +### instruments + +Stores what ARGUS can analyze. + +Examples: + +* EUR/USD +* AAPL +* SPY +* S&P 500 +* BTC-USD + +Recommended first fields: + +```text +id +symbol +name +asset_class +currency +exchange +base_currency +quote_currency +created_at +updated_at +``` + +Example: + +| symbol | name | asset_class | currency | exchange | base_currency | quote_currency | +|---|---|---|---|---|---|---| +| EUR/USD | Euro / US Dollar | fx | null | null | EUR | USD | +| AAPL | Apple Inc. | stock | USD | NASDAQ | null | null | +| SPY | SPDR S&P 500 ETF | etf | USD | NYSE Arca | null | null | + +### price_bars + +Stores historical market data in an OHLCV-ready structure. + +Recommended first fields: + +```text +id +instrument_id +source_id +timestamp +timeframe +open +high +low +close +adjusted_close +volume +created_at +updated_at +``` + +For Frankfurter, the exchange rate can be stored in `close`. + +The other OHLCV fields can stay empty until ARGUS uses data sources that provide them. + +Example: + +| symbol | timestamp | timeframe | open | high | low | close | adjusted_close | volume | +|---|---|---|---:|---:|---:|---:|---:|---:| +| EUR/USD | 2024-01-02 | 1d | null | null | null | 1.095 | null | null | +| AAPL | 2024-01-02 | 1d | 187.15 | 188.44 | 183.89 | 185.64 | 184.25 | 50200000 | + +--- + +## Recommended First Implementation Step + +The first storage implementation should be small and focused. + +Recommended first step: + +```text +Frankfurter data +→ normalize into instruments and price_bars +→ store in DuckDB +→ query with SQL +→ use results for analytics and charts +``` + +Recommended first tables: + +```text +data_sources +instruments +price_bars +``` + +This gives ARGUS a useful storage foundation without adding unnecessary product-level complexity too early. + +--- + +## Future Direction + +Later sprints can expand the storage layer step by step. + +Possible later additions: + +| Future Area | Possible Additions | +|---|---| +| Better source mapping | source-specific symbols, provider metadata | +| Watchlists | user-selected instruments | +| Reports | generated report metadata and history | +| Macro data | FRED indicators and observations | +| Paper trading | simulated orders, positions and portfolio history | +| Server architecture | PostgreSQL | +| SQL tooling | SQLGate with PostgreSQL | +| Cloud direction | managed PostgreSQL or cloud storage | + +SQLGate should be kept for a later PostgreSQL phase. + +It becomes useful when ARGUS moves toward: + +* server-based storage +* stronger database management +* richer metadata +* more stable application state +* user-facing features +* report history +* cloud-ready architecture + +Additional metadata such as documentation links, terms links or provider governance fields can also become useful later. + +For the first DuckDB phase, these details should stay in research documentation instead of the database schema. + +--- + +## Final Recommendation + +ARGUS should start with DuckDB as the first local analytics storage layer. + +DuckDB fits the current phase best because ARGUS needs local analytical SQL workflows, not a full server database yet. + +The first implementation should store historical market data in an OHLCV-ready structure. + +The recommended first data model is: + +```text +data_sources +instruments +price_bars +``` + +Notebook exploration should be the main developer workflow before SQL logic is moved into application code. + +The DuckDB CLI can be used for quick inspection. + +PostgreSQL and SQLGate should be introduced later when ARGUS moves toward a more product-like or cloud-based architecture. \ No newline at end of file From 934bee84f88da5e7d56acb422343cd488c709e52 Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Thu, 25 Jun 2026 15:32:27 +0200 Subject: [PATCH 11/44] docs(#37): polish the first step --- docs/research-databases-and-storage.md | 32 +++++++------------------- 1 file changed, 8 insertions(+), 24 deletions(-) diff --git a/docs/research-databases-and-storage.md b/docs/research-databases-and-storage.md index d91259b..2061a29 100644 --- a/docs/research-databases-and-storage.md +++ b/docs/research-databases-and-storage.md @@ -52,13 +52,8 @@ Useful for: Limitations: * not a server database -* not directly supported by SQLGate * less suitable for multi-user product features later -Fit for ARGUS: - -DuckDB is the best first storage choice because ARGUS currently needs local analytics, not server infrastructure. - --- ### SQLite @@ -81,10 +76,6 @@ Limitations: * not ideal as the main storage layer for historical market data * better for app-state than analytical time-series queries -Fit for ARGUS: - -SQLite is useful later if ARGUS needs simple app-state storage, but it should not be the first storage choice for market analytics. - --- ### PostgreSQL @@ -114,8 +105,6 @@ Fit for ARGUS: PostgreSQL should be introduced later when ARGUS moves toward a server-based or cloud-ready architecture. -It should not be selected first only because SQLGate is available. - --- ## Local, Server and Cloud Options @@ -317,28 +306,23 @@ Example: ## Recommended First Implementation Step -The first storage implementation should be small and focused. +The first storage implementation should not be tied to one specific data provider. + +ARGUS currently works with an existing ExchangeRate API client and evaluates broader market data through yfinance. +Frankfurter may be added later as a stronger FX-oriented historical data source. + +The storage layer should therefore focus on a normalized internal market-data format instead of depending on one API response structure. Recommended first step: ```text -Frankfurter data +active data client → normalize into instruments and price_bars → store in DuckDB → query with SQL → use results for analytics and charts ``` -Recommended first tables: - -```text -data_sources -instruments -price_bars -``` - -This gives ARGUS a useful storage foundation without adding unnecessary product-level complexity too early. - --- ## Future Direction @@ -396,4 +380,4 @@ Notebook exploration should be the main developer workflow before SQL logic is m The DuckDB CLI can be used for quick inspection. -PostgreSQL and SQLGate should be introduced later when ARGUS moves toward a more product-like or cloud-based architecture. \ No newline at end of file +PostgreSQL and SQLGate should be introduced later when ARGUS moves toward a more product-like or cloud-based architecture. From fc6c7cf4ac2d11dac9deda8f9dd6d860c16e22f8 Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Thu, 25 Jun 2026 16:01:24 +0200 Subject: [PATCH 12/44] docs(#41): add dockerfile --- dockerfile | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 dockerfile diff --git a/dockerfile b/dockerfile new file mode 100644 index 0000000..9f0eabd --- /dev/null +++ b/dockerfile @@ -0,0 +1,12 @@ +FROM python:3.11.18-slim + +WORKDIR /app + +COPY pyproject.toml README.md ./ +COPY src/ ./src/ +COPY tests/ ./tests/ + +RUN python -m pip install --upgrade pip \ + && pip install -e ".[dev]" + +CMD ["pytest"] \ No newline at end of file From 549ffafbb3d82ffb4d81899afbd512a8feaad0b1 Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Thu, 25 Jun 2026 16:20:44 +0200 Subject: [PATCH 13/44] docs(#41): add dockerignore --- .dockerignore | 19 +++++++++++++++++++ dockerfile | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..a97f748 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,19 @@ +//Git Stuff +.git +.github +.githooks +.gitignore + +// Pyhon Stuff +.venv +__pycache__ +.pytest_cache + +-- Portfolio stuff +Contributing.md +Licence.md +README.md +CODE_OD_CONDUCT.md +.ruff_cache +docs +.env diff --git a/dockerfile b/dockerfile index 9f0eabd..1dce2e8 100644 --- a/dockerfile +++ b/dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11.18-slim +FROM python:3.11-slim WORKDIR /app From 6e921cb15f258dc06c6a748e9c83493f3b3a1c5e Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Thu, 25 Jun 2026 16:28:16 +0200 Subject: [PATCH 14/44] docs(#41): fix dockerignore --- .dockerignore | 26 ++++++++++++++++---------- README.md | 13 +++++++++++++ 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/.dockerignore b/.dockerignore index a97f748..8c24b79 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,19 +1,25 @@ -//Git Stuff +# Git .git .github .githooks .gitignore -// Pyhon Stuff +# Local environment and secrets .venv +.env + +# Python cache and test cache __pycache__ .pytest_cache - --- Portfolio stuff -Contributing.md -Licence.md -README.md -CODE_OD_CONDUCT.md .ruff_cache -docs -.env +.mypy_cache +.coverage +htmlcov + +# Build artifacts +dist +build +*.egg-info + +# Project docs not needed for the test image +docs \ No newline at end of file diff --git a/README.md b/README.md index 607f0d0..54180b0 100644 --- a/README.md +++ b/README.md @@ -192,6 +192,7 @@ Before running ARGUS locally, make sure you have: - Python 3.11 or newer - Git - pip +- docker - an ExchangeRate API key for live currency conversion. Historical analytics currently use yfinance and do not require an additional API key. Recommended for development: @@ -321,6 +322,18 @@ pytest --- +## Docker + +ARGUS can also be tested inside a minimal Docker container. + +Build the Docker image: + +```bash +docker build -t argus . +``` + +--- + ## Documentation More detailed project documentation lives in [`docs/`](docs/). From 00609455e98655abcda42e358a325c0a4d16035f Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Thu, 25 Jun 2026 16:46:16 +0200 Subject: [PATCH 15/44] docs(#41): update readme --- README.md | 43 +++++++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 54180b0..007b8e9 100644 --- a/README.md +++ b/README.md @@ -200,6 +200,7 @@ Recommended for development: - VS Code - a virtual environment - pytest +- Docker, if you want to run tests in an isolated container environment > [!NOTE] > Runtime dependencies are managed through `pyproject.toml`. @@ -275,7 +276,17 @@ Add your API key: EXCHANGE_RATE_API_KEY=your_api_key_here ``` -### 3. Keep secrets private +### 3. Docker + +ARGUS can also be tested inside a minimal Docker container. + +Build the Docker image: + +```bash +docker build -t argus . +``` + +### 4. Keep secrets private The `.env` file must stay local and should never be committed. @@ -285,7 +296,7 @@ The `.env` file must stay local and should never be committed. --- -## Running ARGUS +## Running ARGUS Locally Start the current Tkinter GUI: @@ -295,6 +306,22 @@ python -m argus.main This starts the local ARGUS prototype with calculator, currency conversion and basic analytics views. +## Running Argus in Docker + +ARGUS includes a minimal Docker setup for running the test suite in an isolated container environment. + +Build the Docker image: + +```bash +docker build -t argus . +``` + +Run ARGUS in a container: + +```bash +docker run --rm argus +``` + ### Legacy CLI / Debug Interface The legacy CLI is still available for quick local checks and debugging: @@ -322,18 +349,6 @@ pytest --- -## Docker - -ARGUS can also be tested inside a minimal Docker container. - -Build the Docker image: - -```bash -docker build -t argus . -``` - ---- - ## Documentation More detailed project documentation lives in [`docs/`](docs/). From e1d4b65980f059e7c075547f7739a580a5f30c63 Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Thu, 25 Jun 2026 17:03:07 +0200 Subject: [PATCH 16/44] docs(#41): improve readme --- README.md | 25 ++++--------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 007b8e9..213fe1d 100644 --- a/README.md +++ b/README.md @@ -120,7 +120,7 @@ README.md - Tkinter - pytest -### Current data source +### Current data sources - ExchangeRate API for live currency conversion - yfinance for historical market-data retrieval and analytics @@ -140,8 +140,6 @@ Planned or likely future technologies include: ### Data processing -- pandas -- NumPy - possibly Polars later for larger datasets ### Storage @@ -153,14 +151,10 @@ Planned or likely future technologies include: ### Visualization and UI -- matplotlib -- Plotly - NiceGUI ### DevOps and deployment -- GitHub Actions -- Docker - Docker Compose - cloud deployment later @@ -192,7 +186,6 @@ Before running ARGUS locally, make sure you have: - Python 3.11 or newer - Git - pip -- docker - an ExchangeRate API key for live currency conversion. Historical analytics currently use yfinance and do not require an additional API key. Recommended for development: @@ -256,7 +249,7 @@ pip install -e ".[dev]" ## API Key Setup -ARGUS currently uses the ExchangeRate API for live currency conversion. +ARGUS uses the ExchangeRate API for live currency conversion. Historical analytics currently use yfinance and do not require an additional API key. ### 1. Create an API key @@ -276,17 +269,7 @@ Add your API key: EXCHANGE_RATE_API_KEY=your_api_key_here ``` -### 3. Docker - -ARGUS can also be tested inside a minimal Docker container. - -Build the Docker image: - -```bash -docker build -t argus . -``` - -### 4. Keep secrets private +### 3. Keep secrets private The `.env` file must stay local and should never be committed. @@ -338,7 +321,7 @@ python src/legacy/debug_main.py ## Running Tests -Run the test suite: +Run the test suite locally: ```bash pytest From 1bc6159c3b0906ad564c464046f3ea3d55c45fcf Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Mon, 29 Jun 2026 10:03:51 +0200 Subject: [PATCH 17/44] feat(#42): add DataSource --- src/argus/domain/internal_models.py | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 src/argus/domain/internal_models.py diff --git a/src/argus/domain/internal_models.py b/src/argus/domain/internal_models.py new file mode 100644 index 0000000..4c6d565 --- /dev/null +++ b/src/argus/domain/internal_models.py @@ -0,0 +1,8 @@ +from dataclasses import dataclass + + +@dataclass +class DataSource: + name: str + provider_kind: str + requires_api_key: bool = False \ No newline at end of file From c5d30dc58a430003228a94281723c34c7fffa927 Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Mon, 29 Jun 2026 10:21:32 +0200 Subject: [PATCH 18/44] feat(#42): add two more models --- ...-data-sources => research-data-sources.md} | 0 src/argus/domain/internal_models.py | 26 ++++++++++++++++--- 2 files changed, 23 insertions(+), 3 deletions(-) rename docs/{research-data-sources => research-data-sources.md} (100%) diff --git a/docs/research-data-sources b/docs/research-data-sources.md similarity index 100% rename from docs/research-data-sources rename to docs/research-data-sources.md diff --git a/src/argus/domain/internal_models.py b/src/argus/domain/internal_models.py index 4c6d565..a8a3018 100644 --- a/src/argus/domain/internal_models.py +++ b/src/argus/domain/internal_models.py @@ -1,8 +1,28 @@ from dataclasses import dataclass - - +from datetime import datetime,date @dataclass class DataSource: name: str provider_kind: str - requires_api_key: bool = False \ No newline at end of file + requires_api_key: bool = False + +@dataclass +class Instruments: + symbol: str + name: str + assetclass: str + currency: str + exchange: str + base_currency: str + quote_currency: str + +@dataclass +class PriveBars: + timestamp: date + timeframe: str + open: float + high: float + low: float + close: float + adjusted_close: float + volume: float \ No newline at end of file From d53e56cc7081336cf1b072726b853ec7a31f5f78 Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Mon, 29 Jun 2026 10:52:54 +0200 Subject: [PATCH 19/44] docs(#42): update research --- docs/research-databases-and-storage.md | 50 ++++++++++++++------------ src/argus/domain/internal_models.py | 32 ++++++++++------- 2 files changed, 47 insertions(+), 35 deletions(-) diff --git a/docs/research-databases-and-storage.md b/docs/research-databases-and-storage.md index 2061a29..217ee46 100644 --- a/docs/research-databases-and-storage.md +++ b/docs/research-databases-and-storage.md @@ -203,7 +203,7 @@ ARGUS should not use a narrow `date | value` table as the main market-data model That would work for simple exchange rates, but it would become limiting once ARGUS adds stocks, ETFs, indices or broader market APIs. -The first model should focus on three tables: +The first model should focus on three related entities: ```text data_sources @@ -211,11 +211,16 @@ instruments price_bars ``` +> [!NOTE] +> The fields below describe the future database-oriented structure. +> Technical fields such as `id`, `instrument_id`, `source_id`, `created_at` and `updated_at` are expected to appear in the database layer. +> Internal Python models may reference related objects directly, for example `source` and `instrument`, before database IDs exist. + ### data_sources Stores where data came from. -Recommended first fields: +Recommended first database fields: ```text id @@ -226,13 +231,13 @@ created_at updated_at ``` -Example: +Example internal/source records: -| name | provider_kind | requires_api_key | -|---|---|---:| -| Frankfurter | fx_rates | false | -| yfinance | market_prices | false | -| FRED | macro_data | true | +| name | provider_kind | requires_api_key | +| ---------------- | ------------- | ---------------: | +| ExchangeRate API | fx_rates | true | +| yfinance | market_prices | false | +| FRED | macro_data | true | ### instruments @@ -246,7 +251,7 @@ Examples: * S&P 500 * BTC-USD -Recommended first fields: +Recommended first database fields: ```text id @@ -261,19 +266,19 @@ created_at updated_at ``` -Example: +Example instrument records: -| symbol | name | asset_class | currency | exchange | base_currency | quote_currency | -|---|---|---|---|---|---|---| -| EUR/USD | Euro / US Dollar | fx | null | null | EUR | USD | -| AAPL | Apple Inc. | stock | USD | NASDAQ | null | null | -| SPY | SPDR S&P 500 ETF | etf | USD | NYSE Arca | null | null | +| symbol | name | asset_class | currency | exchange | base_currency | quote_currency | +| ------- | ---------------- | ----------- | -------- | --------- | ------------- | -------------- | +| EUR/USD | Euro / US Dollar | fx | null | null | EUR | USD | +| AAPL | Apple Inc. | stock | USD | NASDAQ | null | null | +| SPY | SPDR S&P 500 ETF | etf | USD | NYSE Arca | null | null | ### price_bars Stores historical market data in an OHLCV-ready structure. -Recommended first fields: +Recommended first database fields: ```text id @@ -291,19 +296,20 @@ created_at updated_at ``` -For Frankfurter, the exchange rate can be stored in `close`. +FX-style exchange-rate data can be represented as a price bar by storing the rate in `close`. The other OHLCV fields can stay empty until ARGUS uses data sources that provide them. -Example: +Example price bar records shown with joined source and instrument information for readability: -| symbol | timestamp | timeframe | open | high | low | close | adjusted_close | volume | -|---|---|---|---:|---:|---:|---:|---:|---:| -| EUR/USD | 2024-01-02 | 1d | null | null | null | 1.095 | null | null | -| AAPL | 2024-01-02 | 1d | 187.15 | 188.44 | 183.89 | 185.64 | 184.25 | 50200000 | +| source | instrument | timestamp | timeframe | open | high | low | close | adjusted_close | volume | +| -------- | ---------- | ---------- | --------- | -----: | -----: | -----: | -----: | -------------: | -------: | +| yfinance | EUR/USD | 2024-01-02 | 1d | null | null | null | 1.095 | null | null | +| yfinance | AAPL | 2024-01-02 | 1d | 187.15 | 188.44 | 183.89 | 185.64 | 184.25 | 50200000 | --- + ## Recommended First Implementation Step The first storage implementation should not be tied to one specific data provider. diff --git a/src/argus/domain/internal_models.py b/src/argus/domain/internal_models.py index a8a3018..16e9755 100644 --- a/src/argus/domain/internal_models.py +++ b/src/argus/domain/internal_models.py @@ -1,28 +1,34 @@ from dataclasses import dataclass -from datetime import datetime,date +from datetime import datetime, date + + @dataclass class DataSource: name: str provider_kind: str requires_api_key: bool = False + @dataclass -class Instruments: +class Instrument: symbol: str name: str - assetclass: str - currency: str - exchange: str - base_currency: str - quote_currency: str + asset_class: str + currency: str | None = None + exchange: str | None = None + base_currency: str | None = None + quote_currency: str | None = None + @dataclass -class PriveBars: +class PriceBar: + source: DataSource + instrument: Instrument timestamp: date timeframe: str - open: float - high: float - low: float close: float - adjusted_close: float - volume: float \ No newline at end of file + open: float | None = None + high: float | None = None + low: float | None = None + adjusted_close: float | None = None + volume: float | None = None From 1fcf1a6f8f52fdccef9150c14ffb6c022476bec0 Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Mon, 29 Jun 2026 10:57:18 +0200 Subject: [PATCH 20/44] docs(#42): fix markdown and etc --- README.md | 2 +- docs/forecast_research.md | 12 ++++++++++++ docs/research-databases-and-storage.md | 5 ++--- src/argus/domain/internal_models.py | 2 +- 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 213fe1d..1c0d156 100644 --- a/README.md +++ b/README.md @@ -357,4 +357,4 @@ Current focus: - add stronger market metrics - expand pandas-based analytics workflows - improve dashboard usefulness without adding unnecessary chart noise -- document metric definitions, assumptions and data-source behavior \ No newline at end of file +- document metric definitions, assumptions and data-source behavior diff --git a/docs/forecast_research.md b/docs/forecast_research.md index 5db8c1b..b0cc9b7 100644 --- a/docs/forecast_research.md +++ b/docs/forecast_research.md @@ -1,30 +1,40 @@ # Research: First Forecasting Approach for Market Time Series ## 1. Realistic First Prediction Task for ARGUS + A realistic first prediction task for ARGUS is **next-day exchange-rate movement** or **trend direction**. Predicting the exact next value (point forecast) is generally much harder and often less useful for trading/signal workflows than predicting the direction of the movement (up/down). A directional classification task serves as a simple, actionable signal for basic workflows. ## 2. Baseline Methods to Implement First + Before jumping into complex models, the following baselines should be implemented to evaluate the added value of any machine learning model: + - **Naive last-value forecast**: The prediction for the next period is exactly the value from the current period. This is surprisingly hard to beat in random walk-like financial time series. - **Moving average forecast**: A simple rolling average to predict the next value or determine trend direction. - **Simple linear regression**: To capture basic linear trends over a given historical window. ## 3. Libraries: NumPy, pandas, or scikit-learn? + The first implementation should use **pandas** and **scikit-learn**: + - **pandas**: Excellent for time-series manipulation, rolling windows, lagging features, and handling missing data. - **scikit-learn**: Offers robust implementations of simple models (e.g., Linear Regression, Logistic Regression for direction) and provides standardized metrics and cross-validation tools designed for time series (e.g., `TimeSeriesSplit`). ## 4. Evaluation Metrics + For the initial approaches, we should focus on: + - **Directional accuracy**: The percentage of times the model correctly predicts the direction of the price movement (up vs down). This is often more relevant than magnitude errors. - **MAE (Mean Absolute Error)**: If point forecasting is used, MAE is more robust to outliers than RMSE and provides a linear penalty for errors. - **RMSE (Root Mean Squared Error)**: Useful to penalize larger errors more heavily, but should be secondary to directional accuracy for basic signal generation. ## 5. Why is LSTM not the first implementation step? + LSTMs are highly complex, require a large amount of well-structured data to train effectively without overfitting, and are notoriously difficult to tune. For financial time series, which suffer from low signal-to-noise ratios, an LSTM is likely to overfit the training data or collapse to predicting the last known value. Starting with an LSTM obscures whether the underlying data has any predictive power and sets a high barrier for debugging and infrastructure. ## 6. Prerequisites for an LSTM Ticket + Before considering LSTMs or other deep learning approaches, the following must be established: + - A reliable data ingestion and preprocessing pipeline. - Established baseline performance metrics (e.g., a naive model and a linear regression model) to compare against. - Sufficient historical data size. @@ -32,7 +42,9 @@ Before considering LSTMs or other deep learning approaches, the following must b - Hardware/infrastructure to support longer training times and hyperparameter tuning. ## 7. Recommended First Implementation Approach + **Recommendation**: Start with **directional trend prediction** (predicting whether the next value is higher or lower than the current value) using a simple **Logistic Regression** model via **scikit-learn**. + - Use **pandas** to create basic lagged features (e.g., previous returns, moving averages). - Evaluate using **directional accuracy**. - Compare performance strictly against a **naive momentum** (predicting the trend continues) or **majority-class** baseline. diff --git a/docs/research-databases-and-storage.md b/docs/research-databases-and-storage.md index 217ee46..484cd61 100644 --- a/docs/research-databases-and-storage.md +++ b/docs/research-databases-and-storage.md @@ -110,7 +110,7 @@ PostgreSQL should be introduced later when ARGUS moves toward a server-based or ## Local, Server and Cloud Options | Option | Meaning | Fit Now | Fit Later | -|---|---|---:|---:| +| --- | --- | ---: | ---: | | Local storage | Database runs locally inside or next to the project | High | High | | Server database | Database runs as a separate service, for example PostgreSQL | Medium | High | | Cloud storage/database | Managed storage or database in the cloud | Low | High | @@ -309,7 +309,6 @@ Example price bar records shown with joined source and instrument information fo --- - ## Recommended First Implementation Step The first storage implementation should not be tied to one specific data provider. @@ -338,7 +337,7 @@ Later sprints can expand the storage layer step by step. Possible later additions: | Future Area | Possible Additions | -|---|---| +| --- | --- | | Better source mapping | source-specific symbols, provider metadata | | Watchlists | user-selected instruments | | Reports | generated report metadata and history | diff --git a/src/argus/domain/internal_models.py b/src/argus/domain/internal_models.py index 16e9755..3b7630e 100644 --- a/src/argus/domain/internal_models.py +++ b/src/argus/domain/internal_models.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from datetime import datetime, date +from datetime import date @dataclass From cef29404dd3a5d7a087a1528e256c6ad6e3eaa6f Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Mon, 29 Jun 2026 11:53:42 +0200 Subject: [PATCH 21/44] test(#42): tests for models --- src/argus/domain/internal_models.py | 2 +- tests/test_internal_models.py | 60 +++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 tests/test_internal_models.py diff --git a/src/argus/domain/internal_models.py b/src/argus/domain/internal_models.py index 3b7630e..c10eaaa 100644 --- a/src/argus/domain/internal_models.py +++ b/src/argus/domain/internal_models.py @@ -31,4 +31,4 @@ class PriceBar: high: float | None = None low: float | None = None adjusted_close: float | None = None - volume: float | None = None + volume: float | None = None \ No newline at end of file diff --git a/tests/test_internal_models.py b/tests/test_internal_models.py new file mode 100644 index 0000000..d99afd5 --- /dev/null +++ b/tests/test_internal_models.py @@ -0,0 +1,60 @@ +from argus.domain.internal_models import DataSource,Instrument,PriceBar +from datetime import date + +def test_data_source_can_be_created() -> None: + source = DataSource( + name="yfinance", + provider_kind="fx_rates", + ) + + assert source.name == "yfinance" + assert source.provider_kind == "fx_rates" + assert source.requires_api_key is False + +def test_instrument_can_be_created() -> None: + instrument = Instrument( + symbol="EUR/USD", + name="Euro / US Dollar", + asset_class="fx", + base_currency="EUR", + quote_currency="USD" + ) + + assert instrument.symbol == "EUR/USD" + assert instrument.name == "Euro / US Dollar" + assert instrument.asset_class == "fx" + assert instrument.base_currency == "EUR" + assert instrument.quote_currency == "USD" + assert instrument.currency is None + assert instrument.exchange is None + +def test_price_bar_can_be_created() -> None: + source = DataSource( + name="yfinance", + provider_kind="fx_rates", + ) + instrument_rate = Instrument( + symbol="EUR/USD", + name="Euro / US Dollar", + asset_class="fx", + base_currency="EUR", + quote_currency="USD" + ) + pricebar = PriceBar( + source=source, + instrument=instrument_rate, + timestamp=date(2026, 1, 1), + timeframe="1D", + close=1.89 + ) + + assert pricebar.source == "yfinance" + assert pricebar.instrument == "fx_rates" + assert pricebar.timestamp == date(2026, 1, 1) + assert pricebar.timeframe == "1D" + assert pricebar.close == 1.89 + assert pricebar.open is None + assert pricebar.high is None + assert pricebar.low is None + assert pricebar.adjusted_close is None + assert pricebar.volume is None \ No newline at end of file From 70695b67c6bf47839d1dbf1e1ddf95118abc74e6 Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Mon, 29 Jun 2026 11:57:06 +0200 Subject: [PATCH 22/44] test(#42): fix tests --- tests/test_internal_models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_internal_models.py b/tests/test_internal_models.py index d99afd5..e1e8b40 100644 --- a/tests/test_internal_models.py +++ b/tests/test_internal_models.py @@ -48,8 +48,8 @@ def test_price_bar_can_be_created() -> None: close=1.89 ) - assert pricebar.source == "yfinance" - assert pricebar.instrument == "fx_rates" + assert pricebar.source == source + assert pricebar.instrument == instrument_rate assert pricebar.timestamp == date(2026, 1, 1) assert pricebar.timeframe == "1D" assert pricebar.close == 1.89 From a0a503f55dd348dbf8e4924401790e1068f43cc8 Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Mon, 29 Jun 2026 12:04:51 +0200 Subject: [PATCH 23/44] test(#42): a test for OHCLV --- src/argus/domain/internal_models.py | 2 +- tests/test_internal_models.py | 51 ++++++++++++++++++++++++++--- 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/src/argus/domain/internal_models.py b/src/argus/domain/internal_models.py index c10eaaa..3b7630e 100644 --- a/src/argus/domain/internal_models.py +++ b/src/argus/domain/internal_models.py @@ -31,4 +31,4 @@ class PriceBar: high: float | None = None low: float | None = None adjusted_close: float | None = None - volume: float | None = None \ No newline at end of file + volume: float | None = None diff --git a/tests/test_internal_models.py b/tests/test_internal_models.py index e1e8b40..2a032db 100644 --- a/tests/test_internal_models.py +++ b/tests/test_internal_models.py @@ -1,6 +1,7 @@ -from argus.domain.internal_models import DataSource,Instrument,PriceBar +from argus.domain.internal_models import DataSource, Instrument, PriceBar from datetime import date + def test_data_source_can_be_created() -> None: source = DataSource( name="yfinance", @@ -11,13 +12,14 @@ def test_data_source_can_be_created() -> None: assert source.provider_kind == "fx_rates" assert source.requires_api_key is False + def test_instrument_can_be_created() -> None: instrument = Instrument( symbol="EUR/USD", name="Euro / US Dollar", asset_class="fx", base_currency="EUR", - quote_currency="USD" + quote_currency="USD", ) assert instrument.symbol == "EUR/USD" @@ -28,24 +30,27 @@ def test_instrument_can_be_created() -> None: assert instrument.currency is None assert instrument.exchange is None + def test_price_bar_can_be_created() -> None: source = DataSource( name="yfinance", provider_kind="fx_rates", ) + instrument_rate = Instrument( symbol="EUR/USD", name="Euro / US Dollar", asset_class="fx", base_currency="EUR", - quote_currency="USD" + quote_currency="USD", ) + pricebar = PriceBar( source=source, instrument=instrument_rate, timestamp=date(2026, 1, 1), timeframe="1D", - close=1.89 + close=1.89, ) assert pricebar.source == source @@ -57,4 +62,40 @@ def test_price_bar_can_be_created() -> None: assert pricebar.high is None assert pricebar.low is None assert pricebar.adjusted_close is None - assert pricebar.volume is None \ No newline at end of file + assert pricebar.volume is None + + +def test_stock_ohlcv_data_can_be_represented_as_price_bar() -> None: + source = DataSource( + name="yfinance", + provider_kind="market_prices", + ) + + instrument = Instrument( + symbol="AAPL", + name="Apple Inc.", + asset_class="stock", + currency="USD", + exchange="NASDAQ", + ) + + price_bar = PriceBar( + source=source, + instrument=instrument, + timestamp=date(2026, 1, 1), + timeframe="1d", + open=187.15, + high=188.44, + low=183.89, + close=185.64, + adjusted_close=184.25, + volume=50_200_000, + ) + + assert price_bar.instrument.symbol == "AAPL" + assert price_bar.open == 187.15 + assert price_bar.high == 188.44 + assert price_bar.low == 183.89 + assert price_bar.close == 185.64 + assert price_bar.adjusted_close == 184.25 + assert price_bar.volume == 50_200_000 From 0e86e91631bd83283c8021d4e09ec51ea3890595 Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Mon, 29 Jun 2026 12:10:46 +0200 Subject: [PATCH 24/44] style(#42): edit test names --- tests/test_internal_models.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/test_internal_models.py b/tests/test_internal_models.py index 2a032db..ae477fc 100644 --- a/tests/test_internal_models.py +++ b/tests/test_internal_models.py @@ -31,7 +31,7 @@ def test_instrument_can_be_created() -> None: assert instrument.exchange is None -def test_price_bar_can_be_created() -> None: +def test_rate_bar_can_be_created() -> None: source = DataSource( name="yfinance", provider_kind="fx_rates", @@ -45,27 +45,27 @@ def test_price_bar_can_be_created() -> None: quote_currency="USD", ) - pricebar = PriceBar( + price_bar = PriceBar( source=source, instrument=instrument_rate, timestamp=date(2026, 1, 1), - timeframe="1D", + timeframe="1d", close=1.89, ) - assert pricebar.source == source - assert pricebar.instrument == instrument_rate - assert pricebar.timestamp == date(2026, 1, 1) - assert pricebar.timeframe == "1D" - assert pricebar.close == 1.89 - assert pricebar.open is None - assert pricebar.high is None - assert pricebar.low is None - assert pricebar.adjusted_close is None - assert pricebar.volume is None + assert price_bar.source == source + assert price_bar.instrument == instrument_rate + assert price_bar.timestamp == date(2026, 1, 1) + assert price_bar.timeframe == "1D" + assert price_bar.close == 1.89 + assert price_bar.open is None + assert price_bar.high is None + assert price_bar.low is None + assert price_bar.adjusted_close is None + assert price_bar.volume is None -def test_stock_ohlcv_data_can_be_represented_as_price_bar() -> None: +def test_stock_ohlcv_bar_can_be_created() -> None: source = DataSource( name="yfinance", provider_kind="market_prices", From 9f342e707b5efed73c2859ff2672d4137bcbbb74 Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Mon, 29 Jun 2026 12:12:41 +0200 Subject: [PATCH 25/44] test(#42): fix a param --- tests/test_internal_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_internal_models.py b/tests/test_internal_models.py index ae477fc..97df4c6 100644 --- a/tests/test_internal_models.py +++ b/tests/test_internal_models.py @@ -56,7 +56,7 @@ def test_rate_bar_can_be_created() -> None: assert price_bar.source == source assert price_bar.instrument == instrument_rate assert price_bar.timestamp == date(2026, 1, 1) - assert price_bar.timeframe == "1D" + assert price_bar.timeframe == "1d" assert price_bar.close == 1.89 assert price_bar.open is None assert price_bar.high is None From 6f613b2835ad4ba362ad2840c4a5debc07a408bc Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Mon, 29 Jun 2026 12:41:19 +0200 Subject: [PATCH 26/44] feat(#38): create db foundation --- pyproject.toml | 1 + src/argus/storage/__init__.py | 0 src/argus/storage/database.py | 0 3 files changed, 1 insertion(+) create mode 100644 src/argus/storage/__init__.py create mode 100644 src/argus/storage/database.py diff --git a/pyproject.toml b/pyproject.toml index e2dc784..a8ac7b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,7 @@ dependencies = [ "numpy", "matplotlib", "yfinance", + "duckdb", ] [project.optional-dependencies] diff --git a/src/argus/storage/__init__.py b/src/argus/storage/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/argus/storage/database.py b/src/argus/storage/database.py new file mode 100644 index 0000000..e69de29 From d232222db347f0edcd1fa1b258f816a579e75b22 Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Mon, 29 Jun 2026 13:33:32 +0200 Subject: [PATCH 27/44] feat(#38): add initt_db func --- src/argus/storage/database.py | 42 +++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/argus/storage/database.py b/src/argus/storage/database.py index e69de29..b1a5317 100644 --- a/src/argus/storage/database.py +++ b/src/argus/storage/database.py @@ -0,0 +1,42 @@ +import duckdb + +def initialize_database(database_path: str) -> None: + create_datasources_table = """ + CREATE TABLE IF NOT EXISTS data_sources ( + id INTEGER PRIMARY_KEY, + name TEXT, + provider_kind TEXT, + requires_api_key: BOOLEAN DEFAULTS:False + ) + """ + create_intstrument_table = """ + CREATE TABLE IF NOT EXISTS instrument ( + id INTEGER PRIMARY_KEY, + name TEXT, + asset_class TEXT, + currency TEXT or NONE DEFAULTS: NONE, + exchange TEXT or NONE DEFAULTS: NONE, + base_currency TEXT or NONE DEFAULTS: NONE, + quote_currency TEXT or NONE DEFAULTS: NONE + ) + """ + create_price_bar_table = """ + CREATE TABLE IF NOT EXISTS price_bar ( + id INTEGER PRIMARY_KEY, + source_id FOREIGN_KEY, + instrument_id FOREIGN_KEY, + timestamp: date, + timeframe TEXT, + close FLOAT, + open: FLOAT or NONE DEFAULTS: NONE, + high: FLOAT or NONE DEFAULTS: NONE, + low: FLOAT or NONE DEFAULTS: NONE, + adjusted_close FLOAT or NONE DEFAULTS: NONE, + volume: FLOAT or NONE DEFAULTS: NONE + ) + """ + duckdb.connect(database_path) + duckdb.execute(query=create_datasources_table) + duckdb.execute(query=create_intstrument_table) + duckdb.execute(query=create_price_bar_table) + duckdb.close() From b5368eee5b067796033bc62971b21b0ce248cdd2 Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Mon, 29 Jun 2026 13:37:59 +0200 Subject: [PATCH 28/44] feat(#38): add id sequence --- src/argus/storage/database.py | 70 +++++++++++++++++++---------------- 1 file changed, 39 insertions(+), 31 deletions(-) diff --git a/src/argus/storage/database.py b/src/argus/storage/database.py index b1a5317..8e564b1 100644 --- a/src/argus/storage/database.py +++ b/src/argus/storage/database.py @@ -1,42 +1,50 @@ import duckdb + def initialize_database(database_path: str) -> None: + create_data_sources_sequence = """ + CREATE SEQUENCE IF NOT EXISTS data_sources_id_seq; + """ create_datasources_table = """ CREATE TABLE IF NOT EXISTS data_sources ( - id INTEGER PRIMARY_KEY, - name TEXT, - provider_kind TEXT, - requires_api_key: BOOLEAN DEFAULTS:False - ) + id INTEGER PRIMARY KEY DEFAULT keyval('data_sources_id_seq'), + name TEXT NOT NULL UNIQUE, + provider_kind TEXT NOT NULL, + requires_api_key BOOLEAN NOT NULL + ); """ - create_intstrument_table = """ - CREATE TABLE IF NOT EXISTS instrument ( - id INTEGER PRIMARY_KEY, - name TEXT, - asset_class TEXT, - currency TEXT or NONE DEFAULTS: NONE, - exchange TEXT or NONE DEFAULTS: NONE, - base_currency TEXT or NONE DEFAULTS: NONE, - quote_currency TEXT or NONE DEFAULTS: NONE - ) + create_intstruments_table = """ + CREATE TABLE IF NOT EXISTS instruments ( + id INTEGER PRIMARY KEY DEFAULT keyval('data_sources_id_seq'), + symbol TEXT NOT NULL UNIQUE, + name TEXT NOT NULL, + asset_class TEXT NOT NULL, + currency TEXT, + exchange TEXT, + base_currency TEXT, + quote_currency TEXT + ); """ - create_price_bar_table = """ - CREATE TABLE IF NOT EXISTS price_bar ( - id INTEGER PRIMARY_KEY, - source_id FOREIGN_KEY, - instrument_id FOREIGN_KEY, - timestamp: date, - timeframe TEXT, - close FLOAT, - open: FLOAT or NONE DEFAULTS: NONE, - high: FLOAT or NONE DEFAULTS: NONE, - low: FLOAT or NONE DEFAULTS: NONE, - adjusted_close FLOAT or NONE DEFAULTS: NONE, - volume: FLOAT or NONE DEFAULTS: NONE - ) + create_price_bars_table = """ + CREATE TABLE IF NOT EXISTS price_bars ( + id INTEGER PRIMARY KEY DEFAULT keyval('data_sources_id_seq'), + source_id INTEGER NOT NULL, + instrument_id INTEGER NOT NULL, + timestamp DATE NOT NULL, + timeframe TEXT NOT NULL, + close DOUBLE NOT NULL, + open DOUBLE, + high DOUBLE, + low DOUBLE, + adjusted_close DOUBLE, + volume DOUBLE, + FOREIGN KEY (source_id) REFERENCES data_sources (id), + FOREIGN KEY (instrument_id) REFERENCES instruments (id) + ); """ duckdb.connect(database_path) + duckdb.execute(query=create_data_sources_sequence) duckdb.execute(query=create_datasources_table) - duckdb.execute(query=create_intstrument_table) - duckdb.execute(query=create_price_bar_table) + duckdb.execute(query=create_intstruments_table) + duckdb.execute(query=create_price_bars_table) duckdb.close() From 9560b118b92f8df5967aa29940671c9b203f7e32 Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Mon, 29 Jun 2026 14:05:55 +0200 Subject: [PATCH 29/44] feat(#38): insert logic for source --- src/argus/storage/database.py | 51 ++++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 9 deletions(-) diff --git a/src/argus/storage/database.py b/src/argus/storage/database.py index 8e564b1..8eda7d5 100644 --- a/src/argus/storage/database.py +++ b/src/argus/storage/database.py @@ -5,9 +5,15 @@ def initialize_database(database_path: str) -> None: create_data_sources_sequence = """ CREATE SEQUENCE IF NOT EXISTS data_sources_id_seq; """ + create_instruments_sequence = """ + CREATE SEQUENCE IF NOT EXISTS instruemnts_id_seq; + """ + create_price_bars_sequence = """ + CREATE SEQUENCE IF NOT EXISTS price_bars_id_seq; + """ create_datasources_table = """ CREATE TABLE IF NOT EXISTS data_sources ( - id INTEGER PRIMARY KEY DEFAULT keyval('data_sources_id_seq'), + id INTEGER PRIMARY KEY DEFAULT nextval('data_sources_id_seq'), name TEXT NOT NULL UNIQUE, provider_kind TEXT NOT NULL, requires_api_key BOOLEAN NOT NULL @@ -15,7 +21,7 @@ def initialize_database(database_path: str) -> None: """ create_intstruments_table = """ CREATE TABLE IF NOT EXISTS instruments ( - id INTEGER PRIMARY KEY DEFAULT keyval('data_sources_id_seq'), + id INTEGER PRIMARY KEY DEFAULT nextval('instruemnts_id_seq'), symbol TEXT NOT NULL UNIQUE, name TEXT NOT NULL, asset_class TEXT NOT NULL, @@ -27,7 +33,7 @@ def initialize_database(database_path: str) -> None: """ create_price_bars_table = """ CREATE TABLE IF NOT EXISTS price_bars ( - id INTEGER PRIMARY KEY DEFAULT keyval('data_sources_id_seq'), + id INTEGER PRIMARY KEY DEFAULT nextval('price_bars_id_seq'), source_id INTEGER NOT NULL, instrument_id INTEGER NOT NULL, timestamp DATE NOT NULL, @@ -42,9 +48,36 @@ def initialize_database(database_path: str) -> None: FOREIGN KEY (instrument_id) REFERENCES instruments (id) ); """ - duckdb.connect(database_path) - duckdb.execute(query=create_data_sources_sequence) - duckdb.execute(query=create_datasources_table) - duckdb.execute(query=create_intstruments_table) - duckdb.execute(query=create_price_bars_table) - duckdb.close() + connection = duckdb.connect(database_path) + + connection.execute(query=create_data_sources_sequence) + connection.execute(query=create_instruments_sequence) + connection.execute(query=create_price_bars_sequence) + + connection.execute(query=create_datasources_table) + connection.execute(query=create_intstruments_table) + connection.execute(query=create_price_bars_table) + + connection.close() + +def insert_data_source(database_path, source): + connection = duckdb.connect(database_path) + + source = """ + INSERT INTO data_sources (name, provider_kind, requires_api_key) + VALUES (?,?,?); + """ + + connection.execute(query=source) + connection.close() + +def insert_instruemnt(database_path, source): + connection = duckdb.connect(database_path) + + source = """ + INSERT INTO instruments (symbol,name,asset_class) + VALUES (?,?,?); + """ + + connection.execute(query=source) + connection.close() From 4f44f750a62819db1494580c06533abddb508d57 Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Mon, 29 Jun 2026 14:14:11 +0200 Subject: [PATCH 30/44] feat(#38): insert logic - instrument --- src/argus/storage/database.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/src/argus/storage/database.py b/src/argus/storage/database.py index 8eda7d5..f56183a 100644 --- a/src/argus/storage/database.py +++ b/src/argus/storage/database.py @@ -1,4 +1,5 @@ import duckdb +from argus.domain.internal_models import DataSource, PriceBar, Instrument def initialize_database(database_path: str) -> None: @@ -57,27 +58,32 @@ def initialize_database(database_path: str) -> None: connection.execute(query=create_datasources_table) connection.execute(query=create_intstruments_table) connection.execute(query=create_price_bars_table) - + connection.close() -def insert_data_source(database_path, source): - connection = duckdb.connect(database_path) - source = """ +def insert_data_source(database_path, source: DataSource) -> None: + insert_query = """ INSERT INTO data_sources (name, provider_kind, requires_api_key) VALUES (?,?,?); """ - - connection.execute(query=source) + connection = duckdb.connect(database_path) + connection.execute( + query=insert_query, + parameters=[source.name, source.provider_kind, source.requires_api_key], + ) connection.close() -def insert_instruemnt(database_path, source): - connection = duckdb.connect(database_path) - source = """ +def insert_instruemnt(database_path, instrument: Instrument) -> None: + insert_query = """ INSERT INTO instruments (symbol,name,asset_class) VALUES (?,?,?); """ - - connection.execute(query=source) + + connection = duckdb.connect(database_path) + connection.execute( + query=insert_query, + parameters=[instrument.symbol, instrument.name, instrument.asset_class], + ) connection.close() From 476379957122a3371e0266315fdb840de3bbcffe Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Mon, 29 Jun 2026 14:20:13 +0200 Subject: [PATCH 31/44] feat(#38): insert logic - price bar --- src/argus/storage/database.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/argus/storage/database.py b/src/argus/storage/database.py index f56183a..b1fdd21 100644 --- a/src/argus/storage/database.py +++ b/src/argus/storage/database.py @@ -87,3 +87,17 @@ def insert_instruemnt(database_path, instrument: Instrument) -> None: parameters=[instrument.symbol, instrument.name, instrument.asset_class], ) connection.close() + + +def insert_pirce_bar(database_path, price_bar: PriceBar) -> None: + insert_query = """ + INSERT INTO instruments (timestamp,timeframe,close) + VALUES (?,?,?); + """ + + connection = duckdb.connect(database_path) + connection.execute( + query=insert_query, + parameters=[price_bar.timestamp, price_bar.timeframe, price_bar.close], + ) + connection.close() From 2b32db43337a8808727b49a6d20b8501baa8fba5 Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Mon, 29 Jun 2026 14:39:01 +0200 Subject: [PATCH 32/44] feat(#38): add id search for bar --- src/argus/storage/database.py | 42 ++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/src/argus/storage/database.py b/src/argus/storage/database.py index b1fdd21..d3af5f7 100644 --- a/src/argus/storage/database.py +++ b/src/argus/storage/database.py @@ -7,7 +7,7 @@ def initialize_database(database_path: str) -> None: CREATE SEQUENCE IF NOT EXISTS data_sources_id_seq; """ create_instruments_sequence = """ - CREATE SEQUENCE IF NOT EXISTS instruemnts_id_seq; + CREATE SEQUENCE IF NOT EXISTS instruements_id_seq; """ create_price_bars_sequence = """ CREATE SEQUENCE IF NOT EXISTS price_bars_id_seq; @@ -22,7 +22,7 @@ def initialize_database(database_path: str) -> None: """ create_intstruments_table = """ CREATE TABLE IF NOT EXISTS instruments ( - id INTEGER PRIMARY KEY DEFAULT nextval('instruemnts_id_seq'), + id INTEGER PRIMARY KEY DEFAULT nextval('instruements_id_seq'), symbol TEXT NOT NULL UNIQUE, name TEXT NOT NULL, asset_class TEXT NOT NULL, @@ -75,7 +75,7 @@ def insert_data_source(database_path, source: DataSource) -> None: connection.close() -def insert_instruemnt(database_path, instrument: Instrument) -> None: +def insert_instruement(database_path, instrument: Instrument) -> None: insert_query = """ INSERT INTO instruments (symbol,name,asset_class) VALUES (?,?,?); @@ -88,16 +88,42 @@ def insert_instruemnt(database_path, instrument: Instrument) -> None: ) connection.close() +def find_data_source_id(database_path,price_bar: PriceBar): + search_query = """ + SELECT id FROM data_sources + WHERE name=? + """ + connection = duckdb.connect(database_path) + result = connection.execute( + query=search_query, + parameters=[price_bar.source.name], + ) + connection.close() + return result -def insert_pirce_bar(database_path, price_bar: PriceBar) -> None: - insert_query = """ - INSERT INTO instruments (timestamp,timeframe,close) - VALUES (?,?,?); +def find_instrument_id(database_path,price_bar: PriceBar): + search_query = """ + SELECT id FROM instruments + WHERE name=? """ + connection = duckdb.connect(database_path) + connection.execute( + query=search_query, + parameters=[price_bar.instrument.name], + ) + connection.close() +def insert_price_bar(database_path, price_bar: PriceBar) -> None: + insert_query = """ + INSERT INTO price_bars (source_id,instrument_id,timestamp,timeframe,close) + VALUES (?,?,?,?,?); + """ + data_source_id = find_data_source_id(database_path, price_bar) + instrument_id = find_instrument_id(database_path, price_bar) + connection = duckdb.connect(database_path) connection.execute( query=insert_query, - parameters=[price_bar.timestamp, price_bar.timeframe, price_bar.close], + parameters=[data_source_id,instrument_id,price_bar.timestamp, price_bar.timeframe, price_bar.close], ) connection.close() From 76656b262694ff09bdd61a84df6824d4c44b02e5 Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Mon, 29 Jun 2026 15:07:12 +0200 Subject: [PATCH 33/44] feat(#38): improve param select --- src/argus/storage/database.py | 88 +++++++++++++++++++++++++++-------- 1 file changed, 68 insertions(+), 20 deletions(-) diff --git a/src/argus/storage/database.py b/src/argus/storage/database.py index d3af5f7..61d7b13 100644 --- a/src/argus/storage/database.py +++ b/src/argus/storage/database.py @@ -1,4 +1,5 @@ import duckdb +import pandas as pd from argus.domain.internal_models import DataSource, PriceBar, Instrument @@ -62,7 +63,7 @@ def initialize_database(database_path: str) -> None: connection.close() -def insert_data_source(database_path, source: DataSource) -> None: +def insert_data_source(database_path: str, source: DataSource) -> None: insert_query = """ INSERT INTO data_sources (name, provider_kind, requires_api_key) VALUES (?,?,?); @@ -75,10 +76,17 @@ def insert_data_source(database_path, source: DataSource) -> None: connection.close() -def insert_instruement(database_path, instrument: Instrument) -> None: +def insert_instrument(database_path: str, instrument: Instrument) -> None: insert_query = """ - INSERT INTO instruments (symbol,name,asset_class) - VALUES (?,?,?); + INSERT INTO instruments ( + symbol, + name, + asset_class, + currency, + exchange, + base_currency, + quote_currency) + VALUES (?,?,?,?,?,?,?); """ connection = duckdb.connect(database_path) @@ -88,7 +96,8 @@ def insert_instruement(database_path, instrument: Instrument) -> None: ) connection.close() -def find_data_source_id(database_path,price_bar: PriceBar): + +def get_data_source_id(database_path: str, source: DataSource) -> int | None: search_query = """ SELECT id FROM data_sources WHERE name=? @@ -96,34 +105,73 @@ def find_data_source_id(database_path,price_bar: PriceBar): connection = duckdb.connect(database_path) result = connection.execute( query=search_query, - parameters=[price_bar.source.name], - ) + parameters=[source.name], + ).fetchone() connection.close() - return result + if result is None: + return None + return result[0] + -def find_instrument_id(database_path,price_bar: PriceBar): +def get_instrument_id(database_path: str, instrument: Instrument) -> int | None: search_query = """ SELECT id FROM instruments WHERE name=? """ connection = duckdb.connect(database_path) - connection.execute( + result = connection.execute( query=search_query, - parameters=[price_bar.instrument.name], - ) + parameters=[instrument.name], + ).fetchone() connection.close() + if result is None: + return None + return result[0] -def insert_price_bar(database_path, price_bar: PriceBar) -> None: + +def insert_price_bar(database_path: str, price_bar: PriceBar) -> None: insert_query = """ - INSERT INTO price_bars (source_id,instrument_id,timestamp,timeframe,close) - VALUES (?,?,?,?,?); - """ - data_source_id = find_data_source_id(database_path, price_bar) - instrument_id = find_instrument_id(database_path, price_bar) - + INSERT INTO price_bars ( + source_id, + instrument_id, + timestamp, + timeframe, + close, + open, + high, + low, + adjusted_close, + volume + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + """ + source_id = get_data_source_id(database_path, price_bar.source) + instrument_id = get_instrument_id(database_path, price_bar.instrument) + connection = duckdb.connect(database_path) + + if source_id is None: + connection.close() + raise ValueError("Data source does not exist in storage.") + + if instrument_id is None: + connection.close() + raise ValueError("Instrument does not exist in storage.") + connection.execute( query=insert_query, - parameters=[data_source_id,instrument_id,price_bar.timestamp, price_bar.timeframe, price_bar.close], + parameters=[ + source_id, + instrument_id, + price_bar.timestamp, + price_bar.timeframe, + price_bar.close, + price_bar.open, + price_bar.high, + price_bar.low, + price_bar.adjusted_close, + price_bar.volume, + ], ) + connection.close() From be49bd975f62dc207a572a7479b13e0075dc2fa0 Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Mon, 29 Jun 2026 15:28:07 +0200 Subject: [PATCH 34/44] refactor(#38): upsert instead insert --- src/argus/storage/database.py | 80 +++++++++++++++++------------------ 1 file changed, 38 insertions(+), 42 deletions(-) diff --git a/src/argus/storage/database.py b/src/argus/storage/database.py index 61d7b13..fe6eae0 100644 --- a/src/argus/storage/database.py +++ b/src/argus/storage/database.py @@ -63,20 +63,31 @@ def initialize_database(database_path: str) -> None: connection.close() -def insert_data_source(database_path: str, source: DataSource) -> None: +def upsert_source(db: str, source: DataSource) -> int | None: insert_query = """ INSERT INTO data_sources (name, provider_kind, requires_api_key) VALUES (?,?,?); """ - connection = duckdb.connect(database_path) + search_query = """ + SELECT id FROM data_sources + WHERE name=? + """ + result = duckdb.execute( + query=search_query, + parameters=[source.name], + ).fetchone() + if result is not None: + return result[0] + connection = duckdb.connect(db) connection.execute( query=insert_query, parameters=[source.name, source.provider_kind, source.requires_api_key], ) connection.close() + return None -def insert_instrument(database_path: str, instrument: Instrument) -> None: +def upsert_instrument(db: str, instrument: Instrument) -> None: insert_query = """ INSERT INTO instruments ( symbol, @@ -88,48 +99,35 @@ def insert_instrument(database_path: str, instrument: Instrument) -> None: quote_currency) VALUES (?,?,?,?,?,?,?); """ - - connection = duckdb.connect(database_path) - connection.execute( - query=insert_query, - parameters=[instrument.symbol, instrument.name, instrument.asset_class], - ) - connection.close() - - -def get_data_source_id(database_path: str, source: DataSource) -> int | None: - search_query = """ - SELECT id FROM data_sources - WHERE name=? - """ - connection = duckdb.connect(database_path) - result = connection.execute( - query=search_query, - parameters=[source.name], - ).fetchone() - connection.close() - if result is None: - return None - return result[0] - - -def get_instrument_id(database_path: str, instrument: Instrument) -> int | None: search_query = """ SELECT id FROM instruments - WHERE name=? + WHERE symbol=? """ - connection = duckdb.connect(database_path) - result = connection.execute( + + result = duckdb.execute( query=search_query, - parameters=[instrument.name], + parameters=[instrument.symbol], ).fetchone() + if result is not None: + return result[0] + connection = duckdb.connect(db) + connection.execute( + query=insert_query, + parameters=[ + instrument.symbol, + instrument.name, + instrument.asset_class, + instrument.currency, + instrument.exchange, + instrument.base_currency, + instrument.quote_currency, + ], + ) connection.close() - if result is None: - return None - return result[0] + return None -def insert_price_bar(database_path: str, price_bar: PriceBar) -> None: +def insert_price_bar(db: str, price_bar: PriceBar) -> None: insert_query = """ INSERT INTO price_bars ( source_id, @@ -145,11 +143,9 @@ def insert_price_bar(database_path: str, price_bar: PriceBar) -> None: ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?); """ - source_id = get_data_source_id(database_path, price_bar.source) - instrument_id = get_instrument_id(database_path, price_bar.instrument) - - connection = duckdb.connect(database_path) - + connection = duckdb.connect(db) + source_id = upsert_source(db, price_bar.source) + instrument_id = upsert_instrument(db, price_bar.instrument) if source_id is None: connection.close() raise ValueError("Data source does not exist in storage.") From 65ee9863725721438310c4e289ab76704a5de184 Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Mon, 29 Jun 2026 15:51:27 +0200 Subject: [PATCH 35/44] refactor(#38): get_create instead --- src/argus/storage/database.py | 56 ++++++++++++++++++++++------------- 1 file changed, 36 insertions(+), 20 deletions(-) diff --git a/src/argus/storage/database.py b/src/argus/storage/database.py index fe6eae0..e01463c 100644 --- a/src/argus/storage/database.py +++ b/src/argus/storage/database.py @@ -63,7 +63,7 @@ def initialize_database(database_path: str) -> None: connection.close() -def upsert_source(db: str, source: DataSource) -> int | None: +def get_or_create_source(connection, source: DataSource) -> int: insert_query = """ INSERT INTO data_sources (name, provider_kind, requires_api_key) VALUES (?,?,?); @@ -72,22 +72,31 @@ def upsert_source(db: str, source: DataSource) -> int | None: SELECT id FROM data_sources WHERE name=? """ - result = duckdb.execute( + + result = connection.execute( query=search_query, parameters=[source.name], ).fetchone() if result is not None: return result[0] - connection = duckdb.connect(db) + connection.execute( query=insert_query, parameters=[source.name, source.provider_kind, source.requires_api_key], ) - connection.close() - return None + result = connection.execute( + query=search_query, + parameters=[source.name], + ).fetchone() + + if result is None: + raise ValueError("Data source could not be inserted.") + + return result[0] -def upsert_instrument(db: str, instrument: Instrument) -> None: + +def get_or_create_instrument(connection, instrument: Instrument) -> int: insert_query = """ INSERT INTO instruments ( symbol, @@ -104,13 +113,13 @@ def upsert_instrument(db: str, instrument: Instrument) -> None: WHERE symbol=? """ - result = duckdb.execute( + result = connection.execute( query=search_query, parameters=[instrument.symbol], ).fetchone() if result is not None: return result[0] - connection = duckdb.connect(db) + connection.execute( query=insert_query, parameters=[ @@ -123,8 +132,23 @@ def upsert_instrument(db: str, instrument: Instrument) -> None: instrument.quote_currency, ], ) - connection.close() - return None + result = connection.execute( + query=search_query, + parameters=[ + instrument.symbol, + instrument.name, + instrument.asset_class, + instrument.currency, + instrument.exchange, + instrument.base_currency, + instrument.quote_currency, + ], + ).fetchone() + + if result is None: + raise ValueError("Instrument could not be inserted.") + + return result[0] def insert_price_bar(db: str, price_bar: PriceBar) -> None: @@ -144,16 +168,8 @@ def insert_price_bar(db: str, price_bar: PriceBar) -> None: VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?); """ connection = duckdb.connect(db) - source_id = upsert_source(db, price_bar.source) - instrument_id = upsert_instrument(db, price_bar.instrument) - if source_id is None: - connection.close() - raise ValueError("Data source does not exist in storage.") - - if instrument_id is None: - connection.close() - raise ValueError("Instrument does not exist in storage.") - + source_id = get_or_create_source(connection, price_bar.source) + instrument_id = get_or_create_instrument(connection, price_bar.instrument) connection.execute( query=insert_query, parameters=[ From 7cda9561ad9a1a21994e912ded411be6691d2bb3 Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Mon, 29 Jun 2026 16:07:28 +0200 Subject: [PATCH 36/44] style(#38): fix func names --- argus_probe.duckdb | Bin 0 -> 3944448 bytes src/argus/storage/database.py | 63 +++++++++++++++------------------- 2 files changed, 28 insertions(+), 35 deletions(-) create mode 100644 argus_probe.duckdb diff --git a/argus_probe.duckdb b/argus_probe.duckdb new file mode 100644 index 0000000000000000000000000000000000000000..9c9ef34090bb8247f7d0259b65bba5a7b16ef54e GIT binary patch literal 3944448 zcmeF)U5p&bK>*+y@7lZmTl;e7pOBC{2%zwi^M%U~LJ>qcCwI!`lI&?otO!lUJH6}4 zt!M3-UB?y=UgarL0_b=;fe=Ipi8mw^5eXED(@DX0e0hnYAR)n1aKuaU1Uw)Q)!oxG zv$He)+rQb}uVv5FRR2`fS6$UzJ3Zas{Kg;r;vYZt^qViue)2QF7anc8_QD50{J|H8 zFMIA|$#13#cmV}sLdtiiA&G+KGs|2 z+o2l9qg<_;#L)3pYd&53a1y>%S_ogf6(;J*_2p}o#h74>pGd-I7DD+}T(lbA-D)Kv z%nvk7vvN19k~Fjs!lkrIVp8?WkK?5nSQVfOsmoP20_ zPGUi}BqwP{lI|!EY*a_{R-ZNcv`tM8Yf~2nx2Z~Bn;OuGq%CPsn>y6IarxrX;<{Ef zxUC(nG@HqCd2TU%j~QLl(kACt8jYkrm$k69bCT1OVNB#TaL|iOIlDh zJkxH=y%~8xQ$Dms8F{3p+|$3OE&1rMmi(o`E%_^JTk`s2_po-{J=&!0G9HS14v<~9 zxG%4peR#QcIcY9eE?-G!o#F7UbksdKx46{I+Vh#-HO7~&B=vMVYhy3f7A~!e z9a~)b+PcW8O7*KN&E=#TJDEWV9J;o&xRRd*^dyn?1czfE`+$RZJesd>_Txp55O?08 z2k&?`mz@~i4Qoz*h8({8TYq=$-d?`FawkXG$zqDpKDMsLQ?SXjIfkqr$6_NWnq&6_ zqI@ntoTkUS%{gj5^Qq6xoPXuhuf}uT&N<^pwj+^`Zf+uJE7@fu6WJv|V4DQuSmkK! z@2eqXgV5N?rpMjb=cXOUM5~n^y5py`GrpcirRPd2_#8R3ZJfef(v7yj4Q`=Wl*u(I@F60RjXF5FkK+0D*lfFnCar{Y*O-`Nhr7 z@3z|!?eT`6ZtuHY=Fq+?u>unyK!5-N0t5&=rhxIqV}eA;q)LDQ0RjXF5FkK+z>^7#^!Vb*iexPU1hzyV?=4!bE!nt@5+Fc; z009DfL?Az7i1&c%xm8s9Ym69kE3BSOXYY}cDh2@p1PBn=YXbe_{sZj-mfG*>=i(uM z=}iEN`1b4&{`c8`cBt=Y2oNAZfB*pk1PBlyK!5;&krF7z{naor7Y@Wf;a9_f`7nOD zQmfbUdn(O^Ls|5pBpj_Y7FL?&D~+X#Nq3=J;pu8+xl(Q}tu*G6X1P`^H&w?FixDNpPbA?p3!!`~E?N!mPQNS( z^8*=EuDe;Sq@jfnE~T{+ld4xPCuvG??dX}lzVrs^0_3m*I13uaEuM|Z2WpH?lS4Fl-6IVbSad6cBB>~ z)4k(d`tpJF<;_T14c|DJy~Z!&;-Vp(Xf7``Dho;ZTGD9Nmg=RLqiT4zy&J|g(y1Cw z&A$4|7iQ0|&B=#`=OjLWa!YcOb{Odn_Q1w?H19C7{vvIQlf&BLg~4sH($^LT^gpS} zptg9ZdE@fMrNwovac~dMmc(D{$vn7 z7}f`Lk8o+5j)(uAgL2m`wu5!E4=>j)C(Y%`( zjq#-`Nj=@p+Sp6Ag-h#V#}=2qwk~q2QvK>mb2+KT4r)*WhpsIxuHfQF=<;+eZ+b0ZbPPm4gGW7Qx@f5VZcJC3Qy>cf<@u<1B zJswNKH)D$`4$Nb*6%_5Ud)D%x=6G!Dn&YQFH*@}#Prn+^!#n4*9~pjB&i2JEXpT{2 zKt?_~JR?(EB$8I$NR6pvhQKZnIG&wT$00>E+)SJ2flDzIvIjbbg^(T$OYuYJxsUCk zLEcrSbB3&#HpzV258MCGzw8QS;k@YSvGp;Hj%HnCUmZ-o5UV7eZih}V8%CGz=6_Dv z3?#`7LZRowZhn20fnLJDB>mcpUV(nd7 zYU>(phiK;{)4w?t-A?>N$gplFX1%E6Jn+WZmoH3D=XL6p0D<(cdnN(|2oNAZfWRPu zsX>u@DJ~z(7+372I#rgP6ELpWIeRG+AV7cs0RlTCU_7xi?o!=UffJj$wN2QVam6Me z+5`at1PBlyu&V?fFs>MQS)ps7c2)JtECB)pc3R-%PG>;+yHCL1W_Dkl$uWUKVDQ+Y z^BbaJKPNK8C;+_^AV7cs0RjYepTJYQFUup9B;OSkKU)d! z0wX41lrdrzB0B^K5FkK+009C7HbWpk(QmajV_T2C4Wo?5CbI$o0t5&UAV7cs0RjUB z^3g@{3+`f(?2b{l!s^L%c5!FwY6x+;`EaxvR@>9DBuutiYd&A<`+Td^7xA|T`XatH z))(N36dXl7)aCTs7^ViFb%5u_-sTIMEr*?BFTx^nUxERW($0*HssoJTG z((uYCI=F}&SBTngoPGJi^z<&=8D*Kk)(8~G`K{Tytr8$WfB*pk1V&FFAA9tVCpsgp zVS_J2i_xpf&B@W3&Do5t5FkK+009E~O2D7g_EmL1w6H^l7{%8t!@hAb#CWKz1x8%J z<61_%f@H4{I9mj~6Cgl<0D;{iux^Oay+m!`l~(!XP(|q;c;V-G_d?H8eHVVl&ucFH zjGuqB=EBby@m9R>GrIz_d%4@qp;v`%&gFFRy3qlb)19B4d4Be_mqHwF6~W>Hw|LcP zIz7e(XvLL#X4BN#!6LQn^4cd&?PCMWHh(>@%4{f^2aD7q|C5*P zKFjT-_ zaEH?Dxf$=?b0tzB0t5&UAh0okd@$2LuIP-P^FdNJsxF2t*-+XLV`J>rCa_TfLzs;U zTAu)cJuC1_d$wc>Mqt+n3>#uR+7I7i_=0oS?8P!lfB*pk1PJVVf%olufl&e& zGQ=qEdOPd}xcgyAV7e??hx=N%H2^G~-DldI zNy5FU^kE&V?I}k5%Za{-zmLysS+m4nJ=GWS$M5Qk_`m*$|9ZM_iGO`2eI`hJVoEg} zEpD+J=0o?%J!!$>{d8}K>_HxB_Tp1S(kYfT{)jz`edFxQ7pA9+LhtMM+@{qUZz~kf zo-A%lbFdwMsBeDJf2PY9>rTXQ5#L@sv?ju8nnv;2I{^X&2oNAZfB*pk1PBlyuuQ>6Yf2_M<~`-3%q};6R>>(1PBlyK!Csw3K(DPAikst5FkKc zBm{nKByu4W1PBn=8v@1`d!rtef&c*m1PBlyu$KgWeJ>SepOwYXrIjk`}Qye*Sv7QCUu!X}pO50RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBn=y8_eq%2RL8oW6O#Wg ze!KMki5T~E{GYy`pPhMr_O+L?rB24z_s0L}`}vi|(sR#0H~ZRiFD)%DRvPI72jc&! z_WS(xG$w>&`CI3;_w!hcJrVz>?=hY1z4%?G7%s^fWXEDO5yzM%=5Fay;Su7lkLDn5#gNx0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfWU4Ln7&t@dVA*d&HF7A0RjXF zY+RreRzoQD^#9}S@I+6XAOQjd2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5;&kr(*h)$e@w>bbF>-*1@+5FkKc;{v5{W4>0e)aR1+0pM`^ePZMf2r@{3 z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK%hrp`d)eJ?U~az z@3%|@2oNB!ae-1;4WZQ2&5pOj6FqT)1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjX@Uf`+4GtDpG`S6eLw@d^G5FoH|fl@d>JM;YPYcCc3|71Hb zQABtrK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FoG{1P-1G z;h%ndc{R@pzQ0i24LO%b0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZV5~ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV6T`1*Y$nr{119 zee-_HM1TMR0vi`7h1C#BJ)Qq}J3P@7CrE$*0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z!2h>Xhc*DhP!K?W6OuY4Q=wRZQQF-)Ot*j None: CREATE SEQUENCE IF NOT EXISTS data_sources_id_seq; """ create_instruments_sequence = """ - CREATE SEQUENCE IF NOT EXISTS instruements_id_seq; + CREATE SEQUENCE IF NOT EXISTS instruments_id_seq; """ create_price_bars_sequence = """ CREATE SEQUENCE IF NOT EXISTS price_bars_id_seq; """ - create_datasources_table = """ + create_data_sources_table = """ CREATE TABLE IF NOT EXISTS data_sources ( id INTEGER PRIMARY KEY DEFAULT nextval('data_sources_id_seq'), name TEXT NOT NULL UNIQUE, @@ -21,9 +20,9 @@ def initialize_database(database_path: str) -> None: requires_api_key BOOLEAN NOT NULL ); """ - create_intstruments_table = """ + create_instruments_table = """ CREATE TABLE IF NOT EXISTS instruments ( - id INTEGER PRIMARY KEY DEFAULT nextval('instruements_id_seq'), + id INTEGER PRIMARY KEY DEFAULT nextval('instruments_id_seq'), symbol TEXT NOT NULL UNIQUE, name TEXT NOT NULL, asset_class TEXT NOT NULL, @@ -56,8 +55,8 @@ def initialize_database(database_path: str) -> None: connection.execute(query=create_instruments_sequence) connection.execute(query=create_price_bars_sequence) - connection.execute(query=create_datasources_table) - connection.execute(query=create_intstruments_table) + connection.execute(query=create_data_sources_table) + connection.execute(query=create_instruments_table) connection.execute(query=create_price_bars_table) connection.close() @@ -134,15 +133,7 @@ def get_or_create_instrument(connection, instrument: Instrument) -> int: ) result = connection.execute( query=search_query, - parameters=[ - instrument.symbol, - instrument.name, - instrument.asset_class, - instrument.currency, - instrument.exchange, - instrument.base_currency, - instrument.quote_currency, - ], + parameters=[instrument.symbol], ).fetchone() if result is None: @@ -168,22 +159,24 @@ def insert_price_bar(db: str, price_bar: PriceBar) -> None: VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?); """ connection = duckdb.connect(db) - source_id = get_or_create_source(connection, price_bar.source) - instrument_id = get_or_create_instrument(connection, price_bar.instrument) - connection.execute( - query=insert_query, - parameters=[ - source_id, - instrument_id, - price_bar.timestamp, - price_bar.timeframe, - price_bar.close, - price_bar.open, - price_bar.high, - price_bar.low, - price_bar.adjusted_close, - price_bar.volume, - ], - ) - - connection.close() + try: + source_id = get_or_create_source(connection, price_bar.source) + instrument_id = get_or_create_instrument(connection, price_bar.instrument) + connection.execute( + query=insert_query, + parameters=[ + source_id, + instrument_id, + price_bar.timestamp, + price_bar.timeframe, + price_bar.close, + price_bar.open, + price_bar.high, + price_bar.low, + price_bar.adjusted_close, + price_bar.volume, + ], + ) + connection.close() + finally: + connection.close() From 5559513b3563af9a2f3f85cb9c1f01072baa23c9 Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Tue, 30 Jun 2026 08:59:55 +0200 Subject: [PATCH 37/44] feat(#38): add read func --- src/argus/storage/database.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/argus/storage/database.py b/src/argus/storage/database.py index be825ad..504bc9f 100644 --- a/src/argus/storage/database.py +++ b/src/argus/storage/database.py @@ -180,3 +180,26 @@ def insert_price_bar(db: str, price_bar: PriceBar) -> None: connection.close() finally: connection.close() + +def get_price_bar(db: str,source:DataSource,instrument:Instrument,start_time:str,end_time:str): + search_query = """ + SELECT + data_sources.name AS source_name, + instruments.symbol AS instrument_symbol, + price_bars.timestamp, + price_bars.timeframe, + price_bars.open, + price_bars.high, + price_bars.low, + price_bars.close, + price_bars.adjusted_close, + price_bars.volume + FROM price_bars + WHERE data_sources.name = ? AND instruments.symbol = ? AND price_bars.timestamp BETWEEN ? AND ? + JOIN data_sources ON price_bars.source_id = data_sources.id + JOIN instruments ON price_bars.instrument_id = instruments.id + """ + connection = duckdb.connect(db) + result = connection.execute(search_query,parameters=[source.name,instrument.symbol,start_time,end_time]) + connection.close() + return result From d51da47f26f1661cc583d3462767cfe8045be8cc Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Tue, 30 Jun 2026 14:37:09 +0200 Subject: [PATCH 38/44] feat(#38): try for read func --- src/argus/storage/database.py | 62 +++++++++++++++++++++++++---------- 1 file changed, 44 insertions(+), 18 deletions(-) diff --git a/src/argus/storage/database.py b/src/argus/storage/database.py index 504bc9f..c55c50b 100644 --- a/src/argus/storage/database.py +++ b/src/argus/storage/database.py @@ -1,4 +1,6 @@ import duckdb +from datetime import date +import pandas as pd from argus.domain.internal_models import DataSource, PriceBar, Instrument @@ -181,25 +183,49 @@ def insert_price_bar(db: str, price_bar: PriceBar) -> None: finally: connection.close() -def get_price_bar(db: str,source:DataSource,instrument:Instrument,start_time:str,end_time:str): + +def read_price_bars( + db: str, + source: DataSource, + instrument: Instrument, + start_date: date, + end_date: date, +) -> pd.DataFrame: search_query = """ - SELECT - data_sources.name AS source_name, - instruments.symbol AS instrument_symbol, - price_bars.timestamp, - price_bars.timeframe, - price_bars.open, - price_bars.high, - price_bars.low, - price_bars.close, - price_bars.adjusted_close, - price_bars.volume - FROM price_bars - WHERE data_sources.name = ? AND instruments.symbol = ? AND price_bars.timestamp BETWEEN ? AND ? - JOIN data_sources ON price_bars.source_id = data_sources.id - JOIN instruments ON price_bars.instrument_id = instruments.id - """ + SELECT + data_sources.name AS source_name, + instruments.symbol AS instrument_symbol, + price_bars.timestamp, + price_bars.timeframe, + price_bars.open, + price_bars.high, + price_bars.low, + price_bars.close, + price_bars.adjusted_close, + price_bars.volume + FROM price_bars + JOIN data_sources ON price_bars.source_id = data_sources.id + JOIN instruments ON price_bars.instrument_id = instruments.id + WHERE data_sources.name = ? + AND instruments.symbol = ? + AND price_bars.timestamp BETWEEN ? AND ? + ORDER BY price_bars.timestamp; + """ + connection = duckdb.connect(db) - result = connection.execute(search_query,parameters=[source.name,instrument.symbol,start_time,end_time]) + try: + result = connection.execute( + query=search_query, + parameters=[ + source.name, + instrument.symbol, + start_date, + end_date, + ], + ).df() + finally: + connection.close() + connection.close() + return result From 4146e507347f5365ac8ef733efd53372a2d74db6 Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Wed, 1 Jul 2026 09:07:34 +0200 Subject: [PATCH 39/44] test(#38): test for init --- src/argus/storage/database.py | 106 +++++++++++++++------------------ tests/test_storage_database.py | 25 ++++++++ 2 files changed, 73 insertions(+), 58 deletions(-) create mode 100644 tests/test_storage_database.py diff --git a/src/argus/storage/database.py b/src/argus/storage/database.py index c55c50b..fbaae40 100644 --- a/src/argus/storage/database.py +++ b/src/argus/storage/database.py @@ -5,63 +5,56 @@ def initialize_database(database_path: str) -> None: - create_data_sources_sequence = """ - CREATE SEQUENCE IF NOT EXISTS data_sources_id_seq; - """ - create_instruments_sequence = """ - CREATE SEQUENCE IF NOT EXISTS instruments_id_seq; - """ - create_price_bars_sequence = """ - CREATE SEQUENCE IF NOT EXISTS price_bars_id_seq; - """ - create_data_sources_table = """ - CREATE TABLE IF NOT EXISTS data_sources ( - id INTEGER PRIMARY KEY DEFAULT nextval('data_sources_id_seq'), - name TEXT NOT NULL UNIQUE, - provider_kind TEXT NOT NULL, - requires_api_key BOOLEAN NOT NULL - ); - """ - create_instruments_table = """ + queries = [ + "CREATE SEQUENCE IF NOT EXISTS data_sources_id_seq;", + "CREATE SEQUENCE IF NOT EXISTS instruments_id_seq;", + "CREATE SEQUENCE IF NOT EXISTS price_bars_id_seq;", + """ + CREATE TABLE IF NOT EXISTS data_sources ( + id INTEGER PRIMARY KEY DEFAULT nextval('data_sources_id_seq'), + name TEXT NOT NULL UNIQUE, + provider_kind TEXT NOT NULL, + requires_api_key BOOLEAN NOT NULL + ); + """, + """ CREATE TABLE IF NOT EXISTS instruments ( - id INTEGER PRIMARY KEY DEFAULT nextval('instruments_id_seq'), - symbol TEXT NOT NULL UNIQUE, - name TEXT NOT NULL, - asset_class TEXT NOT NULL, - currency TEXT, - exchange TEXT, - base_currency TEXT, - quote_currency TEXT - ); - """ - create_price_bars_table = """ + id INTEGER PRIMARY KEY DEFAULT nextval('instruments_id_seq'), + symbol TEXT NOT NULL UNIQUE, + name TEXT NOT NULL, + asset_class TEXT NOT NULL, + currency TEXT, + exchange TEXT, + base_currency TEXT, + quote_currency TEXT + ); + """, + """ CREATE TABLE IF NOT EXISTS price_bars ( - id INTEGER PRIMARY KEY DEFAULT nextval('price_bars_id_seq'), - source_id INTEGER NOT NULL, - instrument_id INTEGER NOT NULL, - timestamp DATE NOT NULL, - timeframe TEXT NOT NULL, - close DOUBLE NOT NULL, - open DOUBLE, - high DOUBLE, - low DOUBLE, - adjusted_close DOUBLE, - volume DOUBLE, - FOREIGN KEY (source_id) REFERENCES data_sources (id), - FOREIGN KEY (instrument_id) REFERENCES instruments (id) - ); - """ - connection = duckdb.connect(database_path) + id INTEGER PRIMARY KEY DEFAULT nextval('price_bars_id_seq'), + source_id INTEGER NOT NULL, + instrument_id INTEGER NOT NULL, + timestamp DATE NOT NULL, + timeframe TEXT NOT NULL, + close DOUBLE NOT NULL, + open DOUBLE, + high DOUBLE, + low DOUBLE, + adjusted_close DOUBLE, + volume DOUBLE, + FOREIGN KEY (source_id) REFERENCES data_sources (id), + FOREIGN KEY (instrument_id) REFERENCES instruments (id), + UNIQUE (source_id, instrument_id, timestamp, timeframe) + ); + """, + ] - connection.execute(query=create_data_sources_sequence) - connection.execute(query=create_instruments_sequence) - connection.execute(query=create_price_bars_sequence) - - connection.execute(query=create_data_sources_table) - connection.execute(query=create_instruments_table) - connection.execute(query=create_price_bars_table) - - connection.close() + connection = duckdb.connect(database_path) + try: + for query in queries: + connection.execute(query) + finally: + connection.close() def get_or_create_source(connection, source: DataSource) -> int: @@ -179,7 +172,6 @@ def insert_price_bar(db: str, price_bar: PriceBar) -> None: price_bar.volume, ], ) - connection.close() finally: connection.close() @@ -191,6 +183,7 @@ def read_price_bars( start_date: date, end_date: date, ) -> pd.DataFrame: + search_query = """ SELECT data_sources.name AS source_name, @@ -225,7 +218,4 @@ def read_price_bars( ).df() finally: connection.close() - - connection.close() - return result diff --git a/tests/test_storage_database.py b/tests/test_storage_database.py new file mode 100644 index 0000000..1b7dad5 --- /dev/null +++ b/tests/test_storage_database.py @@ -0,0 +1,25 @@ +from datetime import date + +import duckdb + +from argus.domain.internal_models import DataSource, Instrument, PriceBar +from argus.storage.database import ( + initialize_database, + insert_price_bar, + read_price_bars, +) + + +def test_initialize_database_creates_required_tables(tmp_path): + + db = tmp_path / "test.duckdb" + + initialize_database(db) + connection = duckdb.connect(db) + + tables = connection.execute("SHOW TABLES;").fetchall() + table_names = {row[0] for row in tables} + + assert "data_sources" in table_names + assert "instruments" in table_names + assert "price_bars" in table_names From cfb5af9a4ea37477856897a59c3ce8225afbc1d3 Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Wed, 1 Jul 2026 09:42:26 +0200 Subject: [PATCH 40/44] test(#38): add more storage tests --- src/argus/storage/database.py | 9 +- tests/test_exchangerate_client.py | 1 - tests/test_storage_database.py | 137 ++++++++++++++++++++++++++++-- 3 files changed, 135 insertions(+), 12 deletions(-) diff --git a/src/argus/storage/database.py b/src/argus/storage/database.py index fbaae40..f184b65 100644 --- a/src/argus/storage/database.py +++ b/src/argus/storage/database.py @@ -60,7 +60,8 @@ def initialize_database(database_path: str) -> None: def get_or_create_source(connection, source: DataSource) -> int: insert_query = """ INSERT INTO data_sources (name, provider_kind, requires_api_key) - VALUES (?,?,?); + VALUES (?,?,?) + ON CONFLICT DO NOTHING; """ search_query = """ SELECT id FROM data_sources @@ -100,7 +101,8 @@ def get_or_create_instrument(connection, instrument: Instrument) -> int: exchange, base_currency, quote_currency) - VALUES (?,?,?,?,?,?,?); + VALUES (?,?,?,?,?,?,?) + ON CONFLICT DO NOTHING; """ search_query = """ SELECT id FROM instruments @@ -151,7 +153,8 @@ def insert_price_bar(db: str, price_bar: PriceBar) -> None: adjusted_close, volume ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT DO NOTHING; """ connection = duckdb.connect(db) try: diff --git a/tests/test_exchangerate_client.py b/tests/test_exchangerate_client.py index 1f15512..132a8a8 100644 --- a/tests/test_exchangerate_client.py +++ b/tests/test_exchangerate_client.py @@ -65,7 +65,6 @@ def test_check_currency_key_error(monkeypatch, capsys): test_resp.json.return_value = { "result": "success", # not passing "success" bypases the "conversion_rate" checking "error_type": "", - # "conversion_rate" fehlt absichtlich } def test_get_resp(url, timeout): diff --git a/tests/test_storage_database.py b/tests/test_storage_database.py index 1b7dad5..d3c5e00 100644 --- a/tests/test_storage_database.py +++ b/tests/test_storage_database.py @@ -3,23 +3,144 @@ import duckdb from argus.domain.internal_models import DataSource, Instrument, PriceBar -from argus.storage.database import ( - initialize_database, - insert_price_bar, - read_price_bars, -) - +from argus.storage.database import initialize_database, insert_price_bar, read_price_bars def test_initialize_database_creates_required_tables(tmp_path): - db = tmp_path / "test.duckdb" initialize_database(db) connection = duckdb.connect(db) - tables = connection.execute("SHOW TABLES;").fetchall() + connection.close() table_names = {row[0] for row in tables} assert "data_sources" in table_names assert "instruments" in table_names assert "price_bars" in table_names + +def test_data_is_inserted(tmp_path): + source = DataSource( + name="Yahoo", + provider_kind="yfinance_api", + requires_api_key=False + ) + + instrument = Instrument( + symbol="EUR/USD", + name="EUR - USD Rate", + asset_class="fx", + base_currency="EUR", + quote_currency="USD") + + pricebar = PriceBar( + source=source, + instrument=instrument, + timestamp=date(2026,1,1), + timeframe="1d", + close=1.89 + ) + + db = tmp_path / "test.duckdb" + initialize_database(db) + insert_price_bar(db,pricebar) + connection = duckdb.connect(db) + + instrument_count = connection.execute( + "SELECT COUNT(*) FROM instruments;" + ).fetchone() + + source_count = connection.execute( + "SELECT COUNT(*) FROM data_sources;" + ).fetchone() + + price_bar_count = connection.execute( + "SELECT COUNT(*) FROM price_bars;" + ).fetchone() + + assert instrument_count is not None + assert source_count is not None + assert price_bar_count is not None + assert instrument_count[0] == 1 + assert source_count[0] == 1 + assert price_bar_count[0] == 1 + +def test_fx_has_correct_format(tmp_path): + source = DataSource( + name="Yahoo", + provider_kind="yfinance_api", + requires_api_key=False + ) + + instrument = Instrument( + symbol="EUR/USD", + name="EUR - USD Rate", + asset_class="fx", + base_currency="EUR", + quote_currency="USD") + + pricebar = PriceBar( + source=source, + instrument=instrument, + timestamp=date(2026,1,1), + timeframe="1d", + close=1.89 + ) + + db = tmp_path / "test.duckdb" + initialize_database(db) + insert_price_bar(db,pricebar) + connection = duckdb.connect(db) + + price_bar_fx = connection.execute( + "SELECT * FROM price_bars;" + ).fetchone() + connection.close() + + assert price_bar_fx is not None + assert price_bar_fx[0] == 1 + assert price_bar_fx[1] == 1 + assert price_bar_fx[2] == 1 + assert price_bar_fx[3] == date(2026,1,1) + assert price_bar_fx[4] == "1d" + assert price_bar_fx[5] == 1.89 + assert price_bar_fx[6] is None + assert price_bar_fx[7] is None + assert price_bar_fx[8] is None + assert price_bar_fx[9] is None + assert price_bar_fx[10] is None + +def test_duplicates_are_ignored(tmp_path): + source = DataSource( + name="Yahoo", + provider_kind="yfinance_api", + requires_api_key=False + ) + + instrument = Instrument( + symbol="EUR/USD", + name="EUR - USD Rate", + asset_class="fx", + base_currency="EUR", + quote_currency="USD") + + pricebar = PriceBar( + source=source, + instrument=instrument, + timestamp=date(2026,1,1), + timeframe="1d", + close=1.89 + ) + + db = tmp_path / "test.duckdb" + initialize_database(db) + insert_price_bar(db,pricebar) + insert_price_bar(db,pricebar) + connection = duckdb.connect(db) + count = connection.execute( + "SELECT COUNT(*) FROM price_bars;" + ).fetchone() + + assert count is not None + assert count[0] == 1 + + From d00484e894bccf8ac16e4e0bc684e6ff6726cab9 Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Wed, 1 Jul 2026 11:09:49 +0200 Subject: [PATCH 41/44] test(#38): add read tests --- tests/test_storage_database.py | 156 ++++++++++++++++++++++++--------- 1 file changed, 116 insertions(+), 40 deletions(-) diff --git a/tests/test_storage_database.py b/tests/test_storage_database.py index d3c5e00..d513008 100644 --- a/tests/test_storage_database.py +++ b/tests/test_storage_database.py @@ -3,7 +3,12 @@ import duckdb from argus.domain.internal_models import DataSource, Instrument, PriceBar -from argus.storage.database import initialize_database, insert_price_bar, read_price_bars +from argus.storage.database import ( + initialize_database, + insert_price_bar, + read_price_bars, +) + def test_initialize_database_creates_required_tables(tmp_path): db = tmp_path / "test.duckdb" @@ -18,11 +23,10 @@ def test_initialize_database_creates_required_tables(tmp_path): assert "instruments" in table_names assert "price_bars" in table_names + def test_data_is_inserted(tmp_path): source = DataSource( - name="Yahoo", - provider_kind="yfinance_api", - requires_api_key=False + name="Yahoo", provider_kind="yfinance_api", requires_api_key=False ) instrument = Instrument( @@ -30,32 +34,29 @@ def test_data_is_inserted(tmp_path): name="EUR - USD Rate", asset_class="fx", base_currency="EUR", - quote_currency="USD") - + quote_currency="USD", + ) + pricebar = PriceBar( source=source, instrument=instrument, - timestamp=date(2026,1,1), + timestamp=date(2026, 1, 1), timeframe="1d", - close=1.89 + close=1.89, ) db = tmp_path / "test.duckdb" initialize_database(db) - insert_price_bar(db,pricebar) + insert_price_bar(db, pricebar) connection = duckdb.connect(db) instrument_count = connection.execute( "SELECT COUNT(*) FROM instruments;" ).fetchone() - source_count = connection.execute( - "SELECT COUNT(*) FROM data_sources;" - ).fetchone() + source_count = connection.execute("SELECT COUNT(*) FROM data_sources;").fetchone() - price_bar_count = connection.execute( - "SELECT COUNT(*) FROM price_bars;" - ).fetchone() + price_bar_count = connection.execute("SELECT COUNT(*) FROM price_bars;").fetchone() assert instrument_count is not None assert source_count is not None @@ -64,11 +65,10 @@ def test_data_is_inserted(tmp_path): assert source_count[0] == 1 assert price_bar_count[0] == 1 + def test_fx_has_correct_format(tmp_path): source = DataSource( - name="Yahoo", - provider_kind="yfinance_api", - requires_api_key=False + name="Yahoo", provider_kind="yfinance_api", requires_api_key=False ) instrument = Instrument( @@ -76,31 +76,30 @@ def test_fx_has_correct_format(tmp_path): name="EUR - USD Rate", asset_class="fx", base_currency="EUR", - quote_currency="USD") - + quote_currency="USD", + ) + pricebar = PriceBar( source=source, instrument=instrument, - timestamp=date(2026,1,1), + timestamp=date(2026, 1, 1), timeframe="1d", - close=1.89 + close=1.89, ) db = tmp_path / "test.duckdb" initialize_database(db) - insert_price_bar(db,pricebar) + insert_price_bar(db, pricebar) connection = duckdb.connect(db) - price_bar_fx = connection.execute( - "SELECT * FROM price_bars;" - ).fetchone() + price_bar_fx = connection.execute("SELECT * FROM price_bars;").fetchone() connection.close() assert price_bar_fx is not None assert price_bar_fx[0] == 1 assert price_bar_fx[1] == 1 assert price_bar_fx[2] == 1 - assert price_bar_fx[3] == date(2026,1,1) + assert price_bar_fx[3] == date(2026, 1, 1) assert price_bar_fx[4] == "1d" assert price_bar_fx[5] == 1.89 assert price_bar_fx[6] is None @@ -109,11 +108,10 @@ def test_fx_has_correct_format(tmp_path): assert price_bar_fx[9] is None assert price_bar_fx[10] is None + def test_duplicates_are_ignored(tmp_path): source = DataSource( - name="Yahoo", - provider_kind="yfinance_api", - requires_api_key=False + name="Yahoo", provider_kind="yfinance_api", requires_api_key=False ) instrument = Instrument( @@ -121,26 +119,104 @@ def test_duplicates_are_ignored(tmp_path): name="EUR - USD Rate", asset_class="fx", base_currency="EUR", - quote_currency="USD") - + quote_currency="USD", + ) + pricebar = PriceBar( source=source, instrument=instrument, - timestamp=date(2026,1,1), + timestamp=date(2026, 1, 1), timeframe="1d", - close=1.89 + close=1.89, ) db = tmp_path / "test.duckdb" initialize_database(db) - insert_price_bar(db,pricebar) - insert_price_bar(db,pricebar) + insert_price_bar(db, pricebar) + insert_price_bar(db, pricebar) connection = duckdb.connect(db) - count = connection.execute( - "SELECT COUNT(*) FROM price_bars;" - ).fetchone() + count = connection.execute("SELECT COUNT(*) FROM price_bars;").fetchone() assert count is not None assert count[0] == 1 - + +def test_read_price_bars_returns_matching_data(tmp_path): + source = DataSource( + name="Yahoo", + provider_kind="yfinance_api", + requires_api_key=False, + ) + + instrument = Instrument( + symbol="EUR/USD", + name="EUR - USD Rate", + asset_class="fx", + base_currency="EUR", + quote_currency="USD", + ) + + pricebar = PriceBar( + source=source, + instrument=instrument, + timestamp=date(2026, 1, 1), + timeframe="1d", + close=1.89, + ) + + db = tmp_path / "test.duckdb" + initialize_database(db) + insert_price_bar(db, pricebar) + + result = read_price_bars( + db=db, + source=source, + instrument=instrument, + start_date=date(2026, 1, 1), + end_date=date(2026, 1, 31), + ) + + assert result.empty is False + assert len(result) == 1 + assert result.iloc[0]["source_name"] == "Yahoo" + assert result.iloc[0]["instrument_symbol"] == "EUR/USD" + assert result.iloc[0]["timeframe"] == "1d" + assert result.iloc[0]["close"] == 1.89 + + +def test_read_price_bars_returns_empty_dataframe_for_missing_range(tmp_path): + source = DataSource( + name="Yahoo", + provider_kind="yfinance_api", + requires_api_key=False, + ) + + instrument = Instrument( + symbol="EUR/USD", + name="EUR - USD Rate", + asset_class="fx", + base_currency="EUR", + quote_currency="USD", + ) + + pricebar = PriceBar( + source=source, + instrument=instrument, + timestamp=date(2026, 1, 1), + timeframe="1d", + close=1.89, + ) + + db = tmp_path / "test.duckdb" + initialize_database(db) + insert_price_bar(db, pricebar) + + result = read_price_bars( + db=db, + source=source, + instrument=instrument, + start_date=date(2027, 1, 1), + end_date=date(2027, 1, 31), + ) + + assert result.empty is True From d2f515927cabbe5b1651156e7e45fd2d8fb057aa Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Wed, 1 Jul 2026 11:59:55 +0200 Subject: [PATCH 42/44] docs(#38): doc for storage --- README.md | 27 ++++----- docs/roadmap.md | 64 ++++++++++---------- docs/storage.md | 154 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 197 insertions(+), 48 deletions(-) create mode 100644 docs/storage.md diff --git a/README.md b/README.md index 1c0d156..8723389 100644 --- a/README.md +++ b/README.md @@ -125,6 +125,13 @@ README.md - ExchangeRate API for live currency conversion - yfinance for historical market-data retrieval and analytics +### Storage + +- DuckDB — local analytical storage for normalized historical market data + +>[!Note] +> See docs/storage.md for details. + --- ## Planned / Future Tech Stack @@ -138,40 +145,32 @@ Planned or likely future technologies include: - Frankfurter API for historical FX data - possible additional market-data APIs later -### Data processing - -- possibly Polars later for larger datasets - ### Storage - PostgreSQL -- DuckDB -- Parquet -- optional cloud storage ### Visualization and UI - NiceGUI +- Django ### DevOps and deployment - Docker Compose -- cloud deployment later +- Travis CI ### Cloud and data engineering -- Azure, GCP or AWS depending on project direction +- Azure - scheduled ingestion -- data quality checks -- reporting pipelines +- agentic Workflows +- Blob Storage +- scaled analysis ### AI and agentic workflows - LLM-assisted summaries - RAG over stored reports or notes -- agentic data checks -- anomaly monitoring -- human-in-the-loop signal review > [!CAUTION] > AI and agentic features are future-stage ideas. diff --git a/docs/roadmap.md b/docs/roadmap.md index 85706af..bc2cdbd 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -41,17 +41,19 @@ Scope: - rolling volatility - performance analytics - risk analytics -- Extend the current dashboard without adding unnecessary chart noise +- Extend the current dashboard - Add or evaluate new data clients: - - Frankfurter for historical FX data - yfinance for broader market data -- Replace or reduce dependency on the current ExchangeRate API where needed - Improve pandas-based analysis workflows +- Add a local storage for historical market data +- Add report generation and export +- add first prediction feature +- Introduce NiceGUI as a new GUI - Add tests for metric calculations and data transformations -- Document metric definitions, assumptions and chart behavior +- Add CD Pipeline Outcome: -ARGUS becomes a basic market analytics tool, not only a converter. +ARGUS becomes a basic market analytics tool ### Sprint 3 — Storage, Web-Ready UI & Data Architecture @@ -61,53 +63,45 @@ Prepare ARGUS for persistent data workflows and a stronger product interface. Scope: -- Add local storage layer: - - PostgreSQL, DuckDB, SQLite or Parquet depending on use case -- Store historical market data -- Separate ingestion, transformation, analytics and presentation layers more clearly -- Start NiceGUI as the main web-ready UI direction +- Extend local storage layer +- First local ETL Pipeline +- Extend NiceGUI and plan how to combine with modern frotend techstack like django and node.js - Keep Tkinter as legacy/prototype unless still useful -- Keep CLI as internal/debug interface only -- Add clearer architecture documentation -- Prepare the project for larger data workflows and external contributors +- More metrics, more instruments and more (and better) prediction features +- Introduce first LLM summary for reports +- Introduce Snyk and Performance Test to cover perfomance and security of argus +- Improve Code Quality Outcome: -ARGUS has a clearer data architecture and starts moving from local prototype toward a scalable analytics application. +ARGUS is a scalable analytics application that allows to get more insight from market data -### Sprint 4 — Cloud, Pipelines & Portfolio-Grade Data Engineering +### Sprint 4 — Introduction for extended Analysis -**Status:** Future +**Status:** Planned -Turn ARGUS into a stronger end-to-end data engineering project. +Turn ARGUS into a stronger end-to-end data engineering project which is cloud ready. Scope: -- Docker / Docker Compose -- Scheduled data ingestion -- Cloud storage or cloud database -- CI/CD improvements +- Docker Compose +- Intorduce Azure (a simple connection - storage only) +- Better LLM Workflow (introduce RAG) - Data quality checks -- Basic pipeline orchestration -- Reporting layer -- Architecture diagram -- Deployment documentation +- Caching and efficient storing of market data +- More export possibilities for users +- More metrics and better meta data visualization -Target workflow: +Outcome: -```text -API → Ingestion → Storage → Transformation → Analysis → Visualization → CI/CD -``` +ARGUS is ready to interact with the cloud layer and future cloud app. It's able to give the user an transparent and clear analysis of requested market section. ### Sprint 5 — AI-Assisted Research & Agentic Monitoring **Status:** Future vision -Add AI support only after the data, storage, service and reporting layers are stable. - Scope: -- LLM-assisted report summaries -- Explanation of unusual movements +- First Cloud workflows to extend the analysis - RAG over stored market notes, reports or documentation - Agentic checks for data quality, anomalies and recurring market scans - Human-in-the-loop signal review @@ -115,4 +109,6 @@ Scope: Outcome: -ARGUS starts behaving like its name: a system that continuously watches market data, evaluates it and helps generate useful signals. +ARGUS and the cloud app interact with each other. ARGUS become the first time an useful monitoring and alysis tool. +It's the beginn of ARGUS to help the user to find, implement and deploy strategies. Through that ARGUS will +be first time able to give signals and allow paper trading, back tests and controlled trading with agents. diff --git a/docs/storage.md b/docs/storage.md new file mode 100644 index 0000000..27ce106 --- /dev/null +++ b/docs/storage.md @@ -0,0 +1,154 @@ +# ARGUS Storage Layer + +ARGUS uses DuckDB as the local storage layer for normalized market data. + +The storage layer stores ARGUS-internal market data structures and provides reusable historical data for analytics, charts, dashboards and reports. + +The storage design follows the direction described in [`docs/research-databases-and-storage.md`](research-databases-and-storage.md). + +## Storage Workflow + +ARGUS uses a storage-first workflow for historical market data. + +```text +User / GUI / Analytics request + ↓ +Market data service + ↓ +Check DuckDB storage + ↓ +If data exists: + read stored data + return it for analytics, charts or reports + +If data is missing: + fetch data from a client/API + normalize the response into ARGUS-internal data + return the normalized data + save the normalized data in DuckDB +``` + +DuckDB is used to avoid unnecessary repeated API calls and to make historical market data reusable across analytics, dashboard and reporting workflows. + +Fresh API data can be used immediately after normalization and is also persisted so future requests can use the local storage layer first. + +## Schema Overview + +The first storage schema is based on three related entities: + +```text +data_sources +instruments +price_bars +``` + +### `data_sources` + +Stores where market data came from. + +Examples: + +```text +yfinance +ExchangeRate API +Frankfurter +FRED +``` + +Each source describes a provider or API that can deliver market, FX or macro data. + +### `instruments` + +Stores what ARGUS can analyze. + +Examples: + +```text +EUR/USD +AAPL +SPY +BTC-USD +``` + +An instrument represents the internal ARGUS identity of an asset, currency pair, ETF, index or other market object. + +Provider-specific symbols should be normalized before storage. For example: + +```text +yfinance provider symbol: EURUSD=X +ARGUS instrument symbol: EUR/USD +``` + +### `price_bars` + +Stores historical time-series values in an OHLCV-ready structure. + +A price bar belongs to: + +```text +one data source +one instrument +one timestamp +one timeframe +``` + +FX rates are stored as `close` values. + +For simple FX data, the remaining OHLCV fields can stay empty. For broader market data, the same structure can store open, high, low, close, adjusted close and volume values. + +The combination of source, instrument, timestamp and timeframe identifies a unique stored price bar. + +## Internal Models and Storage + +ARGUS uses internal domain models before data is stored: + +```text +DataSource +Instrument +PriceBar +``` + +These models describe the meaning of the data inside ARGUS. + +The storage layer translates these internal models into DuckDB tables: + +```text +DataSource -> data_sources +Instrument -> instruments +PriceBar -> price_bars +``` + +In Python, a `PriceBar` references a `DataSource` and an `Instrument`. + +In DuckDB, this relationship is stored through IDs: + +```text +price_bars.source_id -> data_sources.id +price_bars.instrument_id -> instruments.id +``` + +This keeps the database normalized while still allowing ARGUS to work with meaningful internal models in Python. + +## Reading Stored Data + +Stored price bars can be read by: + +```text +source +instrument +start date +end date +``` + +The storage layer joins `price_bars`, `data_sources` and `instruments` so that stored IDs become readable market data again. + +Read operations return tabular data that can be used by: + +```text +analytics +charts +dashboards +reports +``` + +This allows ARGUS to process stored historical data without depending on raw API response structures. From b930ff8bb021a195125b0e586a98c11c8f2fc1d3 Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Wed, 1 Jul 2026 12:07:46 +0200 Subject: [PATCH 43/44] docs(#38): update roadmap --- docs/roadmap.md | 137 ++++++++++++++++++++++++++++++------------------ 1 file changed, 85 insertions(+), 52 deletions(-) diff --git a/docs/roadmap.md b/docs/roadmap.md index bc2cdbd..c637327 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -27,88 +27,121 @@ Scope: Outcome: Sprint 1 established the local ARGUS foundation with package structure, GUI prototype, analytics prototype, tests, documentation, CI, Dependabot and governance files. -### Sprint 2 — Market Analytics & Data Source Expansion +### Sprint 2 — Reporting & Market Analytics Foundation **Status:** In progress -Move from simple FX conversion toward broader market analytics. +Move ARGUS from a simple FX-focused prototype toward a first usable market analytics and reporting tool. -Scope: +**Scope:** + +- Add stronger market analytics metrics: -- Add stronger market metrics: - cumulative return - strongest / weakest day - rolling volatility - - performance analytics - - risk analytics -- Extend the current dashboard -- Add or evaluate new data clients: + - basic performance analytics + - basic risk analytics +- Add or improve real market data support: + - yfinance for broader market data + - existing FX conversion remains available where useful - Improve pandas-based analysis workflows -- Add a local storage for historical market data +- Introduce local storage for historical market data - Add report generation and export -- add first prediction feature -- Introduce NiceGUI as a new GUI -- Add tests for metric calculations and data transformations -- Add CD Pipeline +- Add a first simple prediction feature +- Introduce NiceGUI as the next GUI direction +- Extend the current dashboard with real market analytics +- Add tests for metric calculations, data transformations and storage behavior +- Improve CI/CD with first deployment or release automation steps -Outcome: -ARGUS becomes a basic market analytics tool +**Outcome:** + +ARGUS becomes a basic market analytics and reporting tool. +Users can fetch market data, store it locally, calculate metrics, generate a first report and view results through a first modern dashboard. -### Sprint 3 — Storage, Web-Ready UI & Data Architecture +--- + +### Sprint 3 — Advanced Local Analytics & Product Quality **Status:** Planned -Prepare ARGUS for persistent data workflows and a stronger product interface. +Expand the local ARGUS application into a stronger analytics product with better data handling, UI structure, predictions and quality checks. -Scope: +**Scope:** -- Extend local storage layer -- First local ETL Pipeline -- Extend NiceGUI and plan how to combine with modern frotend techstack like django and node.js -- Keep Tkinter as legacy/prototype unless still useful -- More metrics, more instruments and more (and better) prediction features -- Introduce first LLM summary for reports -- Introduce Snyk and Performance Test to cover perfomance and security of argus -- Improve Code Quality +- Extend the local storage layer +- Add a first local ETL workflow +- Improve the NiceGUI dashboard structure and usability +- Explore how NiceGUI can later interact with a more modern frontend stack such as Django, React or Node.js-based services +- Keep Tkinter as legacy/prototype unless it is no longer useful +- Add more metrics, instruments and prediction features +- Improve report templates and report structure +- Introduce first LLM-based summaries for generated reports +- Add first performance tests +- Introduce Snyk or another dependency/security scanning workflow +- Improve code quality, test coverage and maintainability -Outcome: -ARGUS is a scalable analytics application that allows to get more insight from market data +**Outcome:** + +ARGUS becomes a more scalable local analytics application. +It can process more instruments, produce better reports, provide first automated summaries and offer more reliable insight into market data. + +--- -### Sprint 4 — Introduction for extended Analysis +### Sprint 4 — Extended Analysis & Cloud-Ready Foundation **Status:** Planned -Turn ARGUS into a stronger end-to-end data engineering project which is cloud ready. +Prepare ARGUS for deeper analysis, cloud interaction and future portfolio-assistant workflows while keeping the local product usable and transparent. -Scope: +**Scope:** -- Docker Compose -- Intorduce Azure (a simple connection - storage only) -- Better LLM Workflow (introduce RAG) -- Data quality checks -- Caching and efficient storing of market data -- More export possibilities for users -- More metrics and better meta data visualization +- Add Docker Compose for a more complete local development setup +- Introduce a first Azure connection, focused on simple storage or artifact exchange +- Improve the LLM workflow +- Introduce a first RAG-ready structure for reports, notes, documentation and stored analysis artifacts +- Add data quality checks +- Improve caching and efficient storage of market data +- Add more export options for users +- Add more metrics and better metadata visualization +- Improve transparency around data sources, generated reports and analysis assumptions +- Prepare clear interfaces for future cloud and assistant workflows -Outcome: +**Outcome:** -ARGUS is ready to interact with the cloud layer and future cloud app. It's able to give the user an transparent and clear analysis of requested market section. +ARGUS becomes ready to interact with a future cloud layer. +The application can produce clearer, more transparent market analysis and prepares the foundation for retrieval-based workflows, stronger automation and future ARGUS Core integration. -### Sprint 5 — AI-Assisted Research & Agentic Monitoring +--- -**Status:** Future vision +### Sprint 5 — Cloud Interaction & Agentic Monitoring Foundation -Scope: +**Status:** Planned -- First Cloud workflows to extend the analysis -- RAG over stored market notes, reports or documentation -- Agentic checks for data quality, anomalies and recurring market scans -- Human-in-the-loop signal review -- Automated monitoring workflows +Start the first cloud-connected ARGUS workflows and introduce the foundation for monitoring, agentic checks and strategy-support features. -Outcome: +**Scope:** + +- Add first cloud workflows that extend local analysis +- Connect local ARGUS workflows with the first cloud-side services +- Extend RAG over stored market notes, reports, documentation and analysis artifacts +- Add agentic checks for: + + - data quality + - anomalies + - recurring market scans + - report consistency +- Add first human-in-the-loop review workflows for signals or strategy ideas +- Add automated monitoring workflows +- Prepare the first foundations for: + + - paper trading + - backtesting + - controlled strategy evaluation + - future portfolio-assistant workflows + +**Outcome:** -ARGUS and the cloud app interact with each other. ARGUS become the first time an useful monitoring and alysis tool. -It's the beginn of ARGUS to help the user to find, implement and deploy strategies. Through that ARGUS will -be first time able to give signals and allow paper trading, back tests and controlled trading with agents. +ARGUS and the first cloud-side services begin to interact. +ARGUS becomes useful not only as an analytics and reporting tool, but also as the first foundation for monitoring, strategy evaluation and controlled market-research workflows. From 90e11de118e429e6c0d9f163a697b58dd59144bd Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Wed, 1 Jul 2026 12:12:17 +0200 Subject: [PATCH 44/44] docs(#38): add docstrings --- src/argus/storage/database.py | 47 +++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/src/argus/storage/database.py b/src/argus/storage/database.py index f184b65..ea7128c 100644 --- a/src/argus/storage/database.py +++ b/src/argus/storage/database.py @@ -5,6 +5,18 @@ def initialize_database(database_path: str) -> None: + """ + Initialize the DuckDB database schema. + + Creates the required sequences and tables for data sources, + instruments, and price bars. + + Args: + database_path (str): Path to the DuckDB database file. + + Returns: + None + """ queries = [ "CREATE SEQUENCE IF NOT EXISTS data_sources_id_seq;", "CREATE SEQUENCE IF NOT EXISTS instruments_id_seq;", @@ -58,6 +70,23 @@ def initialize_database(database_path: str) -> None: def get_or_create_source(connection, source: DataSource) -> int: + """ + Get an existing data source ID or create a new data source. + + Searches for a data source by name. If it already exists, its ID is + returned. Otherwise, the data source is inserted and the new ID is + returned. + + Args: + connection: Active DuckDB connection. + source (DataSource): Data source model containing provider metadata. + + Returns: + int: Database ID of the existing or newly created data source. + + Raises: + ValueError: If the data source could not be inserted or found. + """ insert_query = """ INSERT INTO data_sources (name, provider_kind, requires_api_key) VALUES (?,?,?) @@ -92,6 +121,24 @@ def get_or_create_source(connection, source: DataSource) -> int: def get_or_create_instrument(connection, instrument: Instrument) -> int: + """ + Get an existing instrument ID or create a new instrument. + + Searches for an instrument by symbol. If it already exists, its ID is + returned. Otherwise, the instrument is inserted and the new ID is + returned. + + Args: + connection: Active DuckDB connection. + instrument (Instrument): Instrument model containing symbol and + asset metadata. + + Returns: + int: Database ID of the existing or newly created instrument. + + Raises: + ValueError: If the instrument could not be inserted or found. + """ insert_query = """ INSERT INTO instruments ( symbol,