From 6f613b2835ad4ba362ad2840c4a5debc07a408bc Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Mon, 29 Jun 2026 12:41:19 +0200 Subject: [PATCH 01/19] feat(#38): create db foundation --- pyproject.toml | 1 + src/argus/storage/__init__.py | 0 src/argus/storage/database.py | 0 3 files changed, 1 insertion(+) create mode 100644 src/argus/storage/__init__.py create mode 100644 src/argus/storage/database.py diff --git a/pyproject.toml b/pyproject.toml index e2dc784..a8ac7b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,7 @@ dependencies = [ "numpy", "matplotlib", "yfinance", + "duckdb", ] [project.optional-dependencies] diff --git a/src/argus/storage/__init__.py b/src/argus/storage/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/argus/storage/database.py b/src/argus/storage/database.py new file mode 100644 index 0000000..e69de29 From d232222db347f0edcd1fa1b258f816a579e75b22 Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Mon, 29 Jun 2026 13:33:32 +0200 Subject: [PATCH 02/19] feat(#38): add initt_db func --- src/argus/storage/database.py | 42 +++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/argus/storage/database.py b/src/argus/storage/database.py index e69de29..b1a5317 100644 --- a/src/argus/storage/database.py +++ b/src/argus/storage/database.py @@ -0,0 +1,42 @@ +import duckdb + +def initialize_database(database_path: str) -> None: + create_datasources_table = """ + CREATE TABLE IF NOT EXISTS data_sources ( + id INTEGER PRIMARY_KEY, + name TEXT, + provider_kind TEXT, + requires_api_key: BOOLEAN DEFAULTS:False + ) + """ + create_intstrument_table = """ + CREATE TABLE IF NOT EXISTS instrument ( + id INTEGER PRIMARY_KEY, + name TEXT, + asset_class TEXT, + currency TEXT or NONE DEFAULTS: NONE, + exchange TEXT or NONE DEFAULTS: NONE, + base_currency TEXT or NONE DEFAULTS: NONE, + quote_currency TEXT or NONE DEFAULTS: NONE + ) + """ + create_price_bar_table = """ + CREATE TABLE IF NOT EXISTS price_bar ( + id INTEGER PRIMARY_KEY, + source_id FOREIGN_KEY, + instrument_id FOREIGN_KEY, + timestamp: date, + timeframe TEXT, + close FLOAT, + open: FLOAT or NONE DEFAULTS: NONE, + high: FLOAT or NONE DEFAULTS: NONE, + low: FLOAT or NONE DEFAULTS: NONE, + adjusted_close FLOAT or NONE DEFAULTS: NONE, + volume: FLOAT or NONE DEFAULTS: NONE + ) + """ + duckdb.connect(database_path) + duckdb.execute(query=create_datasources_table) + duckdb.execute(query=create_intstrument_table) + duckdb.execute(query=create_price_bar_table) + duckdb.close() From b5368eee5b067796033bc62971b21b0ce248cdd2 Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Mon, 29 Jun 2026 13:37:59 +0200 Subject: [PATCH 03/19] feat(#38): add id sequence --- src/argus/storage/database.py | 70 +++++++++++++++++++---------------- 1 file changed, 39 insertions(+), 31 deletions(-) diff --git a/src/argus/storage/database.py b/src/argus/storage/database.py index b1a5317..8e564b1 100644 --- a/src/argus/storage/database.py +++ b/src/argus/storage/database.py @@ -1,42 +1,50 @@ import duckdb + def initialize_database(database_path: str) -> None: + create_data_sources_sequence = """ + CREATE SEQUENCE IF NOT EXISTS data_sources_id_seq; + """ create_datasources_table = """ CREATE TABLE IF NOT EXISTS data_sources ( - id INTEGER PRIMARY_KEY, - name TEXT, - provider_kind TEXT, - requires_api_key: BOOLEAN DEFAULTS:False - ) + id INTEGER PRIMARY KEY DEFAULT keyval('data_sources_id_seq'), + name TEXT NOT NULL UNIQUE, + provider_kind TEXT NOT NULL, + requires_api_key BOOLEAN NOT NULL + ); """ - create_intstrument_table = """ - CREATE TABLE IF NOT EXISTS instrument ( - id INTEGER PRIMARY_KEY, - name TEXT, - asset_class TEXT, - currency TEXT or NONE DEFAULTS: NONE, - exchange TEXT or NONE DEFAULTS: NONE, - base_currency TEXT or NONE DEFAULTS: NONE, - quote_currency TEXT or NONE DEFAULTS: NONE - ) + create_intstruments_table = """ + CREATE TABLE IF NOT EXISTS instruments ( + id INTEGER PRIMARY KEY DEFAULT keyval('data_sources_id_seq'), + symbol TEXT NOT NULL UNIQUE, + name TEXT NOT NULL, + asset_class TEXT NOT NULL, + currency TEXT, + exchange TEXT, + base_currency TEXT, + quote_currency TEXT + ); """ - create_price_bar_table = """ - CREATE TABLE IF NOT EXISTS price_bar ( - id INTEGER PRIMARY_KEY, - source_id FOREIGN_KEY, - instrument_id FOREIGN_KEY, - timestamp: date, - timeframe TEXT, - close FLOAT, - open: FLOAT or NONE DEFAULTS: NONE, - high: FLOAT or NONE DEFAULTS: NONE, - low: FLOAT or NONE DEFAULTS: NONE, - adjusted_close FLOAT or NONE DEFAULTS: NONE, - volume: FLOAT or NONE DEFAULTS: NONE - ) + create_price_bars_table = """ + CREATE TABLE IF NOT EXISTS price_bars ( + id INTEGER PRIMARY KEY DEFAULT keyval('data_sources_id_seq'), + source_id INTEGER NOT NULL, + instrument_id INTEGER NOT NULL, + timestamp DATE NOT NULL, + timeframe TEXT NOT NULL, + close DOUBLE NOT NULL, + open DOUBLE, + high DOUBLE, + low DOUBLE, + adjusted_close DOUBLE, + volume DOUBLE, + FOREIGN KEY (source_id) REFERENCES data_sources (id), + FOREIGN KEY (instrument_id) REFERENCES instruments (id) + ); """ duckdb.connect(database_path) + duckdb.execute(query=create_data_sources_sequence) duckdb.execute(query=create_datasources_table) - duckdb.execute(query=create_intstrument_table) - duckdb.execute(query=create_price_bar_table) + duckdb.execute(query=create_intstruments_table) + duckdb.execute(query=create_price_bars_table) duckdb.close() From 9560b118b92f8df5967aa29940671c9b203f7e32 Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Mon, 29 Jun 2026 14:05:55 +0200 Subject: [PATCH 04/19] feat(#38): insert logic for source --- src/argus/storage/database.py | 51 ++++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 9 deletions(-) diff --git a/src/argus/storage/database.py b/src/argus/storage/database.py index 8e564b1..8eda7d5 100644 --- a/src/argus/storage/database.py +++ b/src/argus/storage/database.py @@ -5,9 +5,15 @@ def initialize_database(database_path: str) -> None: create_data_sources_sequence = """ CREATE SEQUENCE IF NOT EXISTS data_sources_id_seq; """ + create_instruments_sequence = """ + CREATE SEQUENCE IF NOT EXISTS instruemnts_id_seq; + """ + create_price_bars_sequence = """ + CREATE SEQUENCE IF NOT EXISTS price_bars_id_seq; + """ create_datasources_table = """ CREATE TABLE IF NOT EXISTS data_sources ( - id INTEGER PRIMARY KEY DEFAULT keyval('data_sources_id_seq'), + id INTEGER PRIMARY KEY DEFAULT nextval('data_sources_id_seq'), name TEXT NOT NULL UNIQUE, provider_kind TEXT NOT NULL, requires_api_key BOOLEAN NOT NULL @@ -15,7 +21,7 @@ def initialize_database(database_path: str) -> None: """ create_intstruments_table = """ CREATE TABLE IF NOT EXISTS instruments ( - id INTEGER PRIMARY KEY DEFAULT keyval('data_sources_id_seq'), + id INTEGER PRIMARY KEY DEFAULT nextval('instruemnts_id_seq'), symbol TEXT NOT NULL UNIQUE, name TEXT NOT NULL, asset_class TEXT NOT NULL, @@ -27,7 +33,7 @@ def initialize_database(database_path: str) -> None: """ create_price_bars_table = """ CREATE TABLE IF NOT EXISTS price_bars ( - id INTEGER PRIMARY KEY DEFAULT keyval('data_sources_id_seq'), + id INTEGER PRIMARY KEY DEFAULT nextval('price_bars_id_seq'), source_id INTEGER NOT NULL, instrument_id INTEGER NOT NULL, timestamp DATE NOT NULL, @@ -42,9 +48,36 @@ def initialize_database(database_path: str) -> None: FOREIGN KEY (instrument_id) REFERENCES instruments (id) ); """ - duckdb.connect(database_path) - duckdb.execute(query=create_data_sources_sequence) - duckdb.execute(query=create_datasources_table) - duckdb.execute(query=create_intstruments_table) - duckdb.execute(query=create_price_bars_table) - duckdb.close() + connection = duckdb.connect(database_path) + + connection.execute(query=create_data_sources_sequence) + connection.execute(query=create_instruments_sequence) + connection.execute(query=create_price_bars_sequence) + + connection.execute(query=create_datasources_table) + connection.execute(query=create_intstruments_table) + connection.execute(query=create_price_bars_table) + + connection.close() + +def insert_data_source(database_path, source): + connection = duckdb.connect(database_path) + + source = """ + INSERT INTO data_sources (name, provider_kind, requires_api_key) + VALUES (?,?,?); + """ + + connection.execute(query=source) + connection.close() + +def insert_instruemnt(database_path, source): + connection = duckdb.connect(database_path) + + source = """ + INSERT INTO instruments (symbol,name,asset_class) + VALUES (?,?,?); + """ + + connection.execute(query=source) + connection.close() From 4f44f750a62819db1494580c06533abddb508d57 Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Mon, 29 Jun 2026 14:14:11 +0200 Subject: [PATCH 05/19] feat(#38): insert logic - instrument --- src/argus/storage/database.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/src/argus/storage/database.py b/src/argus/storage/database.py index 8eda7d5..f56183a 100644 --- a/src/argus/storage/database.py +++ b/src/argus/storage/database.py @@ -1,4 +1,5 @@ import duckdb +from argus.domain.internal_models import DataSource, PriceBar, Instrument def initialize_database(database_path: str) -> None: @@ -57,27 +58,32 @@ def initialize_database(database_path: str) -> None: connection.execute(query=create_datasources_table) connection.execute(query=create_intstruments_table) connection.execute(query=create_price_bars_table) - + connection.close() -def insert_data_source(database_path, source): - connection = duckdb.connect(database_path) - source = """ +def insert_data_source(database_path, source: DataSource) -> None: + insert_query = """ INSERT INTO data_sources (name, provider_kind, requires_api_key) VALUES (?,?,?); """ - - connection.execute(query=source) + connection = duckdb.connect(database_path) + connection.execute( + query=insert_query, + parameters=[source.name, source.provider_kind, source.requires_api_key], + ) connection.close() -def insert_instruemnt(database_path, source): - connection = duckdb.connect(database_path) - source = """ +def insert_instruemnt(database_path, instrument: Instrument) -> None: + insert_query = """ INSERT INTO instruments (symbol,name,asset_class) VALUES (?,?,?); """ - - connection.execute(query=source) + + connection = duckdb.connect(database_path) + connection.execute( + query=insert_query, + parameters=[instrument.symbol, instrument.name, instrument.asset_class], + ) connection.close() From 476379957122a3371e0266315fdb840de3bbcffe Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Mon, 29 Jun 2026 14:20:13 +0200 Subject: [PATCH 06/19] feat(#38): insert logic - price bar --- src/argus/storage/database.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/argus/storage/database.py b/src/argus/storage/database.py index f56183a..b1fdd21 100644 --- a/src/argus/storage/database.py +++ b/src/argus/storage/database.py @@ -87,3 +87,17 @@ def insert_instruemnt(database_path, instrument: Instrument) -> None: parameters=[instrument.symbol, instrument.name, instrument.asset_class], ) connection.close() + + +def insert_pirce_bar(database_path, price_bar: PriceBar) -> None: + insert_query = """ + INSERT INTO instruments (timestamp,timeframe,close) + VALUES (?,?,?); + """ + + connection = duckdb.connect(database_path) + connection.execute( + query=insert_query, + parameters=[price_bar.timestamp, price_bar.timeframe, price_bar.close], + ) + connection.close() From 2b32db43337a8808727b49a6d20b8501baa8fba5 Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Mon, 29 Jun 2026 14:39:01 +0200 Subject: [PATCH 07/19] feat(#38): add id search for bar --- src/argus/storage/database.py | 42 ++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/src/argus/storage/database.py b/src/argus/storage/database.py index b1fdd21..d3af5f7 100644 --- a/src/argus/storage/database.py +++ b/src/argus/storage/database.py @@ -7,7 +7,7 @@ def initialize_database(database_path: str) -> None: CREATE SEQUENCE IF NOT EXISTS data_sources_id_seq; """ create_instruments_sequence = """ - CREATE SEQUENCE IF NOT EXISTS instruemnts_id_seq; + CREATE SEQUENCE IF NOT EXISTS instruements_id_seq; """ create_price_bars_sequence = """ CREATE SEQUENCE IF NOT EXISTS price_bars_id_seq; @@ -22,7 +22,7 @@ def initialize_database(database_path: str) -> None: """ create_intstruments_table = """ CREATE TABLE IF NOT EXISTS instruments ( - id INTEGER PRIMARY KEY DEFAULT nextval('instruemnts_id_seq'), + id INTEGER PRIMARY KEY DEFAULT nextval('instruements_id_seq'), symbol TEXT NOT NULL UNIQUE, name TEXT NOT NULL, asset_class TEXT NOT NULL, @@ -75,7 +75,7 @@ def insert_data_source(database_path, source: DataSource) -> None: connection.close() -def insert_instruemnt(database_path, instrument: Instrument) -> None: +def insert_instruement(database_path, instrument: Instrument) -> None: insert_query = """ INSERT INTO instruments (symbol,name,asset_class) VALUES (?,?,?); @@ -88,16 +88,42 @@ def insert_instruemnt(database_path, instrument: Instrument) -> None: ) connection.close() +def find_data_source_id(database_path,price_bar: PriceBar): + search_query = """ + SELECT id FROM data_sources + WHERE name=? + """ + connection = duckdb.connect(database_path) + result = connection.execute( + query=search_query, + parameters=[price_bar.source.name], + ) + connection.close() + return result -def insert_pirce_bar(database_path, price_bar: PriceBar) -> None: - insert_query = """ - INSERT INTO instruments (timestamp,timeframe,close) - VALUES (?,?,?); +def find_instrument_id(database_path,price_bar: PriceBar): + search_query = """ + SELECT id FROM instruments + WHERE name=? """ + connection = duckdb.connect(database_path) + connection.execute( + query=search_query, + parameters=[price_bar.instrument.name], + ) + connection.close() +def insert_price_bar(database_path, price_bar: PriceBar) -> None: + insert_query = """ + INSERT INTO price_bars (source_id,instrument_id,timestamp,timeframe,close) + VALUES (?,?,?,?,?); + """ + data_source_id = find_data_source_id(database_path, price_bar) + instrument_id = find_instrument_id(database_path, price_bar) + connection = duckdb.connect(database_path) connection.execute( query=insert_query, - parameters=[price_bar.timestamp, price_bar.timeframe, price_bar.close], + parameters=[data_source_id,instrument_id,price_bar.timestamp, price_bar.timeframe, price_bar.close], ) connection.close() From 76656b262694ff09bdd61a84df6824d4c44b02e5 Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Mon, 29 Jun 2026 15:07:12 +0200 Subject: [PATCH 08/19] feat(#38): improve param select --- src/argus/storage/database.py | 88 +++++++++++++++++++++++++++-------- 1 file changed, 68 insertions(+), 20 deletions(-) diff --git a/src/argus/storage/database.py b/src/argus/storage/database.py index d3af5f7..61d7b13 100644 --- a/src/argus/storage/database.py +++ b/src/argus/storage/database.py @@ -1,4 +1,5 @@ import duckdb +import pandas as pd from argus.domain.internal_models import DataSource, PriceBar, Instrument @@ -62,7 +63,7 @@ def initialize_database(database_path: str) -> None: connection.close() -def insert_data_source(database_path, source: DataSource) -> None: +def insert_data_source(database_path: str, source: DataSource) -> None: insert_query = """ INSERT INTO data_sources (name, provider_kind, requires_api_key) VALUES (?,?,?); @@ -75,10 +76,17 @@ def insert_data_source(database_path, source: DataSource) -> None: connection.close() -def insert_instruement(database_path, instrument: Instrument) -> None: +def insert_instrument(database_path: str, instrument: Instrument) -> None: insert_query = """ - INSERT INTO instruments (symbol,name,asset_class) - VALUES (?,?,?); + INSERT INTO instruments ( + symbol, + name, + asset_class, + currency, + exchange, + base_currency, + quote_currency) + VALUES (?,?,?,?,?,?,?); """ connection = duckdb.connect(database_path) @@ -88,7 +96,8 @@ def insert_instruement(database_path, instrument: Instrument) -> None: ) connection.close() -def find_data_source_id(database_path,price_bar: PriceBar): + +def get_data_source_id(database_path: str, source: DataSource) -> int | None: search_query = """ SELECT id FROM data_sources WHERE name=? @@ -96,34 +105,73 @@ def find_data_source_id(database_path,price_bar: PriceBar): connection = duckdb.connect(database_path) result = connection.execute( query=search_query, - parameters=[price_bar.source.name], - ) + parameters=[source.name], + ).fetchone() connection.close() - return result + if result is None: + return None + return result[0] + -def find_instrument_id(database_path,price_bar: PriceBar): +def get_instrument_id(database_path: str, instrument: Instrument) -> int | None: search_query = """ SELECT id FROM instruments WHERE name=? """ connection = duckdb.connect(database_path) - connection.execute( + result = connection.execute( query=search_query, - parameters=[price_bar.instrument.name], - ) + parameters=[instrument.name], + ).fetchone() connection.close() + if result is None: + return None + return result[0] -def insert_price_bar(database_path, price_bar: PriceBar) -> None: + +def insert_price_bar(database_path: str, price_bar: PriceBar) -> None: insert_query = """ - INSERT INTO price_bars (source_id,instrument_id,timestamp,timeframe,close) - VALUES (?,?,?,?,?); - """ - data_source_id = find_data_source_id(database_path, price_bar) - instrument_id = find_instrument_id(database_path, price_bar) - + INSERT INTO price_bars ( + source_id, + instrument_id, + timestamp, + timeframe, + close, + open, + high, + low, + adjusted_close, + volume + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + """ + source_id = get_data_source_id(database_path, price_bar.source) + instrument_id = get_instrument_id(database_path, price_bar.instrument) + connection = duckdb.connect(database_path) + + if source_id is None: + connection.close() + raise ValueError("Data source does not exist in storage.") + + if instrument_id is None: + connection.close() + raise ValueError("Instrument does not exist in storage.") + connection.execute( query=insert_query, - parameters=[data_source_id,instrument_id,price_bar.timestamp, price_bar.timeframe, price_bar.close], + parameters=[ + source_id, + instrument_id, + price_bar.timestamp, + price_bar.timeframe, + price_bar.close, + price_bar.open, + price_bar.high, + price_bar.low, + price_bar.adjusted_close, + price_bar.volume, + ], ) + connection.close() From be49bd975f62dc207a572a7479b13e0075dc2fa0 Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Mon, 29 Jun 2026 15:28:07 +0200 Subject: [PATCH 09/19] refactor(#38): upsert instead insert --- src/argus/storage/database.py | 80 +++++++++++++++++------------------ 1 file changed, 38 insertions(+), 42 deletions(-) diff --git a/src/argus/storage/database.py b/src/argus/storage/database.py index 61d7b13..fe6eae0 100644 --- a/src/argus/storage/database.py +++ b/src/argus/storage/database.py @@ -63,20 +63,31 @@ def initialize_database(database_path: str) -> None: connection.close() -def insert_data_source(database_path: str, source: DataSource) -> None: +def upsert_source(db: str, source: DataSource) -> int | None: insert_query = """ INSERT INTO data_sources (name, provider_kind, requires_api_key) VALUES (?,?,?); """ - connection = duckdb.connect(database_path) + search_query = """ + SELECT id FROM data_sources + WHERE name=? + """ + result = duckdb.execute( + query=search_query, + parameters=[source.name], + ).fetchone() + if result is not None: + return result[0] + connection = duckdb.connect(db) connection.execute( query=insert_query, parameters=[source.name, source.provider_kind, source.requires_api_key], ) connection.close() + return None -def insert_instrument(database_path: str, instrument: Instrument) -> None: +def upsert_instrument(db: str, instrument: Instrument) -> None: insert_query = """ INSERT INTO instruments ( symbol, @@ -88,48 +99,35 @@ def insert_instrument(database_path: str, instrument: Instrument) -> None: quote_currency) VALUES (?,?,?,?,?,?,?); """ - - connection = duckdb.connect(database_path) - connection.execute( - query=insert_query, - parameters=[instrument.symbol, instrument.name, instrument.asset_class], - ) - connection.close() - - -def get_data_source_id(database_path: str, source: DataSource) -> int | None: - search_query = """ - SELECT id FROM data_sources - WHERE name=? - """ - connection = duckdb.connect(database_path) - result = connection.execute( - query=search_query, - parameters=[source.name], - ).fetchone() - connection.close() - if result is None: - return None - return result[0] - - -def get_instrument_id(database_path: str, instrument: Instrument) -> int | None: search_query = """ SELECT id FROM instruments - WHERE name=? + WHERE symbol=? """ - connection = duckdb.connect(database_path) - result = connection.execute( + + result = duckdb.execute( query=search_query, - parameters=[instrument.name], + parameters=[instrument.symbol], ).fetchone() + if result is not None: + return result[0] + connection = duckdb.connect(db) + connection.execute( + query=insert_query, + parameters=[ + instrument.symbol, + instrument.name, + instrument.asset_class, + instrument.currency, + instrument.exchange, + instrument.base_currency, + instrument.quote_currency, + ], + ) connection.close() - if result is None: - return None - return result[0] + return None -def insert_price_bar(database_path: str, price_bar: PriceBar) -> None: +def insert_price_bar(db: str, price_bar: PriceBar) -> None: insert_query = """ INSERT INTO price_bars ( source_id, @@ -145,11 +143,9 @@ def insert_price_bar(database_path: str, price_bar: PriceBar) -> None: ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?); """ - source_id = get_data_source_id(database_path, price_bar.source) - instrument_id = get_instrument_id(database_path, price_bar.instrument) - - connection = duckdb.connect(database_path) - + connection = duckdb.connect(db) + source_id = upsert_source(db, price_bar.source) + instrument_id = upsert_instrument(db, price_bar.instrument) if source_id is None: connection.close() raise ValueError("Data source does not exist in storage.") From 65ee9863725721438310c4e289ab76704a5de184 Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Mon, 29 Jun 2026 15:51:27 +0200 Subject: [PATCH 10/19] refactor(#38): get_create instead --- src/argus/storage/database.py | 56 ++++++++++++++++++++++------------- 1 file changed, 36 insertions(+), 20 deletions(-) diff --git a/src/argus/storage/database.py b/src/argus/storage/database.py index fe6eae0..e01463c 100644 --- a/src/argus/storage/database.py +++ b/src/argus/storage/database.py @@ -63,7 +63,7 @@ def initialize_database(database_path: str) -> None: connection.close() -def upsert_source(db: str, source: DataSource) -> int | None: +def get_or_create_source(connection, source: DataSource) -> int: insert_query = """ INSERT INTO data_sources (name, provider_kind, requires_api_key) VALUES (?,?,?); @@ -72,22 +72,31 @@ def upsert_source(db: str, source: DataSource) -> int | None: SELECT id FROM data_sources WHERE name=? """ - result = duckdb.execute( + + result = connection.execute( query=search_query, parameters=[source.name], ).fetchone() if result is not None: return result[0] - connection = duckdb.connect(db) + connection.execute( query=insert_query, parameters=[source.name, source.provider_kind, source.requires_api_key], ) - connection.close() - return None + result = connection.execute( + query=search_query, + parameters=[source.name], + ).fetchone() + + if result is None: + raise ValueError("Data source could not be inserted.") + + return result[0] -def upsert_instrument(db: str, instrument: Instrument) -> None: + +def get_or_create_instrument(connection, instrument: Instrument) -> int: insert_query = """ INSERT INTO instruments ( symbol, @@ -104,13 +113,13 @@ def upsert_instrument(db: str, instrument: Instrument) -> None: WHERE symbol=? """ - result = duckdb.execute( + result = connection.execute( query=search_query, parameters=[instrument.symbol], ).fetchone() if result is not None: return result[0] - connection = duckdb.connect(db) + connection.execute( query=insert_query, parameters=[ @@ -123,8 +132,23 @@ def upsert_instrument(db: str, instrument: Instrument) -> None: instrument.quote_currency, ], ) - connection.close() - return None + result = connection.execute( + query=search_query, + parameters=[ + instrument.symbol, + instrument.name, + instrument.asset_class, + instrument.currency, + instrument.exchange, + instrument.base_currency, + instrument.quote_currency, + ], + ).fetchone() + + if result is None: + raise ValueError("Instrument could not be inserted.") + + return result[0] def insert_price_bar(db: str, price_bar: PriceBar) -> None: @@ -144,16 +168,8 @@ def insert_price_bar(db: str, price_bar: PriceBar) -> None: VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?); """ connection = duckdb.connect(db) - source_id = upsert_source(db, price_bar.source) - instrument_id = upsert_instrument(db, price_bar.instrument) - if source_id is None: - connection.close() - raise ValueError("Data source does not exist in storage.") - - if instrument_id is None: - connection.close() - raise ValueError("Instrument does not exist in storage.") - + source_id = get_or_create_source(connection, price_bar.source) + instrument_id = get_or_create_instrument(connection, price_bar.instrument) connection.execute( query=insert_query, parameters=[ From 7cda9561ad9a1a21994e912ded411be6691d2bb3 Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Mon, 29 Jun 2026 16:07:28 +0200 Subject: [PATCH 11/19] style(#38): fix func names --- argus_probe.duckdb | Bin 0 -> 3944448 bytes src/argus/storage/database.py | 63 +++++++++++++++------------------- 2 files changed, 28 insertions(+), 35 deletions(-) create mode 100644 argus_probe.duckdb diff --git a/argus_probe.duckdb b/argus_probe.duckdb new file mode 100644 index 0000000000000000000000000000000000000000..9c9ef34090bb8247f7d0259b65bba5a7b16ef54e GIT binary patch literal 3944448 zcmeF)U5p&bK>*+y@7lZmTl;e7pOBC{2%zwi^M%U~LJ>qcCwI!`lI&?otO!lUJH6}4 zt!M3-UB?y=UgarL0_b=;fe=Ipi8mw^5eXED(@DX0e0hnYAR)n1aKuaU1Uw)Q)!oxG zv$He)+rQb}uVv5FRR2`fS6$UzJ3Zas{Kg;r;vYZt^qViue)2QF7anc8_QD50{J|H8 zFMIA|$#13#cmV}sLdtiiA&G+KGs|2 z+o2l9qg<_;#L)3pYd&53a1y>%S_ogf6(;J*_2p}o#h74>pGd-I7DD+}T(lbA-D)Kv z%nvk7vvN19k~Fjs!lkrIVp8?WkK?5nSQVfOsmoP20_ zPGUi}BqwP{lI|!EY*a_{R-ZNcv`tM8Yf~2nx2Z~Bn;OuGq%CPsn>y6IarxrX;<{Ef zxUC(nG@HqCd2TU%j~QLl(kACt8jYkrm$k69bCT1OVNB#TaL|iOIlDh zJkxH=y%~8xQ$Dms8F{3p+|$3OE&1rMmi(o`E%_^JTk`s2_po-{J=&!0G9HS14v<~9 zxG%4peR#QcIcY9eE?-G!o#F7UbksdKx46{I+Vh#-HO7~&B=vMVYhy3f7A~!e z9a~)b+PcW8O7*KN&E=#TJDEWV9J;o&xRRd*^dyn?1czfE`+$RZJesd>_Txp55O?08 z2k&?`mz@~i4Qoz*h8({8TYq=$-d?`FawkXG$zqDpKDMsLQ?SXjIfkqr$6_NWnq&6_ zqI@ntoTkUS%{gj5^Qq6xoPXuhuf}uT&N<^pwj+^`Zf+uJE7@fu6WJv|V4DQuSmkK! z@2eqXgV5N?rpMjb=cXOUM5~n^y5py`GrpcirRPd2_#8R3ZJfef(v7yj4Q`=Wl*u(I@F60RjXF5FkK+0D*lfFnCar{Y*O-`Nhr7 z@3z|!?eT`6ZtuHY=Fq+?u>unyK!5-N0t5&=rhxIqV}eA;q)LDQ0RjXF5FkK+z>^7#^!Vb*iexPU1hzyV?=4!bE!nt@5+Fc; z009DfL?Az7i1&c%xm8s9Ym69kE3BSOXYY}cDh2@p1PBn=YXbe_{sZj-mfG*>=i(uM z=}iEN`1b4&{`c8`cBt=Y2oNAZfB*pk1PBlyK!5;&krF7z{naor7Y@Wf;a9_f`7nOD zQmfbUdn(O^Ls|5pBpj_Y7FL?&D~+X#Nq3=J;pu8+xl(Q}tu*G6X1P`^H&w?FixDNpPbA?p3!!`~E?N!mPQNS( z^8*=EuDe;Sq@jfnE~T{+ld4xPCuvG??dX}lzVrs^0_3m*I13uaEuM|Z2WpH?lS4Fl-6IVbSad6cBB>~ z)4k(d`tpJF<;_T14c|DJy~Z!&;-Vp(Xf7``Dho;ZTGD9Nmg=RLqiT4zy&J|g(y1Cw z&A$4|7iQ0|&B=#`=OjLWa!YcOb{Odn_Q1w?H19C7{vvIQlf&BLg~4sH($^LT^gpS} zptg9ZdE@fMrNwovac~dMmc(D{$vn7 z7}f`Lk8o+5j)(uAgL2m`wu5!E4=>j)C(Y%`( zjq#-`Nj=@p+Sp6Ag-h#V#}=2qwk~q2QvK>mb2+KT4r)*WhpsIxuHfQF=<;+eZ+b0ZbPPm4gGW7Qx@f5VZcJC3Qy>cf<@u<1B zJswNKH)D$`4$Nb*6%_5Ud)D%x=6G!Dn&YQFH*@}#Prn+^!#n4*9~pjB&i2JEXpT{2 zKt?_~JR?(EB$8I$NR6pvhQKZnIG&wT$00>E+)SJ2flDzIvIjbbg^(T$OYuYJxsUCk zLEcrSbB3&#HpzV258MCGzw8QS;k@YSvGp;Hj%HnCUmZ-o5UV7eZih}V8%CGz=6_Dv z3?#`7LZRowZhn20fnLJDB>mcpUV(nd7 zYU>(phiK;{)4w?t-A?>N$gplFX1%E6Jn+WZmoH3D=XL6p0D<(cdnN(|2oNAZfWRPu zsX>u@DJ~z(7+372I#rgP6ELpWIeRG+AV7cs0RlTCU_7xi?o!=UffJj$wN2QVam6Me z+5`at1PBlyu&V?fFs>MQS)ps7c2)JtECB)pc3R-%PG>;+yHCL1W_Dkl$uWUKVDQ+Y z^BbaJKPNK8C;+_^AV7cs0RjYepTJYQFUup9B;OSkKU)d! z0wX41lrdrzB0B^K5FkK+009C7HbWpk(QmajV_T2C4Wo?5CbI$o0t5&UAV7cs0RjUB z^3g@{3+`f(?2b{l!s^L%c5!FwY6x+;`EaxvR@>9DBuutiYd&A<`+Td^7xA|T`XatH z))(N36dXl7)aCTs7^ViFb%5u_-sTIMEr*?BFTx^nUxERW($0*HssoJTG z((uYCI=F}&SBTngoPGJi^z<&=8D*Kk)(8~G`K{Tytr8$WfB*pk1V&FFAA9tVCpsgp zVS_J2i_xpf&B@W3&Do5t5FkK+009E~O2D7g_EmL1w6H^l7{%8t!@hAb#CWKz1x8%J z<61_%f@H4{I9mj~6Cgl<0D;{iux^Oay+m!`l~(!XP(|q;c;V-G_d?H8eHVVl&ucFH zjGuqB=EBby@m9R>GrIz_d%4@qp;v`%&gFFRy3qlb)19B4d4Be_mqHwF6~W>Hw|LcP zIz7e(XvLL#X4BN#!6LQn^4cd&?PCMWHh(>@%4{f^2aD7q|C5*P zKFjT-_ zaEH?Dxf$=?b0tzB0t5&UAh0okd@$2LuIP-P^FdNJsxF2t*-+XLV`J>rCa_TfLzs;U zTAu)cJuC1_d$wc>Mqt+n3>#uR+7I7i_=0oS?8P!lfB*pk1PJVVf%olufl&e& zGQ=qEdOPd}xcgyAV7e??hx=N%H2^G~-DldI zNy5FU^kE&V?I}k5%Za{-zmLysS+m4nJ=GWS$M5Qk_`m*$|9ZM_iGO`2eI`hJVoEg} zEpD+J=0o?%J!!$>{d8}K>_HxB_Tp1S(kYfT{)jz`edFxQ7pA9+LhtMM+@{qUZz~kf zo-A%lbFdwMsBeDJf2PY9>rTXQ5#L@sv?ju8nnv;2I{^X&2oNAZfB*pk1PBlyuuQ>6Yf2_M<~`-3%q};6R>>(1PBlyK!Csw3K(DPAikst5FkKc zBm{nKByu4W1PBn=8v@1`d!rtef&c*m1PBlyu$KgWeJ>SepOwYXrIjk`}Qye*Sv7QCUu!X}pO50RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBn=y8_eq%2RL8oW6O#Wg ze!KMki5T~E{GYy`pPhMr_O+L?rB24z_s0L}`}vi|(sR#0H~ZRiFD)%DRvPI72jc&! z_WS(xG$w>&`CI3;_w!hcJrVz>?=hY1z4%?G7%s^fWXEDO5yzM%=5Fay;Su7lkLDn5#gNx0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfWU4Ln7&t@dVA*d&HF7A0RjXF zY+RreRzoQD^#9}S@I+6XAOQjd2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5;&kr(*h)$e@w>bbF>-*1@+5FkKc;{v5{W4>0e)aR1+0pM`^ePZMf2r@{3 z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK%hrp`d)eJ?U~az z@3%|@2oNB!ae-1;4WZQ2&5pOj6FqT)1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjX@Uf`+4GtDpG`S6eLw@d^G5FoH|fl@d>JM;YPYcCc3|71Hb zQABtrK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FoG{1P-1G z;h%ndc{R@pzQ0i24LO%b0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZV5~ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV6T`1*Y$nr{119 zee-_HM1TMR0vi`7h1C#BJ)Qq}J3P@7CrE$*0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z!2h>Xhc*DhP!K?W6OuY4Q=wRZQQF-)Ot*j None: CREATE SEQUENCE IF NOT EXISTS data_sources_id_seq; """ create_instruments_sequence = """ - CREATE SEQUENCE IF NOT EXISTS instruements_id_seq; + CREATE SEQUENCE IF NOT EXISTS instruments_id_seq; """ create_price_bars_sequence = """ CREATE SEQUENCE IF NOT EXISTS price_bars_id_seq; """ - create_datasources_table = """ + create_data_sources_table = """ CREATE TABLE IF NOT EXISTS data_sources ( id INTEGER PRIMARY KEY DEFAULT nextval('data_sources_id_seq'), name TEXT NOT NULL UNIQUE, @@ -21,9 +20,9 @@ def initialize_database(database_path: str) -> None: requires_api_key BOOLEAN NOT NULL ); """ - create_intstruments_table = """ + create_instruments_table = """ CREATE TABLE IF NOT EXISTS instruments ( - id INTEGER PRIMARY KEY DEFAULT nextval('instruements_id_seq'), + id INTEGER PRIMARY KEY DEFAULT nextval('instruments_id_seq'), symbol TEXT NOT NULL UNIQUE, name TEXT NOT NULL, asset_class TEXT NOT NULL, @@ -56,8 +55,8 @@ def initialize_database(database_path: str) -> None: connection.execute(query=create_instruments_sequence) connection.execute(query=create_price_bars_sequence) - connection.execute(query=create_datasources_table) - connection.execute(query=create_intstruments_table) + connection.execute(query=create_data_sources_table) + connection.execute(query=create_instruments_table) connection.execute(query=create_price_bars_table) connection.close() @@ -134,15 +133,7 @@ def get_or_create_instrument(connection, instrument: Instrument) -> int: ) result = connection.execute( query=search_query, - parameters=[ - instrument.symbol, - instrument.name, - instrument.asset_class, - instrument.currency, - instrument.exchange, - instrument.base_currency, - instrument.quote_currency, - ], + parameters=[instrument.symbol], ).fetchone() if result is None: @@ -168,22 +159,24 @@ def insert_price_bar(db: str, price_bar: PriceBar) -> None: VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?); """ connection = duckdb.connect(db) - source_id = get_or_create_source(connection, price_bar.source) - instrument_id = get_or_create_instrument(connection, price_bar.instrument) - connection.execute( - query=insert_query, - parameters=[ - source_id, - instrument_id, - price_bar.timestamp, - price_bar.timeframe, - price_bar.close, - price_bar.open, - price_bar.high, - price_bar.low, - price_bar.adjusted_close, - price_bar.volume, - ], - ) - - connection.close() + try: + source_id = get_or_create_source(connection, price_bar.source) + instrument_id = get_or_create_instrument(connection, price_bar.instrument) + connection.execute( + query=insert_query, + parameters=[ + source_id, + instrument_id, + price_bar.timestamp, + price_bar.timeframe, + price_bar.close, + price_bar.open, + price_bar.high, + price_bar.low, + price_bar.adjusted_close, + price_bar.volume, + ], + ) + connection.close() + finally: + connection.close() From 5559513b3563af9a2f3f85cb9c1f01072baa23c9 Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Tue, 30 Jun 2026 08:59:55 +0200 Subject: [PATCH 12/19] feat(#38): add read func --- src/argus/storage/database.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/argus/storage/database.py b/src/argus/storage/database.py index be825ad..504bc9f 100644 --- a/src/argus/storage/database.py +++ b/src/argus/storage/database.py @@ -180,3 +180,26 @@ def insert_price_bar(db: str, price_bar: PriceBar) -> None: connection.close() finally: connection.close() + +def get_price_bar(db: str,source:DataSource,instrument:Instrument,start_time:str,end_time:str): + search_query = """ + SELECT + data_sources.name AS source_name, + instruments.symbol AS instrument_symbol, + price_bars.timestamp, + price_bars.timeframe, + price_bars.open, + price_bars.high, + price_bars.low, + price_bars.close, + price_bars.adjusted_close, + price_bars.volume + FROM price_bars + WHERE data_sources.name = ? AND instruments.symbol = ? AND price_bars.timestamp BETWEEN ? AND ? + JOIN data_sources ON price_bars.source_id = data_sources.id + JOIN instruments ON price_bars.instrument_id = instruments.id + """ + connection = duckdb.connect(db) + result = connection.execute(search_query,parameters=[source.name,instrument.symbol,start_time,end_time]) + connection.close() + return result From d51da47f26f1661cc583d3462767cfe8045be8cc Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Tue, 30 Jun 2026 14:37:09 +0200 Subject: [PATCH 13/19] feat(#38): try for read func --- src/argus/storage/database.py | 62 +++++++++++++++++++++++++---------- 1 file changed, 44 insertions(+), 18 deletions(-) diff --git a/src/argus/storage/database.py b/src/argus/storage/database.py index 504bc9f..c55c50b 100644 --- a/src/argus/storage/database.py +++ b/src/argus/storage/database.py @@ -1,4 +1,6 @@ import duckdb +from datetime import date +import pandas as pd from argus.domain.internal_models import DataSource, PriceBar, Instrument @@ -181,25 +183,49 @@ def insert_price_bar(db: str, price_bar: PriceBar) -> None: finally: connection.close() -def get_price_bar(db: str,source:DataSource,instrument:Instrument,start_time:str,end_time:str): + +def read_price_bars( + db: str, + source: DataSource, + instrument: Instrument, + start_date: date, + end_date: date, +) -> pd.DataFrame: search_query = """ - SELECT - data_sources.name AS source_name, - instruments.symbol AS instrument_symbol, - price_bars.timestamp, - price_bars.timeframe, - price_bars.open, - price_bars.high, - price_bars.low, - price_bars.close, - price_bars.adjusted_close, - price_bars.volume - FROM price_bars - WHERE data_sources.name = ? AND instruments.symbol = ? AND price_bars.timestamp BETWEEN ? AND ? - JOIN data_sources ON price_bars.source_id = data_sources.id - JOIN instruments ON price_bars.instrument_id = instruments.id - """ + SELECT + data_sources.name AS source_name, + instruments.symbol AS instrument_symbol, + price_bars.timestamp, + price_bars.timeframe, + price_bars.open, + price_bars.high, + price_bars.low, + price_bars.close, + price_bars.adjusted_close, + price_bars.volume + FROM price_bars + JOIN data_sources ON price_bars.source_id = data_sources.id + JOIN instruments ON price_bars.instrument_id = instruments.id + WHERE data_sources.name = ? + AND instruments.symbol = ? + AND price_bars.timestamp BETWEEN ? AND ? + ORDER BY price_bars.timestamp; + """ + connection = duckdb.connect(db) - result = connection.execute(search_query,parameters=[source.name,instrument.symbol,start_time,end_time]) + try: + result = connection.execute( + query=search_query, + parameters=[ + source.name, + instrument.symbol, + start_date, + end_date, + ], + ).df() + finally: + connection.close() + connection.close() + return result From 4146e507347f5365ac8ef733efd53372a2d74db6 Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Wed, 1 Jul 2026 09:07:34 +0200 Subject: [PATCH 14/19] test(#38): test for init --- src/argus/storage/database.py | 106 +++++++++++++++------------------ tests/test_storage_database.py | 25 ++++++++ 2 files changed, 73 insertions(+), 58 deletions(-) create mode 100644 tests/test_storage_database.py diff --git a/src/argus/storage/database.py b/src/argus/storage/database.py index c55c50b..fbaae40 100644 --- a/src/argus/storage/database.py +++ b/src/argus/storage/database.py @@ -5,63 +5,56 @@ def initialize_database(database_path: str) -> None: - create_data_sources_sequence = """ - CREATE SEQUENCE IF NOT EXISTS data_sources_id_seq; - """ - create_instruments_sequence = """ - CREATE SEQUENCE IF NOT EXISTS instruments_id_seq; - """ - create_price_bars_sequence = """ - CREATE SEQUENCE IF NOT EXISTS price_bars_id_seq; - """ - create_data_sources_table = """ - CREATE TABLE IF NOT EXISTS data_sources ( - id INTEGER PRIMARY KEY DEFAULT nextval('data_sources_id_seq'), - name TEXT NOT NULL UNIQUE, - provider_kind TEXT NOT NULL, - requires_api_key BOOLEAN NOT NULL - ); - """ - create_instruments_table = """ + queries = [ + "CREATE SEQUENCE IF NOT EXISTS data_sources_id_seq;", + "CREATE SEQUENCE IF NOT EXISTS instruments_id_seq;", + "CREATE SEQUENCE IF NOT EXISTS price_bars_id_seq;", + """ + CREATE TABLE IF NOT EXISTS data_sources ( + id INTEGER PRIMARY KEY DEFAULT nextval('data_sources_id_seq'), + name TEXT NOT NULL UNIQUE, + provider_kind TEXT NOT NULL, + requires_api_key BOOLEAN NOT NULL + ); + """, + """ CREATE TABLE IF NOT EXISTS instruments ( - id INTEGER PRIMARY KEY DEFAULT nextval('instruments_id_seq'), - symbol TEXT NOT NULL UNIQUE, - name TEXT NOT NULL, - asset_class TEXT NOT NULL, - currency TEXT, - exchange TEXT, - base_currency TEXT, - quote_currency TEXT - ); - """ - create_price_bars_table = """ + id INTEGER PRIMARY KEY DEFAULT nextval('instruments_id_seq'), + symbol TEXT NOT NULL UNIQUE, + name TEXT NOT NULL, + asset_class TEXT NOT NULL, + currency TEXT, + exchange TEXT, + base_currency TEXT, + quote_currency TEXT + ); + """, + """ CREATE TABLE IF NOT EXISTS price_bars ( - id INTEGER PRIMARY KEY DEFAULT nextval('price_bars_id_seq'), - source_id INTEGER NOT NULL, - instrument_id INTEGER NOT NULL, - timestamp DATE NOT NULL, - timeframe TEXT NOT NULL, - close DOUBLE NOT NULL, - open DOUBLE, - high DOUBLE, - low DOUBLE, - adjusted_close DOUBLE, - volume DOUBLE, - FOREIGN KEY (source_id) REFERENCES data_sources (id), - FOREIGN KEY (instrument_id) REFERENCES instruments (id) - ); - """ - connection = duckdb.connect(database_path) + id INTEGER PRIMARY KEY DEFAULT nextval('price_bars_id_seq'), + source_id INTEGER NOT NULL, + instrument_id INTEGER NOT NULL, + timestamp DATE NOT NULL, + timeframe TEXT NOT NULL, + close DOUBLE NOT NULL, + open DOUBLE, + high DOUBLE, + low DOUBLE, + adjusted_close DOUBLE, + volume DOUBLE, + FOREIGN KEY (source_id) REFERENCES data_sources (id), + FOREIGN KEY (instrument_id) REFERENCES instruments (id), + UNIQUE (source_id, instrument_id, timestamp, timeframe) + ); + """, + ] - connection.execute(query=create_data_sources_sequence) - connection.execute(query=create_instruments_sequence) - connection.execute(query=create_price_bars_sequence) - - connection.execute(query=create_data_sources_table) - connection.execute(query=create_instruments_table) - connection.execute(query=create_price_bars_table) - - connection.close() + connection = duckdb.connect(database_path) + try: + for query in queries: + connection.execute(query) + finally: + connection.close() def get_or_create_source(connection, source: DataSource) -> int: @@ -179,7 +172,6 @@ def insert_price_bar(db: str, price_bar: PriceBar) -> None: price_bar.volume, ], ) - connection.close() finally: connection.close() @@ -191,6 +183,7 @@ def read_price_bars( start_date: date, end_date: date, ) -> pd.DataFrame: + search_query = """ SELECT data_sources.name AS source_name, @@ -225,7 +218,4 @@ def read_price_bars( ).df() finally: connection.close() - - connection.close() - return result diff --git a/tests/test_storage_database.py b/tests/test_storage_database.py new file mode 100644 index 0000000..1b7dad5 --- /dev/null +++ b/tests/test_storage_database.py @@ -0,0 +1,25 @@ +from datetime import date + +import duckdb + +from argus.domain.internal_models import DataSource, Instrument, PriceBar +from argus.storage.database import ( + initialize_database, + insert_price_bar, + read_price_bars, +) + + +def test_initialize_database_creates_required_tables(tmp_path): + + db = tmp_path / "test.duckdb" + + initialize_database(db) + connection = duckdb.connect(db) + + tables = connection.execute("SHOW TABLES;").fetchall() + table_names = {row[0] for row in tables} + + assert "data_sources" in table_names + assert "instruments" in table_names + assert "price_bars" in table_names From cfb5af9a4ea37477856897a59c3ce8225afbc1d3 Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Wed, 1 Jul 2026 09:42:26 +0200 Subject: [PATCH 15/19] test(#38): add more storage tests --- src/argus/storage/database.py | 9 +- tests/test_exchangerate_client.py | 1 - tests/test_storage_database.py | 137 ++++++++++++++++++++++++++++-- 3 files changed, 135 insertions(+), 12 deletions(-) diff --git a/src/argus/storage/database.py b/src/argus/storage/database.py index fbaae40..f184b65 100644 --- a/src/argus/storage/database.py +++ b/src/argus/storage/database.py @@ -60,7 +60,8 @@ def initialize_database(database_path: str) -> None: def get_or_create_source(connection, source: DataSource) -> int: insert_query = """ INSERT INTO data_sources (name, provider_kind, requires_api_key) - VALUES (?,?,?); + VALUES (?,?,?) + ON CONFLICT DO NOTHING; """ search_query = """ SELECT id FROM data_sources @@ -100,7 +101,8 @@ def get_or_create_instrument(connection, instrument: Instrument) -> int: exchange, base_currency, quote_currency) - VALUES (?,?,?,?,?,?,?); + VALUES (?,?,?,?,?,?,?) + ON CONFLICT DO NOTHING; """ search_query = """ SELECT id FROM instruments @@ -151,7 +153,8 @@ def insert_price_bar(db: str, price_bar: PriceBar) -> None: adjusted_close, volume ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT DO NOTHING; """ connection = duckdb.connect(db) try: diff --git a/tests/test_exchangerate_client.py b/tests/test_exchangerate_client.py index 1f15512..132a8a8 100644 --- a/tests/test_exchangerate_client.py +++ b/tests/test_exchangerate_client.py @@ -65,7 +65,6 @@ def test_check_currency_key_error(monkeypatch, capsys): test_resp.json.return_value = { "result": "success", # not passing "success" bypases the "conversion_rate" checking "error_type": "", - # "conversion_rate" fehlt absichtlich } def test_get_resp(url, timeout): diff --git a/tests/test_storage_database.py b/tests/test_storage_database.py index 1b7dad5..d3c5e00 100644 --- a/tests/test_storage_database.py +++ b/tests/test_storage_database.py @@ -3,23 +3,144 @@ import duckdb from argus.domain.internal_models import DataSource, Instrument, PriceBar -from argus.storage.database import ( - initialize_database, - insert_price_bar, - read_price_bars, -) - +from argus.storage.database import initialize_database, insert_price_bar, read_price_bars def test_initialize_database_creates_required_tables(tmp_path): - db = tmp_path / "test.duckdb" initialize_database(db) connection = duckdb.connect(db) - tables = connection.execute("SHOW TABLES;").fetchall() + connection.close() table_names = {row[0] for row in tables} assert "data_sources" in table_names assert "instruments" in table_names assert "price_bars" in table_names + +def test_data_is_inserted(tmp_path): + source = DataSource( + name="Yahoo", + provider_kind="yfinance_api", + requires_api_key=False + ) + + instrument = Instrument( + symbol="EUR/USD", + name="EUR - USD Rate", + asset_class="fx", + base_currency="EUR", + quote_currency="USD") + + pricebar = PriceBar( + source=source, + instrument=instrument, + timestamp=date(2026,1,1), + timeframe="1d", + close=1.89 + ) + + db = tmp_path / "test.duckdb" + initialize_database(db) + insert_price_bar(db,pricebar) + connection = duckdb.connect(db) + + instrument_count = connection.execute( + "SELECT COUNT(*) FROM instruments;" + ).fetchone() + + source_count = connection.execute( + "SELECT COUNT(*) FROM data_sources;" + ).fetchone() + + price_bar_count = connection.execute( + "SELECT COUNT(*) FROM price_bars;" + ).fetchone() + + assert instrument_count is not None + assert source_count is not None + assert price_bar_count is not None + assert instrument_count[0] == 1 + assert source_count[0] == 1 + assert price_bar_count[0] == 1 + +def test_fx_has_correct_format(tmp_path): + source = DataSource( + name="Yahoo", + provider_kind="yfinance_api", + requires_api_key=False + ) + + instrument = Instrument( + symbol="EUR/USD", + name="EUR - USD Rate", + asset_class="fx", + base_currency="EUR", + quote_currency="USD") + + pricebar = PriceBar( + source=source, + instrument=instrument, + timestamp=date(2026,1,1), + timeframe="1d", + close=1.89 + ) + + db = tmp_path / "test.duckdb" + initialize_database(db) + insert_price_bar(db,pricebar) + connection = duckdb.connect(db) + + price_bar_fx = connection.execute( + "SELECT * FROM price_bars;" + ).fetchone() + connection.close() + + assert price_bar_fx is not None + assert price_bar_fx[0] == 1 + assert price_bar_fx[1] == 1 + assert price_bar_fx[2] == 1 + assert price_bar_fx[3] == date(2026,1,1) + assert price_bar_fx[4] == "1d" + assert price_bar_fx[5] == 1.89 + assert price_bar_fx[6] is None + assert price_bar_fx[7] is None + assert price_bar_fx[8] is None + assert price_bar_fx[9] is None + assert price_bar_fx[10] is None + +def test_duplicates_are_ignored(tmp_path): + source = DataSource( + name="Yahoo", + provider_kind="yfinance_api", + requires_api_key=False + ) + + instrument = Instrument( + symbol="EUR/USD", + name="EUR - USD Rate", + asset_class="fx", + base_currency="EUR", + quote_currency="USD") + + pricebar = PriceBar( + source=source, + instrument=instrument, + timestamp=date(2026,1,1), + timeframe="1d", + close=1.89 + ) + + db = tmp_path / "test.duckdb" + initialize_database(db) + insert_price_bar(db,pricebar) + insert_price_bar(db,pricebar) + connection = duckdb.connect(db) + count = connection.execute( + "SELECT COUNT(*) FROM price_bars;" + ).fetchone() + + assert count is not None + assert count[0] == 1 + + From d00484e894bccf8ac16e4e0bc684e6ff6726cab9 Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Wed, 1 Jul 2026 11:09:49 +0200 Subject: [PATCH 16/19] test(#38): add read tests --- tests/test_storage_database.py | 156 ++++++++++++++++++++++++--------- 1 file changed, 116 insertions(+), 40 deletions(-) diff --git a/tests/test_storage_database.py b/tests/test_storage_database.py index d3c5e00..d513008 100644 --- a/tests/test_storage_database.py +++ b/tests/test_storage_database.py @@ -3,7 +3,12 @@ import duckdb from argus.domain.internal_models import DataSource, Instrument, PriceBar -from argus.storage.database import initialize_database, insert_price_bar, read_price_bars +from argus.storage.database import ( + initialize_database, + insert_price_bar, + read_price_bars, +) + def test_initialize_database_creates_required_tables(tmp_path): db = tmp_path / "test.duckdb" @@ -18,11 +23,10 @@ def test_initialize_database_creates_required_tables(tmp_path): assert "instruments" in table_names assert "price_bars" in table_names + def test_data_is_inserted(tmp_path): source = DataSource( - name="Yahoo", - provider_kind="yfinance_api", - requires_api_key=False + name="Yahoo", provider_kind="yfinance_api", requires_api_key=False ) instrument = Instrument( @@ -30,32 +34,29 @@ def test_data_is_inserted(tmp_path): name="EUR - USD Rate", asset_class="fx", base_currency="EUR", - quote_currency="USD") - + quote_currency="USD", + ) + pricebar = PriceBar( source=source, instrument=instrument, - timestamp=date(2026,1,1), + timestamp=date(2026, 1, 1), timeframe="1d", - close=1.89 + close=1.89, ) db = tmp_path / "test.duckdb" initialize_database(db) - insert_price_bar(db,pricebar) + insert_price_bar(db, pricebar) connection = duckdb.connect(db) instrument_count = connection.execute( "SELECT COUNT(*) FROM instruments;" ).fetchone() - source_count = connection.execute( - "SELECT COUNT(*) FROM data_sources;" - ).fetchone() + source_count = connection.execute("SELECT COUNT(*) FROM data_sources;").fetchone() - price_bar_count = connection.execute( - "SELECT COUNT(*) FROM price_bars;" - ).fetchone() + price_bar_count = connection.execute("SELECT COUNT(*) FROM price_bars;").fetchone() assert instrument_count is not None assert source_count is not None @@ -64,11 +65,10 @@ def test_data_is_inserted(tmp_path): assert source_count[0] == 1 assert price_bar_count[0] == 1 + def test_fx_has_correct_format(tmp_path): source = DataSource( - name="Yahoo", - provider_kind="yfinance_api", - requires_api_key=False + name="Yahoo", provider_kind="yfinance_api", requires_api_key=False ) instrument = Instrument( @@ -76,31 +76,30 @@ def test_fx_has_correct_format(tmp_path): name="EUR - USD Rate", asset_class="fx", base_currency="EUR", - quote_currency="USD") - + quote_currency="USD", + ) + pricebar = PriceBar( source=source, instrument=instrument, - timestamp=date(2026,1,1), + timestamp=date(2026, 1, 1), timeframe="1d", - close=1.89 + close=1.89, ) db = tmp_path / "test.duckdb" initialize_database(db) - insert_price_bar(db,pricebar) + insert_price_bar(db, pricebar) connection = duckdb.connect(db) - price_bar_fx = connection.execute( - "SELECT * FROM price_bars;" - ).fetchone() + price_bar_fx = connection.execute("SELECT * FROM price_bars;").fetchone() connection.close() assert price_bar_fx is not None assert price_bar_fx[0] == 1 assert price_bar_fx[1] == 1 assert price_bar_fx[2] == 1 - assert price_bar_fx[3] == date(2026,1,1) + assert price_bar_fx[3] == date(2026, 1, 1) assert price_bar_fx[4] == "1d" assert price_bar_fx[5] == 1.89 assert price_bar_fx[6] is None @@ -109,11 +108,10 @@ def test_fx_has_correct_format(tmp_path): assert price_bar_fx[9] is None assert price_bar_fx[10] is None + def test_duplicates_are_ignored(tmp_path): source = DataSource( - name="Yahoo", - provider_kind="yfinance_api", - requires_api_key=False + name="Yahoo", provider_kind="yfinance_api", requires_api_key=False ) instrument = Instrument( @@ -121,26 +119,104 @@ def test_duplicates_are_ignored(tmp_path): name="EUR - USD Rate", asset_class="fx", base_currency="EUR", - quote_currency="USD") - + quote_currency="USD", + ) + pricebar = PriceBar( source=source, instrument=instrument, - timestamp=date(2026,1,1), + timestamp=date(2026, 1, 1), timeframe="1d", - close=1.89 + close=1.89, ) db = tmp_path / "test.duckdb" initialize_database(db) - insert_price_bar(db,pricebar) - insert_price_bar(db,pricebar) + insert_price_bar(db, pricebar) + insert_price_bar(db, pricebar) connection = duckdb.connect(db) - count = connection.execute( - "SELECT COUNT(*) FROM price_bars;" - ).fetchone() + count = connection.execute("SELECT COUNT(*) FROM price_bars;").fetchone() assert count is not None assert count[0] == 1 - + +def test_read_price_bars_returns_matching_data(tmp_path): + source = DataSource( + name="Yahoo", + provider_kind="yfinance_api", + requires_api_key=False, + ) + + instrument = Instrument( + symbol="EUR/USD", + name="EUR - USD Rate", + asset_class="fx", + base_currency="EUR", + quote_currency="USD", + ) + + pricebar = PriceBar( + source=source, + instrument=instrument, + timestamp=date(2026, 1, 1), + timeframe="1d", + close=1.89, + ) + + db = tmp_path / "test.duckdb" + initialize_database(db) + insert_price_bar(db, pricebar) + + result = read_price_bars( + db=db, + source=source, + instrument=instrument, + start_date=date(2026, 1, 1), + end_date=date(2026, 1, 31), + ) + + assert result.empty is False + assert len(result) == 1 + assert result.iloc[0]["source_name"] == "Yahoo" + assert result.iloc[0]["instrument_symbol"] == "EUR/USD" + assert result.iloc[0]["timeframe"] == "1d" + assert result.iloc[0]["close"] == 1.89 + + +def test_read_price_bars_returns_empty_dataframe_for_missing_range(tmp_path): + source = DataSource( + name="Yahoo", + provider_kind="yfinance_api", + requires_api_key=False, + ) + + instrument = Instrument( + symbol="EUR/USD", + name="EUR - USD Rate", + asset_class="fx", + base_currency="EUR", + quote_currency="USD", + ) + + pricebar = PriceBar( + source=source, + instrument=instrument, + timestamp=date(2026, 1, 1), + timeframe="1d", + close=1.89, + ) + + db = tmp_path / "test.duckdb" + initialize_database(db) + insert_price_bar(db, pricebar) + + result = read_price_bars( + db=db, + source=source, + instrument=instrument, + start_date=date(2027, 1, 1), + end_date=date(2027, 1, 31), + ) + + assert result.empty is True From d2f515927cabbe5b1651156e7e45fd2d8fb057aa Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Wed, 1 Jul 2026 11:59:55 +0200 Subject: [PATCH 17/19] docs(#38): doc for storage --- README.md | 27 ++++----- docs/roadmap.md | 64 ++++++++++---------- docs/storage.md | 154 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 197 insertions(+), 48 deletions(-) create mode 100644 docs/storage.md diff --git a/README.md b/README.md index 1c0d156..8723389 100644 --- a/README.md +++ b/README.md @@ -125,6 +125,13 @@ README.md - ExchangeRate API for live currency conversion - yfinance for historical market-data retrieval and analytics +### Storage + +- DuckDB — local analytical storage for normalized historical market data + +>[!Note] +> See docs/storage.md for details. + --- ## Planned / Future Tech Stack @@ -138,40 +145,32 @@ Planned or likely future technologies include: - Frankfurter API for historical FX data - possible additional market-data APIs later -### Data processing - -- possibly Polars later for larger datasets - ### Storage - PostgreSQL -- DuckDB -- Parquet -- optional cloud storage ### Visualization and UI - NiceGUI +- Django ### DevOps and deployment - Docker Compose -- cloud deployment later +- Travis CI ### Cloud and data engineering -- Azure, GCP or AWS depending on project direction +- Azure - scheduled ingestion -- data quality checks -- reporting pipelines +- agentic Workflows +- Blob Storage +- scaled analysis ### AI and agentic workflows - LLM-assisted summaries - RAG over stored reports or notes -- agentic data checks -- anomaly monitoring -- human-in-the-loop signal review > [!CAUTION] > AI and agentic features are future-stage ideas. diff --git a/docs/roadmap.md b/docs/roadmap.md index 85706af..bc2cdbd 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -41,17 +41,19 @@ Scope: - rolling volatility - performance analytics - risk analytics -- Extend the current dashboard without adding unnecessary chart noise +- Extend the current dashboard - Add or evaluate new data clients: - - Frankfurter for historical FX data - yfinance for broader market data -- Replace or reduce dependency on the current ExchangeRate API where needed - Improve pandas-based analysis workflows +- Add a local storage for historical market data +- Add report generation and export +- add first prediction feature +- Introduce NiceGUI as a new GUI - Add tests for metric calculations and data transformations -- Document metric definitions, assumptions and chart behavior +- Add CD Pipeline Outcome: -ARGUS becomes a basic market analytics tool, not only a converter. +ARGUS becomes a basic market analytics tool ### Sprint 3 — Storage, Web-Ready UI & Data Architecture @@ -61,53 +63,45 @@ Prepare ARGUS for persistent data workflows and a stronger product interface. Scope: -- Add local storage layer: - - PostgreSQL, DuckDB, SQLite or Parquet depending on use case -- Store historical market data -- Separate ingestion, transformation, analytics and presentation layers more clearly -- Start NiceGUI as the main web-ready UI direction +- Extend local storage layer +- First local ETL Pipeline +- Extend NiceGUI and plan how to combine with modern frotend techstack like django and node.js - Keep Tkinter as legacy/prototype unless still useful -- Keep CLI as internal/debug interface only -- Add clearer architecture documentation -- Prepare the project for larger data workflows and external contributors +- More metrics, more instruments and more (and better) prediction features +- Introduce first LLM summary for reports +- Introduce Snyk and Performance Test to cover perfomance and security of argus +- Improve Code Quality Outcome: -ARGUS has a clearer data architecture and starts moving from local prototype toward a scalable analytics application. +ARGUS is a scalable analytics application that allows to get more insight from market data -### Sprint 4 — Cloud, Pipelines & Portfolio-Grade Data Engineering +### Sprint 4 — Introduction for extended Analysis -**Status:** Future +**Status:** Planned -Turn ARGUS into a stronger end-to-end data engineering project. +Turn ARGUS into a stronger end-to-end data engineering project which is cloud ready. Scope: -- Docker / Docker Compose -- Scheduled data ingestion -- Cloud storage or cloud database -- CI/CD improvements +- Docker Compose +- Intorduce Azure (a simple connection - storage only) +- Better LLM Workflow (introduce RAG) - Data quality checks -- Basic pipeline orchestration -- Reporting layer -- Architecture diagram -- Deployment documentation +- Caching and efficient storing of market data +- More export possibilities for users +- More metrics and better meta data visualization -Target workflow: +Outcome: -```text -API → Ingestion → Storage → Transformation → Analysis → Visualization → CI/CD -``` +ARGUS is ready to interact with the cloud layer and future cloud app. It's able to give the user an transparent and clear analysis of requested market section. ### Sprint 5 — AI-Assisted Research & Agentic Monitoring **Status:** Future vision -Add AI support only after the data, storage, service and reporting layers are stable. - Scope: -- LLM-assisted report summaries -- Explanation of unusual movements +- First Cloud workflows to extend the analysis - RAG over stored market notes, reports or documentation - Agentic checks for data quality, anomalies and recurring market scans - Human-in-the-loop signal review @@ -115,4 +109,6 @@ Scope: Outcome: -ARGUS starts behaving like its name: a system that continuously watches market data, evaluates it and helps generate useful signals. +ARGUS and the cloud app interact with each other. ARGUS become the first time an useful monitoring and alysis tool. +It's the beginn of ARGUS to help the user to find, implement and deploy strategies. Through that ARGUS will +be first time able to give signals and allow paper trading, back tests and controlled trading with agents. diff --git a/docs/storage.md b/docs/storage.md new file mode 100644 index 0000000..27ce106 --- /dev/null +++ b/docs/storage.md @@ -0,0 +1,154 @@ +# ARGUS Storage Layer + +ARGUS uses DuckDB as the local storage layer for normalized market data. + +The storage layer stores ARGUS-internal market data structures and provides reusable historical data for analytics, charts, dashboards and reports. + +The storage design follows the direction described in [`docs/research-databases-and-storage.md`](research-databases-and-storage.md). + +## Storage Workflow + +ARGUS uses a storage-first workflow for historical market data. + +```text +User / GUI / Analytics request + ↓ +Market data service + ↓ +Check DuckDB storage + ↓ +If data exists: + read stored data + return it for analytics, charts or reports + +If data is missing: + fetch data from a client/API + normalize the response into ARGUS-internal data + return the normalized data + save the normalized data in DuckDB +``` + +DuckDB is used to avoid unnecessary repeated API calls and to make historical market data reusable across analytics, dashboard and reporting workflows. + +Fresh API data can be used immediately after normalization and is also persisted so future requests can use the local storage layer first. + +## Schema Overview + +The first storage schema is based on three related entities: + +```text +data_sources +instruments +price_bars +``` + +### `data_sources` + +Stores where market data came from. + +Examples: + +```text +yfinance +ExchangeRate API +Frankfurter +FRED +``` + +Each source describes a provider or API that can deliver market, FX or macro data. + +### `instruments` + +Stores what ARGUS can analyze. + +Examples: + +```text +EUR/USD +AAPL +SPY +BTC-USD +``` + +An instrument represents the internal ARGUS identity of an asset, currency pair, ETF, index or other market object. + +Provider-specific symbols should be normalized before storage. For example: + +```text +yfinance provider symbol: EURUSD=X +ARGUS instrument symbol: EUR/USD +``` + +### `price_bars` + +Stores historical time-series values in an OHLCV-ready structure. + +A price bar belongs to: + +```text +one data source +one instrument +one timestamp +one timeframe +``` + +FX rates are stored as `close` values. + +For simple FX data, the remaining OHLCV fields can stay empty. For broader market data, the same structure can store open, high, low, close, adjusted close and volume values. + +The combination of source, instrument, timestamp and timeframe identifies a unique stored price bar. + +## Internal Models and Storage + +ARGUS uses internal domain models before data is stored: + +```text +DataSource +Instrument +PriceBar +``` + +These models describe the meaning of the data inside ARGUS. + +The storage layer translates these internal models into DuckDB tables: + +```text +DataSource -> data_sources +Instrument -> instruments +PriceBar -> price_bars +``` + +In Python, a `PriceBar` references a `DataSource` and an `Instrument`. + +In DuckDB, this relationship is stored through IDs: + +```text +price_bars.source_id -> data_sources.id +price_bars.instrument_id -> instruments.id +``` + +This keeps the database normalized while still allowing ARGUS to work with meaningful internal models in Python. + +## Reading Stored Data + +Stored price bars can be read by: + +```text +source +instrument +start date +end date +``` + +The storage layer joins `price_bars`, `data_sources` and `instruments` so that stored IDs become readable market data again. + +Read operations return tabular data that can be used by: + +```text +analytics +charts +dashboards +reports +``` + +This allows ARGUS to process stored historical data without depending on raw API response structures. From b930ff8bb021a195125b0e586a98c11c8f2fc1d3 Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Wed, 1 Jul 2026 12:07:46 +0200 Subject: [PATCH 18/19] docs(#38): update roadmap --- docs/roadmap.md | 137 ++++++++++++++++++++++++++++++------------------ 1 file changed, 85 insertions(+), 52 deletions(-) diff --git a/docs/roadmap.md b/docs/roadmap.md index bc2cdbd..c637327 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -27,88 +27,121 @@ Scope: Outcome: Sprint 1 established the local ARGUS foundation with package structure, GUI prototype, analytics prototype, tests, documentation, CI, Dependabot and governance files. -### Sprint 2 — Market Analytics & Data Source Expansion +### Sprint 2 — Reporting & Market Analytics Foundation **Status:** In progress -Move from simple FX conversion toward broader market analytics. +Move ARGUS from a simple FX-focused prototype toward a first usable market analytics and reporting tool. -Scope: +**Scope:** + +- Add stronger market analytics metrics: -- Add stronger market metrics: - cumulative return - strongest / weakest day - rolling volatility - - performance analytics - - risk analytics -- Extend the current dashboard -- Add or evaluate new data clients: + - basic performance analytics + - basic risk analytics +- Add or improve real market data support: + - yfinance for broader market data + - existing FX conversion remains available where useful - Improve pandas-based analysis workflows -- Add a local storage for historical market data +- Introduce local storage for historical market data - Add report generation and export -- add first prediction feature -- Introduce NiceGUI as a new GUI -- Add tests for metric calculations and data transformations -- Add CD Pipeline +- Add a first simple prediction feature +- Introduce NiceGUI as the next GUI direction +- Extend the current dashboard with real market analytics +- Add tests for metric calculations, data transformations and storage behavior +- Improve CI/CD with first deployment or release automation steps -Outcome: -ARGUS becomes a basic market analytics tool +**Outcome:** + +ARGUS becomes a basic market analytics and reporting tool. +Users can fetch market data, store it locally, calculate metrics, generate a first report and view results through a first modern dashboard. -### Sprint 3 — Storage, Web-Ready UI & Data Architecture +--- + +### Sprint 3 — Advanced Local Analytics & Product Quality **Status:** Planned -Prepare ARGUS for persistent data workflows and a stronger product interface. +Expand the local ARGUS application into a stronger analytics product with better data handling, UI structure, predictions and quality checks. -Scope: +**Scope:** -- Extend local storage layer -- First local ETL Pipeline -- Extend NiceGUI and plan how to combine with modern frotend techstack like django and node.js -- Keep Tkinter as legacy/prototype unless still useful -- More metrics, more instruments and more (and better) prediction features -- Introduce first LLM summary for reports -- Introduce Snyk and Performance Test to cover perfomance and security of argus -- Improve Code Quality +- Extend the local storage layer +- Add a first local ETL workflow +- Improve the NiceGUI dashboard structure and usability +- Explore how NiceGUI can later interact with a more modern frontend stack such as Django, React or Node.js-based services +- Keep Tkinter as legacy/prototype unless it is no longer useful +- Add more metrics, instruments and prediction features +- Improve report templates and report structure +- Introduce first LLM-based summaries for generated reports +- Add first performance tests +- Introduce Snyk or another dependency/security scanning workflow +- Improve code quality, test coverage and maintainability -Outcome: -ARGUS is a scalable analytics application that allows to get more insight from market data +**Outcome:** + +ARGUS becomes a more scalable local analytics application. +It can process more instruments, produce better reports, provide first automated summaries and offer more reliable insight into market data. + +--- -### Sprint 4 — Introduction for extended Analysis +### Sprint 4 — Extended Analysis & Cloud-Ready Foundation **Status:** Planned -Turn ARGUS into a stronger end-to-end data engineering project which is cloud ready. +Prepare ARGUS for deeper analysis, cloud interaction and future portfolio-assistant workflows while keeping the local product usable and transparent. -Scope: +**Scope:** -- Docker Compose -- Intorduce Azure (a simple connection - storage only) -- Better LLM Workflow (introduce RAG) -- Data quality checks -- Caching and efficient storing of market data -- More export possibilities for users -- More metrics and better meta data visualization +- Add Docker Compose for a more complete local development setup +- Introduce a first Azure connection, focused on simple storage or artifact exchange +- Improve the LLM workflow +- Introduce a first RAG-ready structure for reports, notes, documentation and stored analysis artifacts +- Add data quality checks +- Improve caching and efficient storage of market data +- Add more export options for users +- Add more metrics and better metadata visualization +- Improve transparency around data sources, generated reports and analysis assumptions +- Prepare clear interfaces for future cloud and assistant workflows -Outcome: +**Outcome:** -ARGUS is ready to interact with the cloud layer and future cloud app. It's able to give the user an transparent and clear analysis of requested market section. +ARGUS becomes ready to interact with a future cloud layer. +The application can produce clearer, more transparent market analysis and prepares the foundation for retrieval-based workflows, stronger automation and future ARGUS Core integration. -### Sprint 5 — AI-Assisted Research & Agentic Monitoring +--- -**Status:** Future vision +### Sprint 5 — Cloud Interaction & Agentic Monitoring Foundation -Scope: +**Status:** Planned -- First Cloud workflows to extend the analysis -- RAG over stored market notes, reports or documentation -- Agentic checks for data quality, anomalies and recurring market scans -- Human-in-the-loop signal review -- Automated monitoring workflows +Start the first cloud-connected ARGUS workflows and introduce the foundation for monitoring, agentic checks and strategy-support features. -Outcome: +**Scope:** + +- Add first cloud workflows that extend local analysis +- Connect local ARGUS workflows with the first cloud-side services +- Extend RAG over stored market notes, reports, documentation and analysis artifacts +- Add agentic checks for: + + - data quality + - anomalies + - recurring market scans + - report consistency +- Add first human-in-the-loop review workflows for signals or strategy ideas +- Add automated monitoring workflows +- Prepare the first foundations for: + + - paper trading + - backtesting + - controlled strategy evaluation + - future portfolio-assistant workflows + +**Outcome:** -ARGUS and the cloud app interact with each other. ARGUS become the first time an useful monitoring and alysis tool. -It's the beginn of ARGUS to help the user to find, implement and deploy strategies. Through that ARGUS will -be first time able to give signals and allow paper trading, back tests and controlled trading with agents. +ARGUS and the first cloud-side services begin to interact. +ARGUS becomes useful not only as an analytics and reporting tool, but also as the first foundation for monitoring, strategy evaluation and controlled market-research workflows. From 90e11de118e429e6c0d9f163a697b58dd59144bd Mon Sep 17 00:00:00 2001 From: Lev Gusiev Date: Wed, 1 Jul 2026 12:12:17 +0200 Subject: [PATCH 19/19] docs(#38): add docstrings --- src/argus/storage/database.py | 47 +++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/src/argus/storage/database.py b/src/argus/storage/database.py index f184b65..ea7128c 100644 --- a/src/argus/storage/database.py +++ b/src/argus/storage/database.py @@ -5,6 +5,18 @@ def initialize_database(database_path: str) -> None: + """ + Initialize the DuckDB database schema. + + Creates the required sequences and tables for data sources, + instruments, and price bars. + + Args: + database_path (str): Path to the DuckDB database file. + + Returns: + None + """ queries = [ "CREATE SEQUENCE IF NOT EXISTS data_sources_id_seq;", "CREATE SEQUENCE IF NOT EXISTS instruments_id_seq;", @@ -58,6 +70,23 @@ def initialize_database(database_path: str) -> None: def get_or_create_source(connection, source: DataSource) -> int: + """ + Get an existing data source ID or create a new data source. + + Searches for a data source by name. If it already exists, its ID is + returned. Otherwise, the data source is inserted and the new ID is + returned. + + Args: + connection: Active DuckDB connection. + source (DataSource): Data source model containing provider metadata. + + Returns: + int: Database ID of the existing or newly created data source. + + Raises: + ValueError: If the data source could not be inserted or found. + """ insert_query = """ INSERT INTO data_sources (name, provider_kind, requires_api_key) VALUES (?,?,?) @@ -92,6 +121,24 @@ def get_or_create_source(connection, source: DataSource) -> int: def get_or_create_instrument(connection, instrument: Instrument) -> int: + """ + Get an existing instrument ID or create a new instrument. + + Searches for an instrument by symbol. If it already exists, its ID is + returned. Otherwise, the instrument is inserted and the new ID is + returned. + + Args: + connection: Active DuckDB connection. + instrument (Instrument): Instrument model containing symbol and + asset metadata. + + Returns: + int: Database ID of the existing or newly created instrument. + + Raises: + ValueError: If the instrument could not be inserted or found. + """ insert_query = """ INSERT INTO instruments ( symbol,