From 309db7a65d657ba6f4999f29954324691c6e2ad2 Mon Sep 17 00:00:00 2001 From: Lukas Simonik Date: Wed, 12 Aug 2026 17:40:16 +0200 Subject: [PATCH 1/5] feat(web): add tab menu on URL detail page --- web/main.py | 37 ++++++++++++++++++++++++++++- web/static/detail.css | 50 +++++++++++++++++++++++++++++++++++++++ web/templates/detail.html | 15 ++++++++++++ 3 files changed, 101 insertions(+), 1 deletion(-) diff --git a/web/main.py b/web/main.py index b6b1278..48faf69 100644 --- a/web/main.py +++ b/web/main.py @@ -251,11 +251,39 @@ def __init__(self, url_detail): self.contained_urls = [] +def get_detail_menu(url, show=None): + """Build the tab menu displayed under the URL on the detail page. + + Badge counts are static placeholders for now (see the design mockup); + they will be computed from the new data model tables once implemented. + """ + tabs = [ + ("overview", "Overview", None), + ("content", "Content", 2), + ("sources", "Sources", 2), + ("sandbox", "Sandbox", 1), + ("class_history", "Class. History", 2), + ] + menu = [] + for tab_id, label, count in tabs: + params = {"url": url, "tab": tab_id} + if show: + params["show"] = show + menu.append({ + "id": tab_id, + "label": label, + "count": count, + "href": url_for("detail", **params), + }) + return menu + + @app.route('/detail', methods=['GET', 'POST']) def detail(): user = get_user(flask.request.environ) show = flask.request.args.get('show') url = flask.request.args.get('url') + active_tab = flask.request.args.get('tab', 'overview') with SQLiteWrapper(config.db_path) as db: if flask.request.method == 'POST': @@ -294,7 +322,10 @@ def detail(): "joe-sandbox": f"https://www.joesandbox.com/analysis/search?q={url_detail.hash}" } - return render_template('detail.html', user=user, url=url_detail, sessions=sessions, show=show, links=links, inactive_for=inactive_for) + # tab menu under the URL name (FE menu only for now, counts are placeholders) + menu = get_detail_menu(url, show) + + return render_template('detail.html', user=user, url=url_detail, sessions=sessions, show=show, links=links, inactive_for=inactive_for, menu=menu, active_tab=active_tab) @app.route('/edit_detail', methods=['GET', 'POST']) @@ -387,3 +418,7 @@ def api_url_stats(): "src": ", ".join([s[0] for s in url_sources]), } return make_response(jsonify(return_dict), 200) + +if __name__ == '__main__': + app.run(host='127.0.0.1', port=5000, debug=True) + diff --git a/web/static/detail.css b/web/static/detail.css index 9e4f393..1886754 100755 --- a/web/static/detail.css +++ b/web/static/detail.css @@ -16,6 +16,56 @@ overflow-wrap: anywhere; } +/* --- tab menu under the URL name --- */ +.detail .detail-menu { + margin: 0 20px; + border-bottom: 2px solid #e0e0e0; +} + +.detail .detail-menu ul { + display: flex; + flex-direction: row; + flex-wrap: wrap; + gap: 4px; + list-style: none; + margin: 0; + padding: 0; +} + +.detail .detail-menu-item { + display: inline-flex; + align-items: center; + gap: 8px; + padding: 8px 14px; + margin-bottom: -2px; + color: #818181; + text-decoration: none; + border-bottom: 2px solid transparent; +} + +.detail .detail-menu-item:hover { + color: #00838f; +} + +.detail .detail-menu-item.active { + color: #00acc1; + font-weight: bold; + border-bottom: 2px solid #00acc1; +} + +.detail .detail-menu-badge { + min-width: 20px; + height: 20px; + padding: 0 6px; + line-height: 20px; + text-align: center; + font-size: 12px; + font-weight: normal; + color: white; + background-color: #9e9e9e; + border-radius: 4px; +} + .detail .status { font-weight: bold; } diff --git a/web/templates/detail.html b/web/templates/detail.html index bb4803d..6af0405 100755 --- a/web/templates/detail.html +++ b/web/templates/detail.html @@ -23,6 +23,21 @@ onclick="location.href='{{ url_for('edit_detail', url=url.url) }}';"> + {% if menu %} + + {% endif %} +
From af02086cdfc2005eefe4d322d7c0cfc7c6264006 Mon Sep 17 00:00:00 2001 From: Lukas Simonik Date: Wed, 12 Aug 2026 19:39:35 +0200 Subject: [PATCH 2/5] url_source: store in which honeynets a URL was observed (first/last seen, occurrences) + web Sources tab with per-source stats and derived-from link --- bin/honeynetasia2evaluator.py | 5 ++-- common/utils.py | 30 ++++++++++++++++++++++- install/create_db.sql | 9 ++++--- web/main.py | 40 +++++++++++++++++++++---------- web/static/detail.css | 45 +++++++++++++++++++++++++++++++++++ web/templates/detail.html | 40 ++++++++++++++++++++++++++++++- 6 files changed, 150 insertions(+), 19 deletions(-) diff --git a/bin/honeynetasia2evaluator.py b/bin/honeynetasia2evaluator.py index aa829e8..b77d5eb 100644 --- a/bin/honeynetasia2evaluator.py +++ b/bin/honeynetasia2evaluator.py @@ -13,7 +13,7 @@ sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))) from common.config import Config from common.db import SQLiteWrapper -from common.utils import is_valid, get_domain +from common.utils import is_valid, get_domain, record_url_source def honeynetasia2evaluator(): @@ -39,7 +39,8 @@ def honeynetasia2evaluator(): last_seen = excluded.last_seen, occurrences = urls.occurrences + 1; """, (url, current_date, current_date, get_domain(url))).rowcount - db.execute("INSERT OR IGNORE INTO url_source (url, source) VALUES (?, ?)", (url, "HoneyNet.Asia")) + # Record/update per-source observation statistics (first/last seen, occurrences) + record_url_source(db, url, "HoneyNet.Asia", date=current_date) logger.info(f"{num_inserted} URLs inserted or updated") logger.info("Job finished") diff --git a/common/utils.py b/common/utils.py index 652f00d..fe07e83 100644 --- a/common/utils.py +++ b/common/utils.py @@ -72,6 +72,33 @@ def get_domain(url: str): return None +def record_url_source(db, url, source, date=None, count=1): + """ + Record that a URL was observed in a source (e.g. a honeynet feed). + + Tracks per-source observation statistics in the url_source table: + - first_seen: date of the first observation of the URL in this source + - last_seen: date of the most recent observation (updated to the latest) + - occurrences: how many times the URL was observed in this source + + :param db: database wrapper with an execute() method + :param url: observed URL + :param source: name of the source (honeynet) the URL was observed in + :param date: date of the observation (defaults to today, UTC) + :param count: how many observations (occurrences) to add (default 1) + """ + if date is None: + from datetime import datetime, timezone + date = datetime.now(timezone.utc).strftime('%Y-%m-%d') + db.execute( + """ + INSERT INTO url_source (url, source, first_seen, last_seen, occurrences) VALUES (?, ?, ?, ?, ?) + ON CONFLICT(url, source) DO UPDATE SET + occurrences = url_source.occurrences + excluded.occurrences, + last_seen = MAX(url_source.last_seen, excluded.last_seen); + """, (url, source, date, date, count)) + + def process_new_session(db, config, session, idea_id, detect_time, source, source_url): """ Process a new session: @@ -103,7 +130,8 @@ def process_new_session(db, config, session, idea_id, detect_time, source, sourc ) for url, occurrences in Counter(extracted_urls).items(): db.execute("INSERT OR IGNORE INTO url_session (url, session) VALUES (?, ?)", (url, session_hash)) - db.execute("INSERT OR IGNORE INTO url_source (url, source) VALUES (?, ?)", (url, source)) + # Record/update per-source observation statistics (first/last seen, occurrences) + record_url_source(db, url, source, date=date) if source_url: db.execute("INSERT OR IGNORE INTO discovered_urls (url, src_url) VALUES (?, ?)", (url, source_url)) db.execute( diff --git a/install/create_db.sql b/install/create_db.sql index 3df10a2..2447fd0 100644 --- a/install/create_db.sql +++ b/install/create_db.sql @@ -16,9 +16,12 @@ CREATE TABLE url_session CREATE TABLE url_source ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - url TEXT REFERENCES urls(url), - source TEXT, + id INTEGER PRIMARY KEY AUTOINCREMENT, + url TEXT REFERENCES urls(url), + source TEXT, + first_seen DATE, + last_seen DATE, + occurrences INTEGER DEFAULT 1, CONSTRAINT url_source_unique UNIQUE (url, source) ); diff --git a/web/main.py b/web/main.py index 48faf69..73d8626 100644 --- a/web/main.py +++ b/web/main.py @@ -247,32 +247,38 @@ def __init__(self, url_detail): self.eval_later = url_detail[17] self.ip = get_ip(self.url) self.src = [] + # per-source observation rows for the Sources tab: + # (source, first_seen, last_seen, occurrences, derived_from) + self.source_rows = [] self.src_urls = [] self.contained_urls = [] -def get_detail_menu(url, show=None): +def get_detail_menu(url, show=None, counts=None): """Build the tab menu displayed under the URL on the detail page. - Badge counts are static placeholders for now (see the design mockup); - they will be computed from the new data model tables once implemented. + :param url: the URL whose detail page it is + :param show: optional "show" filter to keep in tab links + :param counts: optional dict mapping tab id -> badge count; tabs without + an entry in the dict display no badge """ + counts = counts or {} tabs = [ - ("overview", "Overview", None), - ("content", "Content", 2), - ("sources", "Sources", 2), - ("sandbox", "Sandbox", 1), - ("class_history", "Class. History", 2), + ("overview", "Overview"), + ("content", "Content"), + ("sources", "Sources"), + ("sandbox", "Sandbox"), + ("class_history", "Class. History"), ] menu = [] - for tab_id, label, count in tabs: + for tab_id, label in tabs: params = {"url": url, "tab": tab_id} if show: params["show"] = show menu.append({ "id": tab_id, "label": label, - "count": count, + "count": counts.get(tab_id), "href": url_for("detail", **params), }) return menu @@ -293,6 +299,16 @@ def detail(): # get url details url_detail = URLDetail(db.execute("SELECT url, first_seen, last_seen, hash, classification, classification_reason, note, reported, occurrences, vt_stats, evaluated, file_mime_type, content_size, threat_label, status, last_active, last_edit, eval_later FROM urls WHERE url = ? LIMIT 1", (url,)).fetchone()) url_detail.src = [row[0] for row in db.execute("SELECT source FROM url_source WHERE url = ?", (url,)).fetchall()] + # per-source observation stats for the Sources tab (source, first_seen, last_seen, occurrences, derived_from) + # "derived_from" is the source URL this URL was extracted from (discovered_urls), + # which allows deriving the original source for URLs extracted from a script hosted on another URL + url_detail.source_rows = db.execute(""" + SELECT us.source, us.first_seen, us.last_seen, us.occurrences, + (SELECT du.src_url FROM discovered_urls du WHERE du.url = us.url LIMIT 1) AS derived_from + FROM url_source us + WHERE us.url = ? + ORDER BY us.source + """, (url,)).fetchall() url_detail.src_urls = db.execute("SELECT src_url FROM discovered_urls WHERE url = ?", (url_detail.url,)).fetchall() url_detail.contained_urls = db.execute("SELECT url FROM discovered_urls WHERE src_url = ?", (url,)).fetchall() sessions = db.execute("SELECT sessions.session, sessions.idea_id FROM sessions JOIN url_session ON url_session.session=sessions.session_hash WHERE url_session.url = ?", (url,)).fetchall() @@ -322,8 +338,8 @@ def detail(): "joe-sandbox": f"https://www.joesandbox.com/analysis/search?q={url_detail.hash}" } - # tab menu under the URL name (FE menu only for now, counts are placeholders) - menu = get_detail_menu(url, show) + # tab menu under the URL name; badge counts reflect real DB data where available + menu = get_detail_menu(url, show, counts={"sources": len(url_detail.source_rows)}) return render_template('detail.html', user=user, url=url_detail, sessions=sessions, show=show, links=links, inactive_for=inactive_for, menu=menu, active_tab=active_tab) diff --git a/web/static/detail.css b/web/static/detail.css index 1886754..cf2f342 100755 --- a/web/static/detail.css +++ b/web/static/detail.css @@ -259,6 +259,51 @@ padding-left: 20px !important; } +/* --- Sources tab table --- */ +.detail .sources-table { + width: 100%; + border-collapse: collapse; +} + +.detail .sources-table th { + text-align: left; + color: #818181; + font-weight: normal; + padding: 8px 12px 8px 0; + border-bottom: 1px solid #e0e0e0; +} + +.detail .sources-table td { + padding: 10px 12px 10px 0; + border-bottom: 1px solid #eee; + vertical-align: middle; +} + +.detail .sources-table th.num, +.detail .sources-table td.num { + text-align: right; +} + +.detail .sources-table td.empty { + color: #818181; + text-align: center; + padding: 20px 0; +} + +.detail .source-badge { + display: inline-block; + padding: 2px 10px; + background-color: #e0e0e0; + border-radius: 4px; + font-size: small; +} + +.detail .sources-table a.src-url { + text-decoration: underline; + color: grey; + overflow-wrap: anywhere; +} + .detail .list-urls { background-color: #f8f8f8; border: 1px solid #ddd; diff --git a/web/templates/detail.html b/web/templates/detail.html index 6af0405..cf6ae26 100755 --- a/web/templates/detail.html +++ b/web/templates/detail.html @@ -39,6 +39,43 @@ {% endif %}
+ {% if active_tab == 'sources' %} + +
+ + + + + + + + + + + {% for row in url.source_rows %} + + + + + + + + {% else %} + + {% endfor %} + +
HoneynetFirst observedLast observedOccurrencesDerived from
{{ row[0] }}{{ row[1] if row[1] else '—' }}{{ row[2] if row[2] else '—' }}{{ row[3] if row[3] is not none else '—' }} + {% if row[4] %} + {{ row[4] }} + {% else %} + — + {% endif %} +
No sources recorded for this URL
+ {% else %} + @@ -207,9 +244,10 @@ {% endif %} + {% endif %} - +
{% if url.evaluated == 'yes' %}
From 3e92e3e081b757745df66c6493f662c5a3eb3fc2 Mon Sep 17 00:00:00 2001 From: Lukas Simonik Date: Thu, 13 Aug 2026 22:55:21 +0200 Subject: [PATCH 3/5] feat: URL content storage with dedup, metadata, and history --- common/content_storage.py | 111 +++++++++++++++++++ common/db_helpers.py | 220 ++++++++++++++++++++++++++++++++++++++ common/utils.py | 123 +++++++++++++++++++-- etc/config.yaml | 9 ++ install/create_db.sql | 110 +++++++++++++++++-- web/main.py | 70 +++++++++++- web/static/detail.css | 94 ++++++++++++++++ web/templates/detail.html | 59 +++++++++- 8 files changed, 773 insertions(+), 23 deletions(-) create mode 100644 common/content_storage.py create mode 100644 common/db_helpers.py diff --git a/common/content_storage.py b/common/content_storage.py new file mode 100644 index 0000000..80d5636 --- /dev/null +++ b/common/content_storage.py @@ -0,0 +1,111 @@ +""" +File-system content storage for URL Evaluator. + +Binary payloads (downloaded URL content / malware samples) are stored on +disk, deduplicated by their SHA-256 digest. The database only keeps metadata +plus the relative ``storage_path`` produced here, so identical content served +by multiple URLs results in a single on-disk file. + +Layout: + ///.blob + +where ``aa``/``bb`` are the first two pairs of hex characters of the digest. +Files are immutable — the same content is never written twice. +""" + +import os +import hashlib +import logging +from pathlib import Path + +logger = logging.getLogger(__name__) + +BLOB_SUFFIX = ".blob" + + +def compute_hashes(content: bytes): + """Return (sha1, sha256) hex digests of *content*.""" + sha1 = hashlib.sha1(content).hexdigest() + sha256 = hashlib.sha256(content).hexdigest() + return sha1, sha256 + + +def storage_path(base_dir, content_hash: str) -> Path: + """Return the absolute :class:`Path` where *content_hash* would be stored. + + The path is derived purely from the hash, so it is stable and independent + of when/by whom the content was downloaded. + """ + rel = relative_storage_path(content_hash) + return Path(base_dir) / rel + + +def relative_storage_path(content_hash: str) -> str: + """Return the relative storage path (``aa/bb/.blob``) for *content_hash*.""" + if not content_hash or len(content_hash) < 4: + raise ValueError(f"content_hash too short to build storage path: {content_hash!r}") + return os.path.join(content_hash[0:2], content_hash[2:4], content_hash + BLOB_SUFFIX) + + +def content_exists(base_dir, content_hash: str) -> bool: + """Return True if a blob for *content_hash* already exists on disk.""" + return storage_path(base_dir, content_hash).is_file() + + +def save_content(base_dir, content: bytes): + """Persist *content* under *base_dir*, deduplicated by SHA-256. + + :param base_dir: base storage directory (from config ``content_storage_path``) + :param content: raw downloaded bytes + :returns: ``(sha256, sha1, relative_path, is_new)`` where ``is_new`` is True + when a file was actually written (False on dedup-hit). + """ + logger.debug(f"save_content: hashing {len(content)} bytes") + sha1, sha256 = compute_hashes(content) + dest = storage_path(base_dir, sha256) + rel = relative_storage_path(sha256) + logger.debug(f"save_content: sha256={sha256}, target={dest}") + + if dest.is_file(): + logger.debug(f"Content deduplicated (already stored): {sha256}") + return sha256, sha1, rel, False + + dest.parent.mkdir(parents=True, exist_ok=True) + # Write atomically: tmp file then rename, so concurrent readers never see a + # partially-written blob. + tmp = dest.with_suffix(dest.suffix + ".tmp") + logger.debug(f"save_content: writing tmp file {tmp}") + with open(tmp, "wb") as fh: + fh.write(content) + os.replace(tmp, dest) + logger.info(f"Stored new content blob: {rel} ({len(content)} bytes)") + logger.debug(f"save_content: atomically moved to {dest}") + return sha256, sha1, rel, True + + +def load_content(base_dir, content_hash: str) -> bytes: + """Read and return the stored bytes for *content_hash*. + + :raises FileNotFoundError: if no blob exists for the given hash. + """ + path = storage_path(base_dir, content_hash) + with open(path, "rb") as fh: + return fh.read() + + +def delete_content(base_dir, content_hash: str) -> bool: + """Remove the blob for *content_hash* if present. Returns True when removed. + + Prune any now-empty parent directories as well. + """ + path = storage_path(base_dir, content_hash) + if not path.is_file(): + return False + path.unlink() + # Try to clean up the (now possibly empty) aa/bb directories. + for parent in (path.parent, path.parent.parent): + try: + parent.rmdir() + except OSError: + pass + return True diff --git a/common/db_helpers.py b/common/db_helpers.py new file mode 100644 index 0000000..e2e29e3 --- /dev/null +++ b/common/db_helpers.py @@ -0,0 +1,220 @@ +""" +Shared database helper functions for URL Evaluator. + +These helpers own all writes to the history-tracking and content tables so +individual backend modules don't duplicate SQL: + +- :func:`record_url_history` – append-on-change audit entries (``url_history``) +- :func:`update_url_field` – update a ``urls`` column and record history +- :func:`set_url_latest_content` – flip the ``url_content.is_latest`` marker +- :func:`persist_content_snapshot` – save a downloaded payload to disk (dedup) + and persist ``content_snapshot`` + ``url_content`` + ``urls`` metadata. + +Used by the evaluator, the web edit handlers and ingestion modules. +""" + +import json +import logging +from datetime import datetime, timezone + +from common.content_storage import save_content + +logger = logging.getLogger(__name__) + +FETCH_IP_TIMEOUT = 5 + +# Columns of the ``urls`` table that update_url_field() is allowed to change. +# Business columns only – primary key and auto-tracked timestamps are excluded. +URL_UPDATABLE_FIELDS = { + "hash", + "classification", + "classification_reason", + "note", + "reported", + "occurrences", + "vt_stats", + "evaluated", + "file_mime_type", + "content_size", + "threat_label", + "status", + "last_active", + "status_changed", + "last_edit", + "eval_later", + "domain", + "latest_content_hash", +} + + +def _now(): + return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S") + + +def record_url_history(db, url, field, old_value, new_value, changed_by="system"): + """Append a change record to ``url_history`` when a value actually changed. + + No row is written when ``old_value == new_value`` so the history stays free + of no-op noise. + """ + if old_value == new_value: + return False + db.execute( + """ + INSERT INTO url_history (url, changed_at, field, old_value, new_value, changed_by) + VALUES (?, ?, ?, ?, ?, ?) + """, + (url, _now(), field, old_value, new_value, changed_by), + ) + return True + + +def update_url_field(db, url, field, new_value, changed_by="system"): + """Update a single ``urls`` column and record the change in history. + + :returns: True when the value changed (and history was written), False when + the new value equals the stored one (no update performed). + :raises ValueError: when *field* is not an updatable ``urls`` column — this + also protects against SQL injection via the column name. + """ + if field not in URL_UPDATABLE_FIELDS: + raise ValueError(f"Invalid URL field: {field}") + + row = db.execute(f"SELECT {field} FROM urls WHERE url = ?", (url,)).fetchone() + old_value = row[0] if row else None + + if old_value == new_value: + return False + + db.execute(f"UPDATE urls SET {field} = ? WHERE url = ?", (new_value, url)) + record_url_history(db, url, field, old_value, new_value, changed_by=changed_by) + return True + + +def set_url_latest_content(db, url, content_hash): + """Mark *content_hash* as the current content of *url* in ``url_content``. + + An existing (url, content_hash) row only gets ``is_latest='yes'`` and a + bumped ``last_seen``; a new one is created with ``first_seen == last_seen``. + All other rows for the URL are flipped to ``is_latest='no'``. + """ + now = _now() + logger.debug(f"set_url_latest_content: marking {content_hash[:16]}... as latest for {url}") + + existing = db.execute( + "SELECT id FROM url_content WHERE url = ? AND content_hash = ?", + (url, content_hash), + ).fetchone() + + if existing: + logger.debug(f"set_url_latest_content: updating existing url_content row id={existing[0]}") + db.execute( + "UPDATE url_content SET last_seen = ?, is_latest = 'yes' WHERE id = ?", + (now, existing[0]), + ) + else: + logger.debug(f"set_url_latest_content: creating new url_content row for {url}") + db.execute( + "INSERT INTO url_content (url, content_hash, first_seen, last_seen, is_latest) VALUES (?, ?, ?, ?, 'yes')", + (url, content_hash, now, now), + ) + + db.execute( + "UPDATE url_content SET is_latest = 'no' WHERE url = ? AND content_hash != ?", + (url, content_hash), + ) + logger.debug(f"set_url_latest_content: other content rows for {url} marked is_latest='no'") + + +def _extract_connection_ips(response): + """Best-effort extraction of (source_ip, server_ip) from a requests response. + + Reads the underlying urllib3 connection socket. Returns ``(None, None)`` + when the information isn't available (e.g. mocked responses in tests). + """ + source_ip = server_ip = None + try: + sock = response.raw._connection.sock + if sock is not None: + local = sock.getsockname() + peer = sock.getpeername() + if local: + source_ip = local[0] + if peer: + server_ip = peer[0] + logger.debug(f"_extract_connection_ips: source_ip={source_ip}, server_ip={server_ip}") + else: + logger.debug("_extract_connection_ips: underlying socket is None") + except Exception: + pass + return source_ip, server_ip + + +def persist_content_snapshot(db, base_dir, url, response, content, mime_type): + """Store a downloaded payload and persist its metadata + history link. + + Steps: + 1. Write *content* to the file storage (SHA-256 dedup). + 2. Insert-or-ignore a ``content_snapshot`` row (one per unique hash). + 3. Refresh the ``url_content`` link and mark it latest. + 4. Update ``urls.hash``/``latest_content_hash``/``file_mime_type``/``content_size`` + and record a ``latest_content_hash`` change in ``url_history``. + + :param base_dir: content storage base directory (config ``content_storage_path``) + :param url: URL the content was downloaded from + :param response: the ``requests`` response object (for status/headers/IPs) + :param content: raw downloaded bytes + :param mime_type: detected MIME type of the content + :returns: dict with the new ``hash`` (sha1), ``latest_content_hash`` (sha256), + ``file_mime_type``, ``content_size`` and ``storage_path``. + """ + logger.debug(f"persist_content_snapshot: persisting {len(content)} bytes for {url} (base_dir={base_dir}, mime={mime_type})") + sha256, sha1, rel_path, _is_new = save_content(base_dir, content) + logger.debug(f"persist_content_snapshot: saved -> sha256={sha256}, sha1={sha1}, path={rel_path}, is_new={_is_new}") + + http_status = getattr(response, "status_code", None) + headers = getattr(response, "headers", {}) or {} + try: + http_headers = json.dumps(dict(headers)) + except (TypeError, ValueError): + http_headers = json.dumps({str(k): str(v) for k, v in headers.items()}) + + source_ip, server_ip = _extract_connection_ips(response) + downloaded_at = _now() + + logger.debug(f"persist_content_snapshot: inserting content_snapshot row (hash={sha256[:16]}..., status={http_status}, src={source_ip}, dst={server_ip})") + db.execute( + """ + INSERT INTO content_snapshot + (content_hash, url, downloaded_at, source_ip, server_ip, http_status, + http_headers, mime_type, content_size, storage_path, sha1, sha256) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(content_hash) DO NOTHING + """, + (sha256, url, downloaded_at, source_ip, server_ip, http_status, + http_headers, mime_type, len(content), rel_path, sha1, sha256), + ) + + previous_hash_row = db.execute( + "SELECT latest_content_hash FROM urls WHERE url = ?", (url,) + ).fetchone() + previous_hash = previous_hash_row[0] if previous_hash_row else None + logger.debug(f"persist_content_snapshot: previous latest_content_hash for {url}: {previous_hash}") + + set_url_latest_content(db, url, sha256) + + db.execute( + "UPDATE urls SET hash = ?, latest_content_hash = ?, file_mime_type = ?, content_size = ? WHERE url = ?", + (sha1, sha256, mime_type, len(content), url), + ) + record_url_history(db, url, "latest_content_hash", previous_hash, sha256, changed_by="system") + logger.debug(f"persist_content_snapshot: urls row updated + history recorded for {url}") + + return { + "hash": sha1, + "latest_content_hash": sha256, + "file_mime_type": mime_type, + "content_size": len(content), + "storage_path": rel_path, + "is_new": _is_new, + } diff --git a/common/utils.py b/common/utils.py index fe07e83..17d060a 100644 --- a/common/utils.py +++ b/common/utils.py @@ -72,7 +72,24 @@ def get_domain(url: str): return None -def record_url_source(db, url, source, date=None, count=1): +def _origin_source_of(db, url): + """ + Resolve the (source, source_detail) of a URL's first observation, used to + derive the original honeynet of a URL extracted from that URL's content. + Dict/tuple-row tolerant. Returns (None, None) when not observed yet. + """ + origin = db.execute( + "SELECT source, source_detail FROM url_source WHERE url = ? ORDER BY observed_at, rowid LIMIT 1", + (url,)).fetchone() + if not origin: + return None, None + if isinstance(origin, dict): + return origin.get("source"), origin.get("source_detail") + return origin[0], origin[1] + + +def record_url_source(db, url, source, date=None, count=1, source_detail=None, origin_url=None, + observed_at=None, idea_id=None, session_hash=None): """ Record that a URL was observed in a source (e.g. a honeynet feed). @@ -80,23 +97,102 @@ def record_url_source(db, url, source, date=None, count=1): - first_seen: date of the first observation of the URL in this source - last_seen: date of the most recent observation (updated to the latest) - occurrences: how many times the URL was observed in this source + - observed_at: when the URL was first observed (kept at the earliest value) + - source_detail / origin_url / idea_id / session_hash: provenance metadata + + For URLs extracted from a script hosted on another URL (``origin_url``), the + original source (``origin_source`` / ``origin_source_detail``) is resolved + from the origin URL's own first observation, so the original honeynet can be + derived. :param db: database wrapper with an execute() method :param url: observed URL :param source: name of the source (honeynet) the URL was observed in :param date: date of the observation (defaults to today, UTC) :param count: how many observations (occurrences) to add (default 1) + :param source_detail: detail of the source (sensor/node name) + :param origin_url: URL from which this URL was extracted (if any) + :param observed_at: when the URL was observed (defaults to now, UTC) + :param idea_id: IDEA event ID + :param session_hash: hash of the session the URL was observed in """ + from datetime import datetime, timezone if date is None: - from datetime import datetime, timezone date = datetime.now(timezone.utc).strftime('%Y-%m-%d') + if observed_at is None: + observed_at = datetime.now(timezone.utc).isoformat() + + # Resolve the original source of a URL extracted from another URL's content, + # so the original honeynet can be derived. Uses the origin URL's first observation. + origin_source = origin_source_detail = None + if origin_url: + origin_source, origin_source_detail = _origin_source_of(db, origin_url) + + # Schema-adaptive column list: the production url_source carries cumulative + # stats columns (first_seen/last_seen/occurrences), but a minimal schema may not. + def _col_name(row): + # PRAGMA table_info row: (cid, name, ...) as tuple, or {'name': ...} with a dict row factory + if isinstance(row, dict): + return row.get("name") + return row[1] + + existing = {_col_name(row) for row in db.execute("PRAGMA table_info(url_source)").fetchall()} + + columns = ["url", "source"] + values = [url, source] + if "first_seen" in existing: + columns += ["first_seen", "last_seen", "occurrences"] + values += [date, date, count] + columns += ["source_detail", "origin_url", "origin_source", "origin_source_detail", + "observed_at", "idea_id", "session_hash"] + values += [source_detail, origin_url, origin_source, origin_source_detail, + observed_at, idea_id, session_hash] + + updates = [] + if "occurrences" in existing: + updates.append("occurrences = url_source.occurrences + excluded.occurrences") + if "last_seen" in existing: + updates.append("last_seen = MAX(url_source.last_seen, excluded.last_seen)") + updates.append("observed_at = MIN(url_source.observed_at, excluded.observed_at)") + + placeholders = ", ".join("?" * len(columns)) db.execute( - """ - INSERT INTO url_source (url, source, first_seen, last_seen, occurrences) VALUES (?, ?, ?, ?, ?) + f""" + INSERT INTO url_source ({", ".join(columns)}) + VALUES ({placeholders}) ON CONFLICT(url, source) DO UPDATE SET - occurrences = url_source.occurrences + excluded.occurrences, - last_seen = MAX(url_source.last_seen, excluded.last_seen); - """, (url, source, date, date, count)) + {", ".join(updates)}; + """, + values) + + +def record_discovered_url(db, url, src_url, discovered_at=None): + """ + Record that a URL was found in the content of another URL. + + Stores the link in discovered_urls, caching the original source of the + ``src_url`` (the URL hosting the script this URL was extracted from), so the + original honeynet can be derived. + + :param db: database wrapper with an execute() method + :param url: the discovered (extracted) URL + :param src_url: the URL in whose content ``url`` was found + :param discovered_at: when the URL was discovered (defaults to now, UTC) + """ + from datetime import datetime, timezone + if discovered_at is None: + discovered_at = datetime.now(timezone.utc).isoformat() + + # Cache the original source of the src_url (its first observation). + origin_source, origin_source_detail = _origin_source_of(db, src_url) + + db.execute( + """ + INSERT INTO discovered_urls (url, src_url, discovered_at, origin_source, origin_source_detail) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(url, src_url) DO NOTHING; + """, + (url, src_url, discovered_at, origin_source, origin_source_detail)) def process_new_session(db, config, session, idea_id, detect_time, source, source_url): @@ -130,10 +226,17 @@ def process_new_session(db, config, session, idea_id, detect_time, source, sourc ) for url, occurrences in Counter(extracted_urls).items(): db.execute("INSERT OR IGNORE INTO url_session (url, session) VALUES (?, ?)", (url, session_hash)) - # Record/update per-source observation statistics (first/last seen, occurrences) - record_url_source(db, url, source, date=date) + # Record/update per-source observation statistics (first/last seen, occurrences, provenance) + record_url_source( + db, url, source, + date=date, + source_detail=source_url, + origin_url=source_url, + observed_at=detect_time, + idea_id=idea_id, + session_hash=session_hash) if source_url: - db.execute("INSERT OR IGNORE INTO discovered_urls (url, src_url) VALUES (?, ?)", (url, source_url)) + record_discovered_url(db, url, source_url, discovered_at=detect_time) db.execute( """ INSERT INTO urls (url, first_seen, last_seen, domain) VALUES (?, ?, ?, ?) diff --git a/etc/config.yaml b/etc/config.yaml index b9490a8..e53e028 100644 --- a/etc/config.yaml +++ b/etc/config.yaml @@ -37,6 +37,15 @@ max_age_invalid: 7 # days # Max size of downloaded content max_file_size: 100 # MB +# Base directory for downloaded content snapshots (deduplicated file storage) +content_storage_path: "/data/url_evaluator/content" + +# Optional soft limit / warning threshold for total content storage (GB); 0 disables +max_storage_size_gb: 0 + +# Placeholder list of sandbox providers for future integration +sandbox_providers: [] + # How often should evaluation blacklist be updated bl_update_time: 15 # minutes diff --git a/install/create_db.sql b/install/create_db.sql index 2447fd0..2f81359 100644 --- a/install/create_db.sql +++ b/install/create_db.sql @@ -16,21 +16,31 @@ CREATE TABLE url_session CREATE TABLE url_source ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - url TEXT REFERENCES urls(url), - source TEXT, - first_seen DATE, - last_seen DATE, - occurrences INTEGER DEFAULT 1, + id INTEGER PRIMARY KEY AUTOINCREMENT, + url TEXT REFERENCES urls(url), + source TEXT, + first_seen DATE, + last_seen DATE, + occurrences INTEGER DEFAULT 1, + source_detail TEXT, + origin_url TEXT, + origin_source TEXT, + origin_source_detail TEXT, + observed_at DATETIME, + idea_id TEXT, + session_hash TEXT REFERENCES sessions(session_hash), CONSTRAINT url_source_unique UNIQUE (url, source) ); CREATE TABLE discovered_urls ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - url TEXT REFERENCES urls(url), - src_url TEXT REFERENCES urls(url), + id INTEGER PRIMARY KEY AUTOINCREMENT, + url TEXT REFERENCES urls(url), + src_url TEXT REFERENCES urls(url), + discovered_at DATETIME, + origin_source TEXT, + origin_source_detail TEXT, CONSTRAINT discovered_urls_unique UNIQUE (url, src_url) ); @@ -56,5 +66,85 @@ CREATE TABLE urls status_changed TEXT DEFAULT 'no' CHECK (status_changed IN ('yes', 'no')), last_edit TEXT, eval_later TEXT DEFAULT 'no' CHECK (eval_later IN ('yes', 'no')), - domain TEXT + domain TEXT, + latest_content_hash TEXT +); + +-- One row per unique downloaded content (deduplicated by SHA-256). +-- Binary payload lives on disk under ///.blob +-- this table only keeps metadata + the storage pointer. +CREATE TABLE content_snapshot +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + content_hash TEXT UNIQUE, + url TEXT, + downloaded_at DATETIME, + source_ip TEXT, + server_ip TEXT, + http_status INTEGER, + http_headers TEXT, + mime_type TEXT, + content_size INTEGER, + storage_path TEXT, + sha1 TEXT, + sha256 TEXT +); + +-- Many-to-many between URLs and content snapshots, carrying per-URL +-- first/last seen timestamps and the "current content" marker. +-- When a URL's content changes, a new row is inserted with is_latest='yes' +-- and previous rows flip to is_latest='no'. Unchanged content just bumps +-- last_seen, giving the UI the "merged" view. +CREATE TABLE url_content +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + url TEXT REFERENCES urls(url), + content_hash TEXT REFERENCES content_snapshot(content_hash), + first_seen DATETIME, + last_seen DATETIME, + is_latest TEXT CHECK (is_latest IN ('yes', 'no')), + + CONSTRAINT url_content_unique UNIQUE (url, content_hash) +); + +-- Audit trail of changes to URL fields (classification, status, note, latest content, ...). +CREATE TABLE url_history +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + url TEXT REFERENCES urls(url), + changed_at DATETIME, + field TEXT, + old_value TEXT, + new_value TEXT, + changed_by TEXT ); + +-- Prepared for future sandbox integration; UI exposes a "request analysis" stub. +CREATE TABLE sandbox_job +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + content_hash TEXT REFERENCES content_snapshot(content_hash), + url TEXT REFERENCES urls(url), + provider TEXT, + external_id TEXT, + status TEXT CHECK (status IN ('pending', 'running', 'completed', 'failed')), + submitted_at DATETIME, + completed_at DATETIME, + report_url TEXT, + report_json TEXT, + requested_by TEXT +); + +-- ---------------------------------------------------------------------------- +-- Indexes for common lookup patterns +-- ---------------------------------------------------------------------------- +CREATE INDEX idx_url_source_lookup ON url_source(url, source, observed_at); +CREATE INDEX idx_url_source_origin ON url_source(origin_url); +CREATE INDEX idx_url_source_session ON url_source(session_hash); +CREATE INDEX idx_url_content_latest ON url_content(url, is_latest); +CREATE INDEX idx_url_content_hash ON url_content(content_hash); +CREATE INDEX idx_url_history_url ON url_history(url, changed_at); +CREATE INDEX idx_content_snapshot_sha1 ON content_snapshot(sha1); +CREATE INDEX idx_sandbox_job_status ON sandbox_job(content_hash, status); +CREATE INDEX idx_sandbox_job_url ON sandbox_job(url); +CREATE INDEX idx_discovered_urls_src ON discovered_urls(src_url); diff --git a/web/main.py b/web/main.py index 73d8626..35f9b94 100644 --- a/web/main.py +++ b/web/main.py @@ -7,9 +7,11 @@ import argparse import logging import base64 +import json +import io from datetime import datetime, timezone -from flask import Flask, jsonify, render_template, make_response, redirect, url_for +from flask import Flask, jsonify, render_template, make_response, redirect, url_for, send_file, abort from werkzeug.exceptions import BadRequestKeyError from pymisp import PyMISP, PyMISPError @@ -18,6 +20,7 @@ from common.config import Config from common.db import SQLiteWrapper from common.utils import is_valid, get_domain +from common.content_storage import load_content # Global variables page = 1 @@ -245,6 +248,7 @@ def __init__(self, url_detail): self.last_active = url_detail[15] self.last_edit = url_detail[16] self.eval_later = url_detail[17] + self.latest_content_hash = url_detail[18] self.ip = get_ip(self.url) self.src = [] # per-source observation rows for the Sources tab: @@ -252,6 +256,8 @@ def __init__(self, url_detail): self.source_rows = [] self.src_urls = [] self.contained_urls = [] + # content history rows for the Content tab (dicts; see detail()) + self.content_rows = [] def get_detail_menu(url, show=None, counts=None): @@ -297,7 +303,38 @@ def detail(): return redirect(url_for('detail', url=url)) # get url details - url_detail = URLDetail(db.execute("SELECT url, first_seen, last_seen, hash, classification, classification_reason, note, reported, occurrences, vt_stats, evaluated, file_mime_type, content_size, threat_label, status, last_active, last_edit, eval_later FROM urls WHERE url = ? LIMIT 1", (url,)).fetchone()) + url_detail = URLDetail(db.execute("SELECT url, first_seen, last_seen, hash, classification, classification_reason, note, reported, occurrences, vt_stats, evaluated, file_mime_type, content_size, threat_label, status, last_active, last_edit, eval_later, latest_content_hash FROM urls WHERE url = ? LIMIT 1", (url,)).fetchone()) + + # Content history for the Content tab (newest first), with per-URL first/last seen + snapshots = db.execute(""" + SELECT uc.content_hash, uc.first_seen, uc.last_seen, uc.is_latest, + cs.downloaded_at, cs.http_status, cs.mime_type, cs.content_size, + cs.storage_path, cs.sha1, cs.http_headers + FROM url_content uc + JOIN content_snapshot cs ON cs.content_hash = uc.content_hash + WHERE uc.url = ? + ORDER BY uc.first_seen DESC + """, (url,)).fetchall() + + prev_hash = None + for content_hash, first_seen, last_seen, is_latest, downloaded_at, http_status, mime, csize, storage_path, sha1, http_headers in snapshots: + row = { + "hash": content_hash, + "first_seen": first_seen, + "last_seen": last_seen, + "downloaded_at": downloaded_at, + "http_status": http_status, + "mime": mime, + "size": csize, + "path": storage_path, # serves as the download link target + "sha1": sha1, + "is_latest": is_latest == "yes", + "headers": json.loads(http_headers) if http_headers else None, + } + # Non-latest rows are "changed" when superseded by different (newer) content + row["changed"] = not row["is_latest"] and prev_hash is not None and prev_hash != content_hash + prev_hash = content_hash + url_detail.content_rows.append(row) url_detail.src = [row[0] for row in db.execute("SELECT source FROM url_source WHERE url = ?", (url,)).fetchall()] # per-source observation stats for the Sources tab (source, first_seen, last_seen, occurrences, derived_from) # "derived_from" is the source URL this URL was extracted from (discovered_urls), @@ -435,6 +472,35 @@ def api_url_stats(): } return make_response(jsonify(return_dict), 200) + +@app.route('/content/download', methods=['GET']) +def download_content(): + """Serve a stored content blob by its SHA-256 hash.""" + content_hash = flask.request.args.get('hash') + if not content_hash: + abort(404) + try: + data = load_content(config.content_storage_path, content_hash) + except FileNotFoundError: + abort(404) + return send_file(io.BytesIO(data), download_name=content_hash[:16], as_attachment=True) + + +@app.route('/sandbox/request', methods=['POST']) +def request_sandbox(): + """Stub: record a sandbox analysis request for a content snapshot.""" + user = get_user(flask.request.environ) + content_hash = flask.request.form.get('hash') or flask.request.args.get('hash') + url = flask.request.form.get('url') or flask.request.args.get('url') + if not content_hash: + abort(404) + with SQLiteWrapper(config.db_path) as db: + db.execute( + "INSERT INTO sandbox_job (content_hash, url, status, submitted_at, requested_by) VALUES (?, ?, 'pending', ?, ?)", + (content_hash, url, datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S'), user)) + return redirect(url_for('detail', url=url, tab='sandbox')) + + if __name__ == '__main__': app.run(host='127.0.0.1', port=5000, debug=True) diff --git a/web/static/detail.css b/web/static/detail.css index cf2f342..10c5c1c 100755 --- a/web/static/detail.css +++ b/web/static/detail.css @@ -312,4 +312,98 @@ max-height: 100px; overflow: scroll; /* margin: 10px; */ +} + +/* --- Content tab table --- */ +.detail .content-table { + width: 100%; + border-collapse: collapse; +} + +.detail .content-table th { + text-align: left; + color: #818181; + font-weight: normal; + padding: 8px 12px 8px 0; + border-bottom: 1px solid #e0e0e0; +} + +.detail .content-table td { + padding: 10px 12px 10px 0; + border-bottom: 1px solid #eee; + vertical-align: middle; +} + +.detail .content-table th.num, +.detail .content-table td.num { + text-align: right; +} + +.detail .content-table td.hash { + font-family: "Lucida Console", "Courier New", monospace; + font-size: smaller; + white-space: nowrap; +} + +.detail .content-table td.empty { + color: #818181; + text-align: center; + padding: 20px 0; +} + +/* File path download link */ +.detail .content-table a.content-download { + text-decoration: underline; + color: #00acc1; + overflow-wrap: anywhere; +} + +/* "content changed" badge + highlighted row */ +.detail .content-table tr.content-changed td { + background-color: #fff4e5; +} + +.detail .badge-changed { + display: inline-block; + padding: 2px 10px; + background-color: #f0a83c; + color: #fff; + border-radius: 12px; + font-size: small; + white-space: nowrap; +} + +/* Collapsible HTTP headers */ +.detail .content-headers-row td { + padding-top: 0; + background-color: #fafafa; +} + +.detail .content-headers summary { + cursor: pointer; + color: #818181; + font-size: small; +} + +.detail .content-headers dl { + margin: 8px 0 0 0; + font-size: small; +} + +.detail .content-headers dl div { + display: flex; + padding: 2px 0; + border-bottom: 1px solid #f0f0f0; +} + +.detail .content-headers dt { + font-weight: bold; + min-width: 220px; + color: #555; + margin-left: 0; +} + +.detail .content-headers dd { + margin: 0; + overflow-wrap: anywhere; } \ No newline at end of file diff --git a/web/templates/detail.html b/web/templates/detail.html index cf6ae26..9a5a989 100755 --- a/web/templates/detail.html +++ b/web/templates/detail.html @@ -39,7 +39,64 @@ {% endif %}
- {% if active_tab == 'sources' %} + {% if active_tab == 'content' %} + +
+ + + + + + + + + + + + + + {% for row in url.content_rows %} + + + + + + + + + + + {% if row.headers %} + + + + {% endif %} + {% else %} + + {% endfor %} + +
First seenLast seenFile pathSHA-256Mime typeSizeHTTP
{{ row.first_seen or '—' }}{{ row.last_seen or '—' }} + {% if row.path %} + {{ row.path }} + {% else %} + — + {% endif %} + {{ row.hash[:16] }}…{{ row.mime or '—' }}{{ row.size if row.size is not none else '—' }}{{ row.http_status or '—' }} + {% if row.changed %}content changed{% endif %} +
+
+ HTTP response headers +
+ {% for k, v in row.headers.items() %} +
{{ k }}
{{ v }}
+ {% endfor %} +
+
+
No content downloaded for this URL yet.
+ {% elif active_tab == 'sources' %}