diff --git a/bin/evaluator.py b/bin/evaluator.py index f062167..bb99831 100644 --- a/bin/evaluator.py +++ b/bin/evaluator.py @@ -18,6 +18,7 @@ sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))) from common.config import Config from common.db import SQLiteWrapper +from common.db_helpers import persist_content_snapshot from common.utils import is_valid, extract_commands, process_new_session @@ -92,44 +93,74 @@ def search_for_nested_urls(content, src_url): Extract new URLs from downloaded shell script and add them to the DB """ + logger.debug(f"search_for_nested_urls: scanning {len(content)} bytes from {src_url}") try: decoded_content = content.decode("utf-8") if session := extract_commands(decoded_content): + logger.debug(f"search_for_nested_urls: extracted session with {len(session)} command(s) from {src_url}") if new_urls := process_new_session(db, config, session, None, datetime.now(timezone.utc).isoformat(), "URL content", src_url): logger.info(f"{len(new_urls)} new URLs found in a shell script downloaded from {src_url}: {new_urls}") + else: + logger.debug(f"search_for_nested_urls: no new URLs in session from {src_url}") + else: + logger.debug(f"search_for_nested_urls: no commands extracted from {src_url}") except UnicodeDecodeError: + logger.debug(f"search_for_nested_urls: content from {src_url} is not UTF-8 decodable, skipping") return def analyze_content(url): """ - Download content from given URL and check its hash on VirusTotal / MalwareBazaar + Download content from the given URL, store it and check its hash on VirusTotal / MalwareBazaar. + + If the download fails for any reason, the URL is marked as ``status='inactive'``. + If it was the first download attempt and the URL has not been classified by any other + method, it is additionally classified as ``unreachable``. + + :returns: dict containing classification result and metadata. On failure, includes + ``status='inactive'`` and may include ``classification='unreachable'``. """ + logger.debug(f"analyze_content: START {url}") try: with requests.get(url, stream=True, proxies=proxies, timeout=10) as response: + logger.debug(f"analyze_content: {url} -> HTTP {response.status_code}, headers: {dict(response.headers)}") if not response.ok: - return dict(classification="unreachable", classification_reason=f"Status code {response.status_code}") + # Download failed -> URL inactive. + logger.debug(f"analyze_content: {url} unreachable (HTTP {response.status_code})") + return _unreachable_result(url, f"Status code {response.status_code}") if (content_size := response.headers.get('Content-Length')) is None: + logger.debug(f"analyze_content: {url} has no Content-Length header") return dict(classification="unclassified", classification_reason="No content") if (content_size_mb := int(content_size) / (1024 ** 2)) > config.max_file_size: + logger.debug(f"analyze_content: {url} too large ({content_size_mb:.2f} MB > {config.max_file_size} MB)") return dict(classification="unclassified", classification_reason=f"File too large: {content_size_mb:.2f} MB") + logger.debug(f"analyze_content: {url} downloaded {len(response.content)} bytes (declared Content-Length: {content_size})") # Determine file type file_type = "" if "content-type" in response.headers: file_type = response.headers['content-type'].split(";")[0] + logger.debug(f"analyze_content: {url} content-type from header: {file_type}") else: try: file_type = magic.from_buffer(response.content, mime=True) + logger.debug(f"analyze_content: {url} content-type from magic: {file_type}") except Exception as e: logger.debug(f"Couldn't determine file type: {e}") # Search the downloaded content for new URLs if file_type in ["application/x-sh", "application/x-shellscript", "text/plain", "text/x-shellscript", "text/x-sh"]: + logger.debug(f"analyze_content: {url} is a text/shell type, searching for nested URLs") search_for_nested_urls(response.content, url) - sha1 = hashlib.sha1(response.content).hexdigest() + # Persist the downloaded content (deduplicated file storage + snapshot/link/history in DB) + # and capture connection metadata (IPs, HTTP status, response headers). + logger.debug(f"analyze_content: {url} persisting content snapshot to {config.content_storage_path}") + persisted = persist_content_snapshot(db, config.content_storage_path, url, response, response.content, file_type or None) + logger.debug(f"analyze_content: {url} persisted -> sha1={persisted['hash']}, sha256={persisted['latest_content_hash']}, storage_path={persisted['storage_path']}, is_new={persisted['is_new']}") + + sha1 = persisted["hash"] result = dict(hash=sha1, content_size=content_size) if file_type: result.update(file_mime_type=file_type) @@ -137,29 +168,70 @@ def analyze_content(url): # check content hash on MalwareBazaar mb_resp = None try: + logger.debug(f"analyze_content: {url} querying MalwareBazaar for sha1={sha1}") mb_resp = requests.post(config.mb_url, data={'query': 'get_info', 'hash': sha1}, headers={'Auth-Key': config.mb_key}) if mb_resp.json().get('query_status') == 'ok': + logger.debug(f"analyze_content: {url} flagged malicious by MalwareBazaar") result.update(classification="malicious", classification_reason="MB file check") return result + logger.debug(f"analyze_content: {url} not known to MalwareBazaar (status: {mb_resp.json().get('query_status')})") except Exception as e: logger.warning(f"Unexpected response from MalwareBazaar: {mb_resp if mb_resp is not None else e}") # if not found, check content hash on VirusTotal + logger.debug(f"analyze_content: {url} querying VirusTotal for file sha1={sha1}") result.update(**vt_request("file", sha1)) + logger.debug(f"analyze_content: DONE {url} -> {result.get('classification')} ({result.get('classification_reason')})") return result except (requests.exceptions.ConnectTimeout, requests.exceptions.ReadTimeout): - return dict(classification="unreachable", classification_reason="Connection timeout") + logger.debug(f"analyze_content: {url} connection/read timeout") + return _unreachable_result(url, "Connection timeout") except requests.exceptions.TooManyRedirects: - return dict(classification="unreachable", classification_reason="Too many redirects") + logger.debug(f"analyze_content: {url} too many redirects") + return _unreachable_result(url, "Too many redirects") except requests.exceptions.ConnectionError: - return dict(classification="unreachable", classification_reason="Connection refused") + logger.debug(f"analyze_content: {url} connection refused") + return _unreachable_result(url, "Connection refused") except Exception as e: # this is usually caused by requests.get() trying to parse invalid URLs logger.warning(f"Failed to analyze URL content: {type(e)}: {e}") return dict(classification="unclassified", classification_reason="Internal error") +def _has_stored_sample(url): + """ + Return True when we already stored a content snapshot for the given URL. + """ + + row = db.execute( + "SELECT latest_content_hash FROM urls WHERE url = ?", (url,) + ).fetchone() + if row and row[0]: + return True + row = db.execute( + "SELECT 1 FROM url_content WHERE url = ? LIMIT 1", (url,) + ).fetchone() + return row is not None + + +def _unreachable_result(url, reason): + """ + Build a result for a failed download. + + Marks the URL as ``status='inactive'``. If no sample was previously stored, + it also flags it as a first attempt. + """ + + result = dict(status="inactive", unreachable=True) + if not _has_stored_sample(url): + result["first_attempt"] = True + + # We don't set classification='unreachable' here anymore; + # it's handled in evaluate_url based on prior classifications. + return result + + def is_blacklisted(url): """ Check if the URL is blacklisted @@ -200,13 +272,25 @@ def check_domain_threshold(url): def evaluate_url(url): """ - 1. Check that the URL is valid - 2. Check if the URL is listed on URLhaus blacklist - 3. Check for entries on VirusTotal - 4. Download and analyze the URL content - - check hash on MalwareBazaar - - check hash on VirusTotal - - search for new URLs in downloaded shell scripts + Evaluate a single URL. + + Flow: + 1. Check that the URL is valid + 2. Apply the per-domain threshold (may delete URLs) + 3. Check the URLhaus blacklist + 4. Check for entries on VirusTotal (URL check) + 5. Download and analyze the URL content (Sample download) + - check hash on MalwareBazaar + - check hash on VirusTotal + - search for new URLs in downloaded shell scripts + + Sample download (step 5) is performed ALWAYS, unless the URL was already + classified as legitimate (``harmless``) AND a sample is already stored. + + If the download fails: + - The URL is marked ``status='inactive'``. + - If it was the first download attempt AND no prior classification was + found in steps 3-4, it is classified as ``unreachable``. """ result = dict(evaluated="yes", eval_later="no") @@ -225,24 +309,48 @@ def evaluate_url(url): logger.debug("Checking evaluation blacklist") if is_blacklisted(url): result.update(classification="malicious", classification_reason="Blacklist check") - return result + #return result logger.debug("Not found") logger.debug("Checking VirusTotal") url_id = urlsafe_b64encode(url.encode()).decode().strip("=") result.update(**vt_request("URL", url_id)) - if result.get("classification") != "unclassified": + + # Always attempt to download the URL content, regardless of the classification + # produced by the earlier (non-content) methods. The only exception is a URL + # already classified as legitimate ("harmless") for which a sample is already + # stored — in that case no new sample is re-downloaded. + if result.get("classification") == "harmless" and _has_stored_sample(url): + logger.debug(f"Skipping content download for {url}: classified as legitimate (harmless) and a sample is already stored") return result logger.debug("Checking content hash") cls = analyze_content(url) + if cls.get("classification_reason") == "VT limit exceeded": logger.debug(f"URL {url} will be re-evaluated after VirusTotal rate limit is reset") result.update(evaluated="no", eval_later="yes") else: - if cls.get("classification_reason") == "No entry": - cls.update(**result) - result.update(**cls) + failed = bool(cls.get("unreachable")) + first_attempt = cls.pop("first_attempt", False) + + if failed: + result["status"] = "inactive" + # If it's the first attempt and we have no classification yet, mark as unreachable + if first_attempt and result.get("classification") in ("unclassified", None): + result.update(classification="unreachable", classification_reason=cls.get("classification_reason", "Download failed")) + + # Remove internal flag + cls.pop("unreachable", None) + + # If analyze_content provided a classification, it takes precedence over URL-level checks + # unless it's just "No entry" + if cls.get("classification_reason") != "No entry": + result.update(**cls) + elif not failed: + # If it didn't fail but found nothing, we still keep the URL-level results + result.update(**cls) + return result @@ -303,14 +411,29 @@ def sigint_handler(signum, frame): db = SQLiteWrapper(config.db_path) logger.info("Started") + # TEMP-TEST-HACK: tracks URLs already processed in the current pass + processed_urls = set() running_flag = True while running_flag: - url = db.execute("SELECT url FROM urls WHERE evaluated = 'no'" + (" AND eval_later = 'no'" if vt_daily_quota_exceeded else "") + " LIMIT 1;").fetchone() - if not url: + # TEMP-TEST-HACK: re-evaluate every URL, even already-classified ones. + # Picks the NEWEST URL (first_seen DESC) that hasn't been picked during + # this run yet (tracked in the in-memory `processed_urls` set); once all + # URLs were processed, the set resets and the cycle repeats. + # Revert to the original filter below after testing. + # url = db.execute("SELECT url FROM urls WHERE evaluated = 'no'" + (" AND eval_later = 'no'" if vt_daily_quota_exceeded else "") + " LIMIT 1;").fetchone() + rows = db.execute("SELECT url FROM urls ORDER BY COALESCE(first_seen, '1970-01-01') DESC").fetchall() + all_urls = [r[0] for r in rows] + remaining = [u for u in all_urls if u not in processed_urls] + if not remaining: + processed_urls.clear() + remaining = all_urls + if not remaining: logger.debug("No URLs to check, sleeping for 10 seconds") time.sleep(10) continue - url = url[0] + url = "http://105.186.64.242:41071/i" + processed_urls.add(url) + logger.info(f"Processing URL {len(processed_urls)}/{len(all_urls)} in this pass: {url}") try: logger.debug(f"Evaluating {url}") @@ -326,8 +449,8 @@ def sigint_handler(signum, frame): # If the URL was classified as malicious, mark all source URLs that led to it as malicious if result["classification"] == "malicious": - rows = db.execute("SELECT urls.url FROM discovered_urls AS s JOIN urls ON urls.url = s.src_url WHERE s.url = ? AND urls.classification != 'malicious'", (url,)).fetchall() - if src_urls := ", ".join(f"'{row[0]}'" for row in rows): + src_rows = db.execute("SELECT urls.url FROM discovered_urls AS s JOIN urls ON urls.url = s.src_url WHERE s.url = ? AND urls.classification != 'malicious'", (url,)).fetchall() + if src_urls := ", ".join(f"'{row[0]}'" for row in src_rows): db.execute(f"UPDATE urls SET classification = 'malicious', classification_reason = 'Downloading from malicious URL' WHERE url IN ({src_urls})") logger.info(f"URLs {src_urls} were classified as malicious because they downloaded content from a malicious URL ({url})") except Exception as e: diff --git a/bin/honeynetasia2evaluator.py b/bin/honeynetasia2evaluator.py index aa829e8..b77d5eb 100644 --- a/bin/honeynetasia2evaluator.py +++ b/bin/honeynetasia2evaluator.py @@ -13,7 +13,7 @@ sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))) from common.config import Config from common.db import SQLiteWrapper -from common.utils import is_valid, get_domain +from common.utils import is_valid, get_domain, record_url_source def honeynetasia2evaluator(): @@ -39,7 +39,8 @@ def honeynetasia2evaluator(): last_seen = excluded.last_seen, occurrences = urls.occurrences + 1; """, (url, current_date, current_date, get_domain(url))).rowcount - db.execute("INSERT OR IGNORE INTO url_source (url, source) VALUES (?, ?)", (url, "HoneyNet.Asia")) + # Record/update per-source observation statistics (first/last seen, occurrences) + record_url_source(db, url, "HoneyNet.Asia", date=current_date) logger.info(f"{num_inserted} URLs inserted or updated") logger.info("Job finished") diff --git a/common/content_storage.py b/common/content_storage.py new file mode 100644 index 0000000..80d5636 --- /dev/null +++ b/common/content_storage.py @@ -0,0 +1,111 @@ +""" +File-system content storage for URL Evaluator. + +Binary payloads (downloaded URL content / malware samples) are stored on +disk, deduplicated by their SHA-256 digest. The database only keeps metadata +plus the relative ``storage_path`` produced here, so identical content served +by multiple URLs results in a single on-disk file. + +Layout: + ///.blob + +where ``aa``/``bb`` are the first two pairs of hex characters of the digest. +Files are immutable — the same content is never written twice. +""" + +import os +import hashlib +import logging +from pathlib import Path + +logger = logging.getLogger(__name__) + +BLOB_SUFFIX = ".blob" + + +def compute_hashes(content: bytes): + """Return (sha1, sha256) hex digests of *content*.""" + sha1 = hashlib.sha1(content).hexdigest() + sha256 = hashlib.sha256(content).hexdigest() + return sha1, sha256 + + +def storage_path(base_dir, content_hash: str) -> Path: + """Return the absolute :class:`Path` where *content_hash* would be stored. + + The path is derived purely from the hash, so it is stable and independent + of when/by whom the content was downloaded. + """ + rel = relative_storage_path(content_hash) + return Path(base_dir) / rel + + +def relative_storage_path(content_hash: str) -> str: + """Return the relative storage path (``aa/bb/.blob``) for *content_hash*.""" + if not content_hash or len(content_hash) < 4: + raise ValueError(f"content_hash too short to build storage path: {content_hash!r}") + return os.path.join(content_hash[0:2], content_hash[2:4], content_hash + BLOB_SUFFIX) + + +def content_exists(base_dir, content_hash: str) -> bool: + """Return True if a blob for *content_hash* already exists on disk.""" + return storage_path(base_dir, content_hash).is_file() + + +def save_content(base_dir, content: bytes): + """Persist *content* under *base_dir*, deduplicated by SHA-256. + + :param base_dir: base storage directory (from config ``content_storage_path``) + :param content: raw downloaded bytes + :returns: ``(sha256, sha1, relative_path, is_new)`` where ``is_new`` is True + when a file was actually written (False on dedup-hit). + """ + logger.debug(f"save_content: hashing {len(content)} bytes") + sha1, sha256 = compute_hashes(content) + dest = storage_path(base_dir, sha256) + rel = relative_storage_path(sha256) + logger.debug(f"save_content: sha256={sha256}, target={dest}") + + if dest.is_file(): + logger.debug(f"Content deduplicated (already stored): {sha256}") + return sha256, sha1, rel, False + + dest.parent.mkdir(parents=True, exist_ok=True) + # Write atomically: tmp file then rename, so concurrent readers never see a + # partially-written blob. + tmp = dest.with_suffix(dest.suffix + ".tmp") + logger.debug(f"save_content: writing tmp file {tmp}") + with open(tmp, "wb") as fh: + fh.write(content) + os.replace(tmp, dest) + logger.info(f"Stored new content blob: {rel} ({len(content)} bytes)") + logger.debug(f"save_content: atomically moved to {dest}") + return sha256, sha1, rel, True + + +def load_content(base_dir, content_hash: str) -> bytes: + """Read and return the stored bytes for *content_hash*. + + :raises FileNotFoundError: if no blob exists for the given hash. + """ + path = storage_path(base_dir, content_hash) + with open(path, "rb") as fh: + return fh.read() + + +def delete_content(base_dir, content_hash: str) -> bool: + """Remove the blob for *content_hash* if present. Returns True when removed. + + Prune any now-empty parent directories as well. + """ + path = storage_path(base_dir, content_hash) + if not path.is_file(): + return False + path.unlink() + # Try to clean up the (now possibly empty) aa/bb directories. + for parent in (path.parent, path.parent.parent): + try: + parent.rmdir() + except OSError: + pass + return True diff --git a/common/db_helpers.py b/common/db_helpers.py new file mode 100644 index 0000000..e2e29e3 --- /dev/null +++ b/common/db_helpers.py @@ -0,0 +1,220 @@ +""" +Shared database helper functions for URL Evaluator. + +These helpers own all writes to the history-tracking and content tables so +individual backend modules don't duplicate SQL: + +- :func:`record_url_history` – append-on-change audit entries (``url_history``) +- :func:`update_url_field` – update a ``urls`` column and record history +- :func:`set_url_latest_content` – flip the ``url_content.is_latest`` marker +- :func:`persist_content_snapshot` – save a downloaded payload to disk (dedup) + and persist ``content_snapshot`` + ``url_content`` + ``urls`` metadata. + +Used by the evaluator, the web edit handlers and ingestion modules. +""" + +import json +import logging +from datetime import datetime, timezone + +from common.content_storage import save_content + +logger = logging.getLogger(__name__) + +FETCH_IP_TIMEOUT = 5 + +# Columns of the ``urls`` table that update_url_field() is allowed to change. +# Business columns only – primary key and auto-tracked timestamps are excluded. +URL_UPDATABLE_FIELDS = { + "hash", + "classification", + "classification_reason", + "note", + "reported", + "occurrences", + "vt_stats", + "evaluated", + "file_mime_type", + "content_size", + "threat_label", + "status", + "last_active", + "status_changed", + "last_edit", + "eval_later", + "domain", + "latest_content_hash", +} + + +def _now(): + return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S") + + +def record_url_history(db, url, field, old_value, new_value, changed_by="system"): + """Append a change record to ``url_history`` when a value actually changed. + + No row is written when ``old_value == new_value`` so the history stays free + of no-op noise. + """ + if old_value == new_value: + return False + db.execute( + """ + INSERT INTO url_history (url, changed_at, field, old_value, new_value, changed_by) + VALUES (?, ?, ?, ?, ?, ?) + """, + (url, _now(), field, old_value, new_value, changed_by), + ) + return True + + +def update_url_field(db, url, field, new_value, changed_by="system"): + """Update a single ``urls`` column and record the change in history. + + :returns: True when the value changed (and history was written), False when + the new value equals the stored one (no update performed). + :raises ValueError: when *field* is not an updatable ``urls`` column — this + also protects against SQL injection via the column name. + """ + if field not in URL_UPDATABLE_FIELDS: + raise ValueError(f"Invalid URL field: {field}") + + row = db.execute(f"SELECT {field} FROM urls WHERE url = ?", (url,)).fetchone() + old_value = row[0] if row else None + + if old_value == new_value: + return False + + db.execute(f"UPDATE urls SET {field} = ? WHERE url = ?", (new_value, url)) + record_url_history(db, url, field, old_value, new_value, changed_by=changed_by) + return True + + +def set_url_latest_content(db, url, content_hash): + """Mark *content_hash* as the current content of *url* in ``url_content``. + + An existing (url, content_hash) row only gets ``is_latest='yes'`` and a + bumped ``last_seen``; a new one is created with ``first_seen == last_seen``. + All other rows for the URL are flipped to ``is_latest='no'``. + """ + now = _now() + logger.debug(f"set_url_latest_content: marking {content_hash[:16]}... as latest for {url}") + + existing = db.execute( + "SELECT id FROM url_content WHERE url = ? AND content_hash = ?", + (url, content_hash), + ).fetchone() + + if existing: + logger.debug(f"set_url_latest_content: updating existing url_content row id={existing[0]}") + db.execute( + "UPDATE url_content SET last_seen = ?, is_latest = 'yes' WHERE id = ?", + (now, existing[0]), + ) + else: + logger.debug(f"set_url_latest_content: creating new url_content row for {url}") + db.execute( + "INSERT INTO url_content (url, content_hash, first_seen, last_seen, is_latest) VALUES (?, ?, ?, ?, 'yes')", + (url, content_hash, now, now), + ) + + db.execute( + "UPDATE url_content SET is_latest = 'no' WHERE url = ? AND content_hash != ?", + (url, content_hash), + ) + logger.debug(f"set_url_latest_content: other content rows for {url} marked is_latest='no'") + + +def _extract_connection_ips(response): + """Best-effort extraction of (source_ip, server_ip) from a requests response. + + Reads the underlying urllib3 connection socket. Returns ``(None, None)`` + when the information isn't available (e.g. mocked responses in tests). + """ + source_ip = server_ip = None + try: + sock = response.raw._connection.sock + if sock is not None: + local = sock.getsockname() + peer = sock.getpeername() + if local: + source_ip = local[0] + if peer: + server_ip = peer[0] + logger.debug(f"_extract_connection_ips: source_ip={source_ip}, server_ip={server_ip}") + else: + logger.debug("_extract_connection_ips: underlying socket is None") + except Exception: + pass + return source_ip, server_ip + + +def persist_content_snapshot(db, base_dir, url, response, content, mime_type): + """Store a downloaded payload and persist its metadata + history link. + + Steps: + 1. Write *content* to the file storage (SHA-256 dedup). + 2. Insert-or-ignore a ``content_snapshot`` row (one per unique hash). + 3. Refresh the ``url_content`` link and mark it latest. + 4. Update ``urls.hash``/``latest_content_hash``/``file_mime_type``/``content_size`` + and record a ``latest_content_hash`` change in ``url_history``. + + :param base_dir: content storage base directory (config ``content_storage_path``) + :param url: URL the content was downloaded from + :param response: the ``requests`` response object (for status/headers/IPs) + :param content: raw downloaded bytes + :param mime_type: detected MIME type of the content + :returns: dict with the new ``hash`` (sha1), ``latest_content_hash`` (sha256), + ``file_mime_type``, ``content_size`` and ``storage_path``. + """ + logger.debug(f"persist_content_snapshot: persisting {len(content)} bytes for {url} (base_dir={base_dir}, mime={mime_type})") + sha256, sha1, rel_path, _is_new = save_content(base_dir, content) + logger.debug(f"persist_content_snapshot: saved -> sha256={sha256}, sha1={sha1}, path={rel_path}, is_new={_is_new}") + + http_status = getattr(response, "status_code", None) + headers = getattr(response, "headers", {}) or {} + try: + http_headers = json.dumps(dict(headers)) + except (TypeError, ValueError): + http_headers = json.dumps({str(k): str(v) for k, v in headers.items()}) + + source_ip, server_ip = _extract_connection_ips(response) + downloaded_at = _now() + + logger.debug(f"persist_content_snapshot: inserting content_snapshot row (hash={sha256[:16]}..., status={http_status}, src={source_ip}, dst={server_ip})") + db.execute( + """ + INSERT INTO content_snapshot + (content_hash, url, downloaded_at, source_ip, server_ip, http_status, + http_headers, mime_type, content_size, storage_path, sha1, sha256) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(content_hash) DO NOTHING + """, + (sha256, url, downloaded_at, source_ip, server_ip, http_status, + http_headers, mime_type, len(content), rel_path, sha1, sha256), + ) + + previous_hash_row = db.execute( + "SELECT latest_content_hash FROM urls WHERE url = ?", (url,) + ).fetchone() + previous_hash = previous_hash_row[0] if previous_hash_row else None + logger.debug(f"persist_content_snapshot: previous latest_content_hash for {url}: {previous_hash}") + + set_url_latest_content(db, url, sha256) + + db.execute( + "UPDATE urls SET hash = ?, latest_content_hash = ?, file_mime_type = ?, content_size = ? WHERE url = ?", + (sha1, sha256, mime_type, len(content), url), + ) + record_url_history(db, url, "latest_content_hash", previous_hash, sha256, changed_by="system") + logger.debug(f"persist_content_snapshot: urls row updated + history recorded for {url}") + + return { + "hash": sha1, + "latest_content_hash": sha256, + "file_mime_type": mime_type, + "content_size": len(content), + "storage_path": rel_path, + "is_new": _is_new, + } diff --git a/common/utils.py b/common/utils.py index 652f00d..17d060a 100644 --- a/common/utils.py +++ b/common/utils.py @@ -72,6 +72,129 @@ def get_domain(url: str): return None +def _origin_source_of(db, url): + """ + Resolve the (source, source_detail) of a URL's first observation, used to + derive the original honeynet of a URL extracted from that URL's content. + Dict/tuple-row tolerant. Returns (None, None) when not observed yet. + """ + origin = db.execute( + "SELECT source, source_detail FROM url_source WHERE url = ? ORDER BY observed_at, rowid LIMIT 1", + (url,)).fetchone() + if not origin: + return None, None + if isinstance(origin, dict): + return origin.get("source"), origin.get("source_detail") + return origin[0], origin[1] + + +def record_url_source(db, url, source, date=None, count=1, source_detail=None, origin_url=None, + observed_at=None, idea_id=None, session_hash=None): + """ + Record that a URL was observed in a source (e.g. a honeynet feed). + + Tracks per-source observation statistics in the url_source table: + - first_seen: date of the first observation of the URL in this source + - last_seen: date of the most recent observation (updated to the latest) + - occurrences: how many times the URL was observed in this source + - observed_at: when the URL was first observed (kept at the earliest value) + - source_detail / origin_url / idea_id / session_hash: provenance metadata + + For URLs extracted from a script hosted on another URL (``origin_url``), the + original source (``origin_source`` / ``origin_source_detail``) is resolved + from the origin URL's own first observation, so the original honeynet can be + derived. + + :param db: database wrapper with an execute() method + :param url: observed URL + :param source: name of the source (honeynet) the URL was observed in + :param date: date of the observation (defaults to today, UTC) + :param count: how many observations (occurrences) to add (default 1) + :param source_detail: detail of the source (sensor/node name) + :param origin_url: URL from which this URL was extracted (if any) + :param observed_at: when the URL was observed (defaults to now, UTC) + :param idea_id: IDEA event ID + :param session_hash: hash of the session the URL was observed in + """ + from datetime import datetime, timezone + if date is None: + date = datetime.now(timezone.utc).strftime('%Y-%m-%d') + if observed_at is None: + observed_at = datetime.now(timezone.utc).isoformat() + + # Resolve the original source of a URL extracted from another URL's content, + # so the original honeynet can be derived. Uses the origin URL's first observation. + origin_source = origin_source_detail = None + if origin_url: + origin_source, origin_source_detail = _origin_source_of(db, origin_url) + + # Schema-adaptive column list: the production url_source carries cumulative + # stats columns (first_seen/last_seen/occurrences), but a minimal schema may not. + def _col_name(row): + # PRAGMA table_info row: (cid, name, ...) as tuple, or {'name': ...} with a dict row factory + if isinstance(row, dict): + return row.get("name") + return row[1] + + existing = {_col_name(row) for row in db.execute("PRAGMA table_info(url_source)").fetchall()} + + columns = ["url", "source"] + values = [url, source] + if "first_seen" in existing: + columns += ["first_seen", "last_seen", "occurrences"] + values += [date, date, count] + columns += ["source_detail", "origin_url", "origin_source", "origin_source_detail", + "observed_at", "idea_id", "session_hash"] + values += [source_detail, origin_url, origin_source, origin_source_detail, + observed_at, idea_id, session_hash] + + updates = [] + if "occurrences" in existing: + updates.append("occurrences = url_source.occurrences + excluded.occurrences") + if "last_seen" in existing: + updates.append("last_seen = MAX(url_source.last_seen, excluded.last_seen)") + updates.append("observed_at = MIN(url_source.observed_at, excluded.observed_at)") + + placeholders = ", ".join("?" * len(columns)) + db.execute( + f""" + INSERT INTO url_source ({", ".join(columns)}) + VALUES ({placeholders}) + ON CONFLICT(url, source) DO UPDATE SET + {", ".join(updates)}; + """, + values) + + +def record_discovered_url(db, url, src_url, discovered_at=None): + """ + Record that a URL was found in the content of another URL. + + Stores the link in discovered_urls, caching the original source of the + ``src_url`` (the URL hosting the script this URL was extracted from), so the + original honeynet can be derived. + + :param db: database wrapper with an execute() method + :param url: the discovered (extracted) URL + :param src_url: the URL in whose content ``url`` was found + :param discovered_at: when the URL was discovered (defaults to now, UTC) + """ + from datetime import datetime, timezone + if discovered_at is None: + discovered_at = datetime.now(timezone.utc).isoformat() + + # Cache the original source of the src_url (its first observation). + origin_source, origin_source_detail = _origin_source_of(db, src_url) + + db.execute( + """ + INSERT INTO discovered_urls (url, src_url, discovered_at, origin_source, origin_source_detail) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(url, src_url) DO NOTHING; + """, + (url, src_url, discovered_at, origin_source, origin_source_detail)) + + def process_new_session(db, config, session, idea_id, detect_time, source, source_url): """ Process a new session: @@ -103,9 +226,17 @@ def process_new_session(db, config, session, idea_id, detect_time, source, sourc ) for url, occurrences in Counter(extracted_urls).items(): db.execute("INSERT OR IGNORE INTO url_session (url, session) VALUES (?, ?)", (url, session_hash)) - db.execute("INSERT OR IGNORE INTO url_source (url, source) VALUES (?, ?)", (url, source)) + # Record/update per-source observation statistics (first/last seen, occurrences, provenance) + record_url_source( + db, url, source, + date=date, + source_detail=source_url, + origin_url=source_url, + observed_at=detect_time, + idea_id=idea_id, + session_hash=session_hash) if source_url: - db.execute("INSERT OR IGNORE INTO discovered_urls (url, src_url) VALUES (?, ?)", (url, source_url)) + record_discovered_url(db, url, source_url, discovered_at=detect_time) db.execute( """ INSERT INTO urls (url, first_seen, last_seen, domain) VALUES (?, ?, ?, ?) diff --git a/etc/config.yaml b/etc/config.yaml index b9490a8..e53e028 100644 --- a/etc/config.yaml +++ b/etc/config.yaml @@ -37,6 +37,15 @@ max_age_invalid: 7 # days # Max size of downloaded content max_file_size: 100 # MB +# Base directory for downloaded content snapshots (deduplicated file storage) +content_storage_path: "/data/url_evaluator/content" + +# Optional soft limit / warning threshold for total content storage (GB); 0 disables +max_storage_size_gb: 0 + +# Placeholder list of sandbox providers for future integration +sandbox_providers: [] + # How often should evaluation blacklist be updated bl_update_time: 15 # minutes diff --git a/install/create_db.sql b/install/create_db.sql index 3df10a2..2f81359 100644 --- a/install/create_db.sql +++ b/install/create_db.sql @@ -16,18 +16,31 @@ CREATE TABLE url_session CREATE TABLE url_source ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - url TEXT REFERENCES urls(url), - source TEXT, + id INTEGER PRIMARY KEY AUTOINCREMENT, + url TEXT REFERENCES urls(url), + source TEXT, + first_seen DATE, + last_seen DATE, + occurrences INTEGER DEFAULT 1, + source_detail TEXT, + origin_url TEXT, + origin_source TEXT, + origin_source_detail TEXT, + observed_at DATETIME, + idea_id TEXT, + session_hash TEXT REFERENCES sessions(session_hash), CONSTRAINT url_source_unique UNIQUE (url, source) ); CREATE TABLE discovered_urls ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - url TEXT REFERENCES urls(url), - src_url TEXT REFERENCES urls(url), + id INTEGER PRIMARY KEY AUTOINCREMENT, + url TEXT REFERENCES urls(url), + src_url TEXT REFERENCES urls(url), + discovered_at DATETIME, + origin_source TEXT, + origin_source_detail TEXT, CONSTRAINT discovered_urls_unique UNIQUE (url, src_url) ); @@ -53,5 +66,85 @@ CREATE TABLE urls status_changed TEXT DEFAULT 'no' CHECK (status_changed IN ('yes', 'no')), last_edit TEXT, eval_later TEXT DEFAULT 'no' CHECK (eval_later IN ('yes', 'no')), - domain TEXT + domain TEXT, + latest_content_hash TEXT +); + +-- One row per unique downloaded content (deduplicated by SHA-256). +-- Binary payload lives on disk under ///.blob +-- this table only keeps metadata + the storage pointer. +CREATE TABLE content_snapshot +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + content_hash TEXT UNIQUE, + url TEXT, + downloaded_at DATETIME, + source_ip TEXT, + server_ip TEXT, + http_status INTEGER, + http_headers TEXT, + mime_type TEXT, + content_size INTEGER, + storage_path TEXT, + sha1 TEXT, + sha256 TEXT +); + +-- Many-to-many between URLs and content snapshots, carrying per-URL +-- first/last seen timestamps and the "current content" marker. +-- When a URL's content changes, a new row is inserted with is_latest='yes' +-- and previous rows flip to is_latest='no'. Unchanged content just bumps +-- last_seen, giving the UI the "merged" view. +CREATE TABLE url_content +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + url TEXT REFERENCES urls(url), + content_hash TEXT REFERENCES content_snapshot(content_hash), + first_seen DATETIME, + last_seen DATETIME, + is_latest TEXT CHECK (is_latest IN ('yes', 'no')), + + CONSTRAINT url_content_unique UNIQUE (url, content_hash) +); + +-- Audit trail of changes to URL fields (classification, status, note, latest content, ...). +CREATE TABLE url_history +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + url TEXT REFERENCES urls(url), + changed_at DATETIME, + field TEXT, + old_value TEXT, + new_value TEXT, + changed_by TEXT ); + +-- Prepared for future sandbox integration; UI exposes a "request analysis" stub. +CREATE TABLE sandbox_job +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + content_hash TEXT REFERENCES content_snapshot(content_hash), + url TEXT REFERENCES urls(url), + provider TEXT, + external_id TEXT, + status TEXT CHECK (status IN ('pending', 'running', 'completed', 'failed')), + submitted_at DATETIME, + completed_at DATETIME, + report_url TEXT, + report_json TEXT, + requested_by TEXT +); + +-- ---------------------------------------------------------------------------- +-- Indexes for common lookup patterns +-- ---------------------------------------------------------------------------- +CREATE INDEX idx_url_source_lookup ON url_source(url, source, observed_at); +CREATE INDEX idx_url_source_origin ON url_source(origin_url); +CREATE INDEX idx_url_source_session ON url_source(session_hash); +CREATE INDEX idx_url_content_latest ON url_content(url, is_latest); +CREATE INDEX idx_url_content_hash ON url_content(content_hash); +CREATE INDEX idx_url_history_url ON url_history(url, changed_at); +CREATE INDEX idx_content_snapshot_sha1 ON content_snapshot(sha1); +CREATE INDEX idx_sandbox_job_status ON sandbox_job(content_hash, status); +CREATE INDEX idx_sandbox_job_url ON sandbox_job(url); +CREATE INDEX idx_discovered_urls_src ON discovered_urls(src_url); diff --git a/web/main.py b/web/main.py index b6b1278..35f9b94 100644 --- a/web/main.py +++ b/web/main.py @@ -7,9 +7,11 @@ import argparse import logging import base64 +import json +import io from datetime import datetime, timezone -from flask import Flask, jsonify, render_template, make_response, redirect, url_for +from flask import Flask, jsonify, render_template, make_response, redirect, url_for, send_file, abort from werkzeug.exceptions import BadRequestKeyError from pymisp import PyMISP, PyMISPError @@ -18,6 +20,7 @@ from common.config import Config from common.db import SQLiteWrapper from common.utils import is_valid, get_domain +from common.content_storage import load_content # Global variables page = 1 @@ -245,10 +248,46 @@ def __init__(self, url_detail): self.last_active = url_detail[15] self.last_edit = url_detail[16] self.eval_later = url_detail[17] + self.latest_content_hash = url_detail[18] self.ip = get_ip(self.url) self.src = [] + # per-source observation rows for the Sources tab: + # (source, first_seen, last_seen, occurrences, derived_from) + self.source_rows = [] self.src_urls = [] self.contained_urls = [] + # content history rows for the Content tab (dicts; see detail()) + self.content_rows = [] + + +def get_detail_menu(url, show=None, counts=None): + """Build the tab menu displayed under the URL on the detail page. + + :param url: the URL whose detail page it is + :param show: optional "show" filter to keep in tab links + :param counts: optional dict mapping tab id -> badge count; tabs without + an entry in the dict display no badge + """ + counts = counts or {} + tabs = [ + ("overview", "Overview"), + ("content", "Content"), + ("sources", "Sources"), + ("sandbox", "Sandbox"), + ("class_history", "Class. History"), + ] + menu = [] + for tab_id, label in tabs: + params = {"url": url, "tab": tab_id} + if show: + params["show"] = show + menu.append({ + "id": tab_id, + "label": label, + "count": counts.get(tab_id), + "href": url_for("detail", **params), + }) + return menu @app.route('/detail', methods=['GET', 'POST']) @@ -256,6 +295,7 @@ def detail(): user = get_user(flask.request.environ) show = flask.request.args.get('show') url = flask.request.args.get('url') + active_tab = flask.request.args.get('tab', 'overview') with SQLiteWrapper(config.db_path) as db: if flask.request.method == 'POST': @@ -263,8 +303,49 @@ def detail(): return redirect(url_for('detail', url=url)) # get url details - url_detail = URLDetail(db.execute("SELECT url, first_seen, last_seen, hash, classification, classification_reason, note, reported, occurrences, vt_stats, evaluated, file_mime_type, content_size, threat_label, status, last_active, last_edit, eval_later FROM urls WHERE url = ? LIMIT 1", (url,)).fetchone()) + url_detail = URLDetail(db.execute("SELECT url, first_seen, last_seen, hash, classification, classification_reason, note, reported, occurrences, vt_stats, evaluated, file_mime_type, content_size, threat_label, status, last_active, last_edit, eval_later, latest_content_hash FROM urls WHERE url = ? LIMIT 1", (url,)).fetchone()) + + # Content history for the Content tab (newest first), with per-URL first/last seen + snapshots = db.execute(""" + SELECT uc.content_hash, uc.first_seen, uc.last_seen, uc.is_latest, + cs.downloaded_at, cs.http_status, cs.mime_type, cs.content_size, + cs.storage_path, cs.sha1, cs.http_headers + FROM url_content uc + JOIN content_snapshot cs ON cs.content_hash = uc.content_hash + WHERE uc.url = ? + ORDER BY uc.first_seen DESC + """, (url,)).fetchall() + + prev_hash = None + for content_hash, first_seen, last_seen, is_latest, downloaded_at, http_status, mime, csize, storage_path, sha1, http_headers in snapshots: + row = { + "hash": content_hash, + "first_seen": first_seen, + "last_seen": last_seen, + "downloaded_at": downloaded_at, + "http_status": http_status, + "mime": mime, + "size": csize, + "path": storage_path, # serves as the download link target + "sha1": sha1, + "is_latest": is_latest == "yes", + "headers": json.loads(http_headers) if http_headers else None, + } + # Non-latest rows are "changed" when superseded by different (newer) content + row["changed"] = not row["is_latest"] and prev_hash is not None and prev_hash != content_hash + prev_hash = content_hash + url_detail.content_rows.append(row) url_detail.src = [row[0] for row in db.execute("SELECT source FROM url_source WHERE url = ?", (url,)).fetchall()] + # per-source observation stats for the Sources tab (source, first_seen, last_seen, occurrences, derived_from) + # "derived_from" is the source URL this URL was extracted from (discovered_urls), + # which allows deriving the original source for URLs extracted from a script hosted on another URL + url_detail.source_rows = db.execute(""" + SELECT us.source, us.first_seen, us.last_seen, us.occurrences, + (SELECT du.src_url FROM discovered_urls du WHERE du.url = us.url LIMIT 1) AS derived_from + FROM url_source us + WHERE us.url = ? + ORDER BY us.source + """, (url,)).fetchall() url_detail.src_urls = db.execute("SELECT src_url FROM discovered_urls WHERE url = ?", (url_detail.url,)).fetchall() url_detail.contained_urls = db.execute("SELECT url FROM discovered_urls WHERE src_url = ?", (url,)).fetchall() sessions = db.execute("SELECT sessions.session, sessions.idea_id FROM sessions JOIN url_session ON url_session.session=sessions.session_hash WHERE url_session.url = ?", (url,)).fetchall() @@ -294,7 +375,10 @@ def detail(): "joe-sandbox": f"https://www.joesandbox.com/analysis/search?q={url_detail.hash}" } - return render_template('detail.html', user=user, url=url_detail, sessions=sessions, show=show, links=links, inactive_for=inactive_for) + # tab menu under the URL name; badge counts reflect real DB data where available + menu = get_detail_menu(url, show, counts={"sources": len(url_detail.source_rows)}) + + return render_template('detail.html', user=user, url=url_detail, sessions=sessions, show=show, links=links, inactive_for=inactive_for, menu=menu, active_tab=active_tab) @app.route('/edit_detail', methods=['GET', 'POST']) @@ -387,3 +471,36 @@ def api_url_stats(): "src": ", ".join([s[0] for s in url_sources]), } return make_response(jsonify(return_dict), 200) + + +@app.route('/content/download', methods=['GET']) +def download_content(): + """Serve a stored content blob by its SHA-256 hash.""" + content_hash = flask.request.args.get('hash') + if not content_hash: + abort(404) + try: + data = load_content(config.content_storage_path, content_hash) + except FileNotFoundError: + abort(404) + return send_file(io.BytesIO(data), download_name=content_hash[:16], as_attachment=True) + + +@app.route('/sandbox/request', methods=['POST']) +def request_sandbox(): + """Stub: record a sandbox analysis request for a content snapshot.""" + user = get_user(flask.request.environ) + content_hash = flask.request.form.get('hash') or flask.request.args.get('hash') + url = flask.request.form.get('url') or flask.request.args.get('url') + if not content_hash: + abort(404) + with SQLiteWrapper(config.db_path) as db: + db.execute( + "INSERT INTO sandbox_job (content_hash, url, status, submitted_at, requested_by) VALUES (?, ?, 'pending', ?, ?)", + (content_hash, url, datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S'), user)) + return redirect(url_for('detail', url=url, tab='sandbox')) + + +if __name__ == '__main__': + app.run(host='127.0.0.1', port=5000, debug=True) + diff --git a/web/static/detail.css b/web/static/detail.css index 9e4f393..10c5c1c 100755 --- a/web/static/detail.css +++ b/web/static/detail.css @@ -16,6 +16,56 @@ overflow-wrap: anywhere; } +/* --- tab menu under the URL name --- */ +.detail .detail-menu { + margin: 0 20px; + border-bottom: 2px solid #e0e0e0; +} + +.detail .detail-menu ul { + display: flex; + flex-direction: row; + flex-wrap: wrap; + gap: 4px; + list-style: none; + margin: 0; + padding: 0; +} + +.detail .detail-menu-item { + display: inline-flex; + align-items: center; + gap: 8px; + padding: 8px 14px; + margin-bottom: -2px; + color: #818181; + text-decoration: none; + border-bottom: 2px solid transparent; +} + +.detail .detail-menu-item:hover { + color: #00838f; +} + +.detail .detail-menu-item.active { + color: #00acc1; + font-weight: bold; + border-bottom: 2px solid #00acc1; +} + +.detail .detail-menu-badge { + min-width: 20px; + height: 20px; + padding: 0 6px; + line-height: 20px; + text-align: center; + font-size: 12px; + font-weight: normal; + color: white; + background-color: #9e9e9e; + border-radius: 4px; +} + .detail .status { font-weight: bold; } @@ -209,6 +259,51 @@ padding-left: 20px !important; } +/* --- Sources tab table --- */ +.detail .sources-table { + width: 100%; + border-collapse: collapse; +} + +.detail .sources-table th { + text-align: left; + color: #818181; + font-weight: normal; + padding: 8px 12px 8px 0; + border-bottom: 1px solid #e0e0e0; +} + +.detail .sources-table td { + padding: 10px 12px 10px 0; + border-bottom: 1px solid #eee; + vertical-align: middle; +} + +.detail .sources-table th.num, +.detail .sources-table td.num { + text-align: right; +} + +.detail .sources-table td.empty { + color: #818181; + text-align: center; + padding: 20px 0; +} + +.detail .source-badge { + display: inline-block; + padding: 2px 10px; + background-color: #e0e0e0; + border-radius: 4px; + font-size: small; +} + +.detail .sources-table a.src-url { + text-decoration: underline; + color: grey; + overflow-wrap: anywhere; +} + .detail .list-urls { background-color: #f8f8f8; border: 1px solid #ddd; @@ -217,4 +312,98 @@ max-height: 100px; overflow: scroll; /* margin: 10px; */ +} + +/* --- Content tab table --- */ +.detail .content-table { + width: 100%; + border-collapse: collapse; +} + +.detail .content-table th { + text-align: left; + color: #818181; + font-weight: normal; + padding: 8px 12px 8px 0; + border-bottom: 1px solid #e0e0e0; +} + +.detail .content-table td { + padding: 10px 12px 10px 0; + border-bottom: 1px solid #eee; + vertical-align: middle; +} + +.detail .content-table th.num, +.detail .content-table td.num { + text-align: right; +} + +.detail .content-table td.hash { + font-family: "Lucida Console", "Courier New", monospace; + font-size: smaller; + white-space: nowrap; +} + +.detail .content-table td.empty { + color: #818181; + text-align: center; + padding: 20px 0; +} + +/* File path download link */ +.detail .content-table a.content-download { + text-decoration: underline; + color: #00acc1; + overflow-wrap: anywhere; +} + +/* "content changed" badge + highlighted row */ +.detail .content-table tr.content-changed td { + background-color: #fff4e5; +} + +.detail .badge-changed { + display: inline-block; + padding: 2px 10px; + background-color: #f0a83c; + color: #fff; + border-radius: 12px; + font-size: small; + white-space: nowrap; +} + +/* Collapsible HTTP headers */ +.detail .content-headers-row td { + padding-top: 0; + background-color: #fafafa; +} + +.detail .content-headers summary { + cursor: pointer; + color: #818181; + font-size: small; +} + +.detail .content-headers dl { + margin: 8px 0 0 0; + font-size: small; +} + +.detail .content-headers dl div { + display: flex; + padding: 2px 0; + border-bottom: 1px solid #f0f0f0; +} + +.detail .content-headers dt { + font-weight: bold; + min-width: 220px; + color: #555; + margin-left: 0; +} + +.detail .content-headers dd { + margin: 0; + overflow-wrap: anywhere; } \ No newline at end of file diff --git a/web/templates/detail.html b/web/templates/detail.html index bb4803d..d1b2fa0 100755 --- a/web/templates/detail.html +++ b/web/templates/detail.html @@ -23,7 +23,115 @@ onclick="location.href='{{ url_for('edit_detail', url=url.url) }}';"> + {% if menu %} + + {% endif %} +
+ {% if active_tab == 'content' %} + + + + + + + + + + + + + + + + {% for row in url.content_rows %} + + + + + + + + + + + {% if row.headers %} + + + + {% endif %} + {% else %} + + {% endfor %} + +
First seenLast seenFile pathSHA-256Mime typeSizeHTTP
{{ row.first_seen or '—' }}{{ row.last_seen or '—' }} + {% if row.path %} +

{{ row.path }}

+ — + {% endif %} + 		{{ row.hash[:16] }}…{{ row.mime or '—' }}{{ row.size if row.size is not none else '—' }}{{ row.http_status or '—' }} + {% if row.changed %}content changed{% endif %} +
+
+ HTTP response headers +
+ {% for k, v in row.headers.items() %} +
{{ k }}
{{ v }}
+ {% endfor %} +
+
+
No content downloaded for this URL yet.
+ {% elif active_tab == 'sources' %} + + + + + + + + + + + + + {% for row in url.source_rows %} + + + + + + + + {% else %} + + {% endfor %} + +
HoneynetFirst observedLast observedOccurrencesDerived from
{{ row[0] }}{{ row[1] if row[1] else '—' }}{{ row[2] if row[2] else '—' }}{{ row[3] if row[3] is not none else '—' }} + {% if row[4] %} + {{ row[4] }} + {% else %} + — + {% endif %} +
No sources recorded for this URL
+ {% else %} + @@ -192,9 +300,10 @@ {% endif %} + {% endif %} - +
{% if url.evaluated == 'yes' %}