diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 66251cb..309e648 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -9,7 +9,7 @@ "name": "engraphis-memory", "source": "./", "description": "Discipline for giving agents durable, scoped, explainable memory across sessions and repos with the Engraphis MCP tools.", - "version": "1.0.1" + "version": "1.1.0" } ] } diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index ed29620..5444c25 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "engraphis-memory", - "version": "1.0.1", + "version": "1.1.0", "description": "Give agents durable, scoped, explainable memory across sessions and repos via the Engraphis MCP tools. Use when you learn something worth keeping, need prior context before acting, or ask why/how a fact changed. Covers remember/recall, why/timeline, forget/pin/correct, sessions, and code search.", "author": { "name": "The Engraphis Authors", diff --git a/.claude-plugin/skill-assets.sha256 b/.claude-plugin/skill-assets.sha256 index 5cbcced..2f197f0 100644 --- a/.claude-plugin/skill-assets.sha256 +++ b/.claude-plugin/skill-assets.sha256 @@ -1,5 +1,5 @@ -5c5336f5cc0e46c118dd13aace779555c07df1d5ec89828c02b1763a39bb637b .claude-plugin/marketplace.json -5dcc1fed4a7f2a527b1f756b90dc12d9b80fe393df0090e22a603f71c1b4faba .claude-plugin/plugin.json +f02c452829de41bae8ab81a6e74a3a9f89895c51dc9ba01cd3947c4baf0622e2 .claude-plugin/marketplace.json +a17b520f3c1f3002aeba29d9312f9091d5a281035996c2e9c90b0397f4fb9fb9 .claude-plugin/plugin.json 89bf2728e44a3c877e31982d387fcfab10fbe065eb6441b2792375ca5105c07c skills/engraphis-memory/SKILL.md a295b0448e2ff372ddd8ea4e0bc8dc53f3d4bd56ff1fbd0d88cf4b6179511ce1 skills/engraphis-memory/references/CONVENTIONS.md 45f4b4ad9dbfd39f2b377083d9b3eec5eed7cba7cb8fa139e3f420bdd6105343 skills/engraphis-memory/references/SCOPING.md diff --git a/.env.example b/.env.example index 7be5648..ce5808c 100644 --- a/.env.example +++ b/.env.example @@ -10,7 +10,9 @@ ENGRAPHIS_PORT=8700 # clients. Hosted vendor, relay, compute, and worker roles are not distributed here. ENGRAPHIS_SERVICE_MODE=customer # The dashboard base URL is derived from ENGRAPHIS_HOST:ENGRAPHIS_PORT. -# Set ENGRAPHIS_DASHBOARD_URL for a canonical public HTTPS URL behind a reverse proxy. +# Set ENGRAPHIS_DASHBOARD_URL for a canonical public HTTPS URL behind a reverse proxy; +# it extends the dashboard MCP origin allow-list and drives legacy-inspector redirects. +# ENGRAPHIS_DASHBOARD_URL=https://engraphis.example.com # Update reminder. When on (default), the server checks for a newer Engraphis release # once a day and surfaces it in the dashboard banner, the startup log, and over MCP. @@ -151,7 +153,17 @@ ENGRAPHIS_LLM_API_KEY=sk-your-key-here # ENGRAPHIS_CLOUD_COMPUTE_URL=https://compute.engraphis.com # ENGRAPHIS_CLOUD_ORGANIZATION_ID=org_replace_me -# Prefer the owner-only ~/.engraphis/cloud_session.json created during hosted onboarding. +# Prefer the owner-only ~/.engraphis/cloud_session.json written by `engraphis connect`: +# +# engraphis connect --token engr_ct_... # the command your account portal shows +# printf %s "$TOKEN" | engraphis connect --token - # keep the token out of shell history +# +# That is the supported way to connect a client. It redeems the one-time connect token, saves +# the rotating refresh credential with 0600 permissions, and keeps it rotated afterwards, so +# none of the variables below are needed on an interactive machine. Set +# ENGRAPHIS_CLOUD_COMPUTE_URL (or pass --compute-url) only if your account portal shows a +# compute endpoint different from the default. See docs/AGENT_CONNECT.md. +# # For non-interactive deployments a refresh credential may be injected as a bootstrap secret. # It rotates on use; the owner-only saved replacement takes precedence afterward, even while # the environment variable remains set. Never commit it. Bind environment-only bootstrap @@ -203,3 +215,65 @@ ENGRAPHIS_LLM_API_KEY=sk-your-key-here # Optional credential-redacted JSON logs for hosted customer deployments. # ENGRAPHIS_JSON_LOGS=1 + +# ── Advanced / Operational ─────────────────────────────────────────────── +# These settings are used by advanced deployments, operators, and internal +# subsystems. Leave commented unless your deployment needs them. + +# Logging — defaults to INFO, text format. +# ENGRAPHIS_LOG_LEVEL=INFO +# ENGRAPHIS_LOG_FORMAT=text +# ENGRAPHIS_LOG_JSON=0 + +# CORS — comma-separated origins allowed to call the dashboard REST API. +# Default: http://127.0.0.1: and +# http://localhost:. Explicit values replace both loopback origins. +# ENGRAPHIS_CORS_ORIGINS=https://myapp.example.com + +# Rate limiting — requests per window. Default: 0 (disabled), with a 60s window +# used when ENGRAPHIS_RATE_LIMIT is set above zero. +# ENGRAPHIS_RATE_LIMIT=0 +# ENGRAPHIS_RATE_WINDOW=60 + +# HTTPS security origin — used for HTTP-to-HTTPS redirects and security headers. +# ENGRAPHIS_PUBLIC_URL wins; when unset, security falls back to +# ENGRAPHIS_DASHBOARD_URL, then the legacy ENGRAPHIS_RELAY_PUBLIC_URL setting. +# These settings do not change CORS; configure ENGRAPHIS_CORS_ORIGINS separately. +# ENGRAPHIS_PUBLIC_URL=https://engraphis.example.com +# ENGRAPHIS_RELAY_PUBLIC_URL=https://relay.example.com + +# Trusted local peers — comma-separated addresses exempt from rate limiting. +# ENGRAPHIS_LOCAL_TRUSTED_PEERS=127.0.0.1,::1 + +# Import roots — semicolon-separated (Windows) or colon-separated (POSIX) +# directories allowed as import sources. +# ENGRAPHIS_IMPORT_ROOTS=/srv/docs:/home/user/notes + +# Memory engine tuning — decay halflife (days), chunk sizing (tokens), +# proactive context loop, and reranker model. +# ENGRAPHIS_DECAY_HALFLIFE_DAYS=30 +# ENGRAPHIS_CHUNK_TOKENS=512 +# ENGRAPHIS_CHUNK_MAX=2048 +# ENGRAPHIS_CHUNK_OVERLAP=64 +# ENGRAPHIS_LOOP_INTERVAL=300 +# ENGRAPHIS_LOOP_TOP_K=10 +# ENGRAPHIS_RERANK_MODEL=cross-encoder/ms-marco-MiniLM-L-6-v2 + +# Workspace allow-list — comma-separated names. Empty = all allowed. +# ENGRAPHIS_WORKSPACES=acme,personal + +# Cloud Sync relay — endpoint and optional token for self-hosted relay. +# ENGRAPHIS_RELAY_URL=https://relay.example.com +# ENGRAPHIS_SYNC_TOKEN= +# ENGRAPHIS_SYNC_READ_ONLY=0 + +# Hosted plan upgrade URLs — override the default upgrade landing pages. +# ENGRAPHIS_UPGRADE_URL= +# ENGRAPHIS_PRO_UPGRADE_URL= +# ENGRAPHIS_TEAM_UPGRADE_URL= + +# Update check cache duration (seconds). Default: 86400 (1 day). +# ENGRAPHIS_UPDATE_CACHE=86400 + +# Legacy inspector port (retired 2026-07-10; redirects to dashboard). +# ENGRAPHIS_INSPECTOR_PORT=8710 diff --git a/.gitignore b/.gitignore index 5f53528..6e3ac55 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,8 @@ undelivered_license_keys.tsv # lives next to the DB or in ~/.engraphis/). Regenerable local state, not content. automation.json autosync.json +# Dashboard update-check cache; machine-local, regenerable runtime state. +.engraphis_update_check.json .venv/ venv/ dist/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 3902eb7..27e26dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ All notable changes to Engraphis are documented here. Format loosely follows ## [Unreleased] +## [1.1.0] - 2026-07-26 + +Public 1.1.0 hosted-connect and graph-experience release. + ### Removed - **"Signed compliance export" is no longer advertised on any surface.** It was granted to @@ -25,6 +29,35 @@ All notable changes to Engraphis are documented here. Format loosely follows the capability is not implemented and name the working alternative, instead of implying it is available in Engraphis Cloud. +### Added + +- **`engraphis connect --token engr_ct_…`** — the missing client half of device connect. + `cloud_session.save_bootstrap()` is the only writer of `~/.engraphis/cloud_session.json`, + and it had no production caller: the docs told paying customers to prefer a file nothing + created, so a purchased installation could not be connected without hand-writing state. + The new command redeems the one-time connect token from the account portal against + `POST /v1/devices/connect`, saves the returned session with owner-only permissions, and + verifies `cloud_session.configured()` before reporting success. The token is sent in the + request body and nowhere else — never printed, logged, or written to disk — and every + refusal maps to fixed, actionable copy (an expired or already-used token is not confused + with a lapsed subscription). Session storage is pre-flighted before the exchange, so an + unwritable state directory or a `cloud_session.json` replaced by a link fails the command + *without* spending the single-use token — the customer fixes the path and retries with the + same token instead of returning to the portal for a new one. Faults that can only happen + *after* the exchange — a reply truncated mid-body (`http.client.IncompleteRead`), or an + endpoint that stops resolving before the session is written (`CloudUrlUnresolved`) — are + reported as errors that say the token was already used, rather than escaping as tracebacks + that leave the customer unable to tell whether to retry. Also installed as + `engraphis-connect`. +- An `engraphis` front-door command that dispatches to the existing `engraphis-` + entry points, so the command the account portal displays is runnable as shown. +- A stable per-installation identity at `~/.engraphis/client_identity.json` (random ULIDs, + not a hardware fingerprint) so reconnecting a machine updates its existing installation + instead of registering a new device every time. +- An opt-in canvas graph engine (`?graph-engine=next`) with the existing ForceGraph + D3 + visual modes, deterministic communities, accessible classic fallback, lazy CSP-confined + assets, bounded simulations, and large/dense-graph rendering guards. + ### Changed - Managed compute consent now travels with the cloud account: an installation connected to @@ -34,6 +67,12 @@ All notable changes to Engraphis are documented here. Format loosely follows `ENGRAPHIS_MANAGED_COMPUTE_CONSENT` remains as an explicit operator override (`=0` opts a connected installation back out, `=1` forces it on regardless of session state) and is no longer surfaced anywhere in the UI. +- Hosted plan presentation now follows the control plane's active, trial, expired, and + lapsed states; renewal follows the plan actually held, account actions use the + plan-neutral portal, and a spent trial is never offered again. +- Device-connect and refresh responses validate credential fields as strings, preserve + single-use-token semantics across truncated responses, and refuse unsafe session-storage + paths before redeeming a token. ## [1.0.1] - 2026-07-24 diff --git a/MANIFEST.in b/MANIFEST.in index 4ac8777..4192c7b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,6 +3,7 @@ global-exclude *.pyo recursive-include engraphis/static *.html *.css *.js *.png *.ico recursive-include engraphis/static/vendor * +recursive-include engraphis/dashboard_assets *.js include engraphis/commercial_manifest.json include LICENSE NOTICE README.md CHANGELOG.md include pyproject.toml diff --git a/README.md b/README.md index 7a2cd22..9c0c997 100644 --- a/README.md +++ b/README.md @@ -664,7 +664,7 @@ All via environment (or `.env`): | `ENGRAPHIS_LLM_MODEL` | `gpt-4o-mini` | Model name (provider-specific) | | `ENGRAPHIS_LLM_API_KEY` | — | API key for chat/synthesis, `llm` / `llm_structured` extraction, and structured consolidation | | `ENGRAPHIS_LLM_BASE_URL` | — | Base URL for openrouter / custom OpenAI-compatible endpoints | -| `ENGRAPHIS_LLM_AUTO_EXTRACT` | `1` | After a successful live connection test, automatically switch the running engine to `llm_structured`; the dashboard's extraction Off button persists `0`, and its On button restores `1` | +| `ENGRAPHIS_LLM_AUTO_EXTRACT` | `0` | Opt in to switching the running engine to `llm_structured` after a successful live connection test; the dashboard's extraction Off button persists `0`, and its On button restores `1` | | `ENGRAPHIS_FORWARDED_ALLOW_IPS` | *(none)* | Proxies trusted for forwarded client/TLS headers (`*` only when the service is reachable exclusively through that proxy) | | `ENGRAPHIS_LOCAL_TRUSTED_PEERS` | *(none)* | Exact peers/CIDRs treated as local without forwarding headers; intended for the shipped loopback-published Compose bridge, not public deployments | | `ENGRAPHIS_CLOUD_CONTROL_URL` | hosted default | Official entitlement, organization, and credential control API | diff --git a/docs/AGENT_CONNECT.md b/docs/AGENT_CONNECT.md index 67e2b37..4ea833f 100644 --- a/docs/AGENT_CONNECT.md +++ b/docs/AGENT_CONNECT.md @@ -25,8 +25,9 @@ organization: 1. The organization owner starts Team or purchases a subscription in Engraphis Cloud. 2. The owner invites named members and assigns roles in the hosted dashboard. -3. A member accepts the invitation and creates a scoped agent/device credential. -4. The member configures their agent with the hosted URL and the one-time credential. +3. A member accepts the invitation and generates a one-time **connect token** (`engr_ct_…`). + The account portal shows the exact command to run. +4. The member runs that command on the machine being connected (see below). 5. The hosted service rechecks organization membership, role, scopes, entitlement version, and workspace binding on every request. @@ -37,6 +38,53 @@ The hosted onboarding flow provides the exact endpoint and client snippet for th organization. Do not substitute the URL of a public self-hosted image: that image intentionally has no Team identity backend. +## Connect a machine: `engraphis connect` + +Copy the command from your account portal and run it on the machine you are connecting: + +```bash +engraphis connect --token engr_ct_... +``` + +That redeems the token against `POST /v1/devices/connect` on the control plane and writes the +owner-only session file `~/.engraphis/cloud_session.json` (mode `0600`). The dashboard, the MCP +server, and Cloud Sync all read that file, so no environment secret is needed afterwards. Rerun +`engraphis connect` with a fresh token on every machine you want connected. + +Useful options: + +| Option | Effect | +| --- | --- | +| `--token -` | Read the token from stdin, so it never enters shell history. | +| `--workspace WS_ID` | Bind this device to a single workspace. | +| `--label TEXT` | Name this installation in your account portal. | +| `--device-name TEXT` | Override the device name (defaults to the hostname). | +| `--control-url URL` | Point at a non-default control plane. | +| `--compute-url URL` | Set the managed compute endpoint (also `ENGRAPHIS_CLOUD_COMPUTE_URL`). | +| `--json` | Print a redacted, machine-readable summary. | + +The same command is installed as `engraphis-connect`, matching the other `engraphis-*` scripts. + +Connect tokens are **single-use and short-lived**. The service answers every refusal — expired, +already redeemed, or never valid — with the same `401`, so the client reports all three +possibilities and the fix is always the same: generate a new token in the account portal. A `402` +means the subscription itself has lapsed; fix billing rather than the token. + +Because the token is single-use, the client checks that it can actually write the session file +*before* redeeming it. If the state directory is not writable, or `cloud_session.json` has been +replaced by a symlink, a hard link, or a directory, the command fails immediately, names the path +to fix, and sends nothing — your token is untouched, so you can correct the path and rerun the +same command rather than issuing a new token. + +The token is a credential. It is sent in the request body and nowhere else — it is never +printed, never logged, and never written to disk. What *is* written is the rotating refresh +credential the service returns, which is why the session file is owner-only. + +The command also mints a stable per-installation identity at `~/.engraphis/client_identity.json` +(random ULIDs, not a hardware fingerprint) so reconnecting the same machine updates the existing +installation instead of registering a new device every time. Both files honour +`ENGRAPHIS_STATE_DIR`; a distinct state directory is a distinct installation. + ## Credential lifecycle Hosted access uses short-lived access tokens plus rotating refresh credentials. A refresh family @@ -45,7 +93,10 @@ service; the raw replacement is returned once and must be kept in an owner-only or secrets manager. Customer-side environment variables are documented in [`.env.example`](../.env.example). Prefer -the onboarding-created `~/.engraphis/cloud_session.json` over long-lived environment secrets. +the `~/.engraphis/cloud_session.json` that `engraphis connect --token` writes over long-lived +environment secrets: it holds a rotating credential, it is owner-only, and it is the path the +client keeps up to date on its own. Environment secrets are for non-interactive deployments that +cannot run the connect command. ## Trial and grace diff --git a/docs/images/knowledge-graph.png b/docs/images/knowledge-graph.png index fb737eb..9b67fe8 100644 Binary files a/docs/images/knowledge-graph.png and b/docs/images/knowledge-graph.png differ diff --git a/engraphis/__init__.py b/engraphis/__init__.py index 2e9166c..90979c1 100644 --- a/engraphis/__init__.py +++ b/engraphis/__init__.py @@ -7,4 +7,4 @@ except PackageNotFoundError: # source tree without an installed distribution # Keep in step with [project] version in pyproject.toml — tests/test_packaging.py # pins the two together so a release cannot ship them out of sync. - __version__ = "1.0.1" + __version__ = "1.1.0" diff --git a/engraphis/app.py b/engraphis/app.py index 8be27a6..29d3e88 100644 --- a/engraphis/app.py +++ b/engraphis/app.py @@ -24,7 +24,7 @@ from engraphis.logging_setup import configure_logging from engraphis.netutil import client_ip from engraphis.routes.memory import router as memory_router -from engraphis.routes.vault import router as vault_router +from engraphis.routes.vault import VAULT_UPLOAD_REQUEST_BYTES, router as vault_router from engraphis.stores import get_conn, init_db logger = logging.getLogger("engraphis") @@ -35,6 +35,100 @@ # Readiness cache: only a *successful* embedder init is cached, so a transient # failure is re-checked on the next probe instead of wedging the pod NotReady. _embedder_ok: bool = False +_UPLOAD_LIMIT_PATHS = frozenset({ + "/api/workspaces/import-files", + "/memory/vaults/upload-folder", + "/memory/vaults/upload-folder-smart", +}) + + +class _RequestBodyTooLarge(Exception): + """Internal signal used by the streaming ASGI request limiter.""" + + +class _VaultUploadLimitMiddleware: + """Reject oversized file imports before multipart parsing/spooling. + + ``Content-Length`` provides an immediate fast-fail. The receive wrapper is still + required because clients can omit or lie about that header, including HTTP/1.1 + chunked uploads. A fronting proxy should configure an equal or lower request-body + ceiling; this in-process guard remains the last line of defense for direct access. + """ + + def __init__(self, app, max_bytes: int): + self.app = app + self.max_bytes = max_bytes + + async def __call__(self, scope, receive, send): + if ( + scope["type"] != "http" + or scope["method"] != "POST" + or scope["path"].rstrip("/") not in _UPLOAD_LIMIT_PATHS + ): + await self.app(scope, receive, send) + return + + raw_lengths = [ + value for name, value in scope.get("headers", []) + if name.lower() == b"content-length" + ] + if len(raw_lengths) > 1: + await JSONResponse( + {"error": "invalid content-length"}, + status_code=400, + )(scope, receive, send) + return + if raw_lengths: + try: + declared_length = int(raw_lengths[0]) + except (TypeError, ValueError): + declared_length = -1 + if declared_length < 0: + await JSONResponse( + {"error": "invalid content-length"}, + status_code=400, + )(scope, receive, send) + return + if declared_length > self.max_bytes: + await self._too_large(scope, receive, send) + return + + received = 0 + limit_exceeded = False + + async def limited_receive(): + nonlocal limit_exceeded, received + message = await receive() + if message["type"] == "http.request": + received += len(message.get("body", b"")) + if received > self.max_bytes: + limit_exceeded = True + raise _RequestBodyTooLarge + return message + + async def guarded_send(message): + # FastAPI converts arbitrary body-parser exceptions into a generic 400. + # Suppress that replacement response once our receive wrapper has observed + # the real cause; the middleware emits the canonical 413 below. + if limit_exceeded: + return + await send(message) + + try: + await self.app(scope, limited_receive, guarded_send) + except _RequestBodyTooLarge: + pass + if limit_exceeded: + await self._too_large(scope, receive, send) + + async def _too_large(self, scope, receive, send): + await JSONResponse( + { + "error": "request body too large", + "max_bytes": self.max_bytes, + }, + status_code=413, + )(scope, receive, send) def _embedder_ready() -> bool: @@ -115,6 +209,10 @@ def create_app() -> FastAPI: allow_methods=["*"], allow_headers=["*"], ) + app.add_middleware( + _VaultUploadLimitMiddleware, + max_bytes=VAULT_UPLOAD_REQUEST_BYTES, + ) # Optional bearer-token auth. Active only when ENGRAPHIS_API_TOKEN is set. # Health-type probes (liveness + readiness) stay unauthenticated by convention. diff --git a/engraphis/backends/postgres_schema.py b/engraphis/backends/postgres_schema.py index bbd12f0..979559b 100644 --- a/engraphis/backends/postgres_schema.py +++ b/engraphis/backends/postgres_schema.py @@ -6,8 +6,11 @@ from __future__ import annotations import hashlib +import ipaddress import os -from typing import Any, Optional +import socket +from typing import Any, Optional, Union +from urllib.parse import urlparse from engraphis.core.interfaces import SchemaSnapshot @@ -32,19 +35,91 @@ def _bounded_env_int(name: str, default: int, maximum: int) -> int: return max(1, min(maximum, value)) +def _global_unicast( + address: Union[ipaddress.IPv4Address, ipaddress.IPv6Address], +) -> bool: + """Return whether *address* is safe for an untrusted outbound connection.""" + mapped = getattr(address, "ipv4_mapped", None) + if mapped is not None: + address = mapped + return ( + address.is_global + and not address.is_multicast + and not address.is_unspecified + and not address.is_loopback + and not address.is_link_local + and not address.is_private + and not address.is_reserved + ) + + +def _validate_dsn_host(dsn: str) -> tuple[str, Optional[str]]: + """Return the TLS hostname and pinned socket address for a safe DSN.""" + env_dsn = os.environ.get("ENGRAPHIS_POSTGRES_DSN", "") + if env_dsn and dsn.strip() == env_dsn.strip(): + return "", None + try: + parsed = urlparse(dsn) + hostname = parsed.hostname + except (TypeError, ValueError) as exc: + raise PostgresIntrospectionError("invalid PostgreSQL DSN") from exc + if not hostname: + raise PostgresIntrospectionError("PostgreSQL DSN must include a hostname") + loopback = { + "localhost": "127.0.0.1", + "127.0.0.1": "127.0.0.1", + "::1": "::1", + } + if hostname.lower() in loopback: + return hostname, loopback[hostname.lower()] + try: + infos = socket.getaddrinfo( + hostname, + parsed.port or 5432, + type=socket.SOCK_STREAM, + proto=socket.IPPROTO_TCP, + ) + except (OSError, ValueError) as exc: + raise PostgresIntrospectionError("cannot resolve PostgreSQL host") from exc + addresses: list[str] = [] + for _, _, _, _, sockaddr in infos: + try: + addr = ipaddress.ip_address(sockaddr[0]) + except ValueError: + raise PostgresIntrospectionError( + "PostgreSQL host resolved to an invalid address" + ) + if not _global_unicast(addr): + raise PostgresIntrospectionError( + "PostgreSQL DSN must resolve only to global unicast addresses" + ) + canonical = str(addr) + if canonical not in addresses: + addresses.append(canonical) + if not addresses: + raise PostgresIntrospectionError("cannot resolve PostgreSQL host") + return hostname, addresses[0] + + def _connect(dsn: str): + hostname, hostaddr = _validate_dsn_host(dsn) timeout = _bounded_env_int( "ENGRAPHIS_POSTGRES_CONNECT_TIMEOUT", _DEFAULT_CONNECT_TIMEOUT_SECONDS, _MAX_CONNECT_TIMEOUT_SECONDS, ) + connect_kwargs: dict[str, Any] = {"connect_timeout": timeout} + if hostaddr is not None: + # libpq connects to hostaddr without another DNS lookup, while host remains + # available for TLS certificate verification and password-file matching. + connect_kwargs.update(host=hostname, hostaddr=hostaddr) try: import psycopg - return psycopg.connect(dsn, connect_timeout=timeout) + return psycopg.connect(dsn, **connect_kwargs) except ImportError: try: import psycopg2 - return psycopg2.connect(dsn, connect_timeout=timeout) + return psycopg2.connect(dsn, **connect_kwargs) except ImportError as exc: raise PostgresIntrospectionError( "PostgreSQL introspection needs psycopg: " diff --git a/engraphis/cloud_session.py b/engraphis/cloud_session.py index b539afe..b638227 100644 --- a/engraphis/cloud_session.py +++ b/engraphis/cloud_session.py @@ -6,9 +6,11 @@ """ from __future__ import annotations +import http.client import json import os import stat +import tempfile import threading import time import urllib.error @@ -74,12 +76,36 @@ def _reachable_cloud_base_url(value: str) -> str: raise CloudSessionError("Engraphis Cloud is temporarily unreachable.") from exc +def _server_compute_url(response: dict) -> str: + """Return the refresh response's compute endpoint only when it is safe to persist.""" + + value = response.get("compute_url") + if not isinstance(value, str) or not value.strip(): + return "" + try: + return validate_cloud_base_url(value) + except (CloudUrlUnresolved, ValueError): + # This optional field cannot be allowed to clear or poison a known-good endpoint. + # The rotated credential is still persisted below, so a later valid response can + # repair a session whose compute endpoint was not supplied yet. + return "" + + def _session_path() -> Path: root = os.environ.get("ENGRAPHIS_STATE_DIR", "").strip() base = Path(root).expanduser() if root else Path.home() / ".engraphis" return base / "cloud_session.json" +def _refresh_lock_path() -> Path: + return _session_path().with_name(".cloud_session.refresh.lock") + + +#: Exactly the flags ``_refresh_lock`` opens the lock with, so ``preflight_save`` cannot +#: drift from the access the real save needs. +_LOCK_OPEN_FLAGS = os.O_RDWR | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0) + + @contextmanager def _refresh_lock(): """Serialize spend-and-rotate of the single-use refresh credential. @@ -89,11 +115,11 @@ def _refresh_lock(): so every process coordinates on one stable filesystem object. """ with _REFRESH_THREAD_LOCK: - lock_path = _session_path().with_name(".cloud_session.refresh.lock") + lock_path = _refresh_lock_path() try: lock_path.parent.mkdir(parents=True, exist_ok=True) expected = private_file_stat(lock_path, allow_missing=True) - flags = os.O_RDWR | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0) + flags = _LOCK_OPEN_FLAGS if expected is None: try: descriptor = os.open( @@ -212,8 +238,108 @@ def _save(value: dict) -> None: atomic_private_text(path, json.dumps(value, sort_keys=True, separators=(",", ":"))) +def preflight_save() -> Path: + """Verify the session file can be saved, without saving anything. Returns its path. + + :func:`save_bootstrap` runs *after* the control plane has answered, so a state + directory that lost its permissions, or a ``cloud_session.json`` that is a symlink or + a hard link, was only discovered once a single-use connect token had already been + consumed: the customer was left with a spent token, no session, and a trip back to + the portal for a new one. Any caller about to spend a one-shot credential must run + this first, so a storage fault costs nothing. + + The checks are the ones the write path itself applies -- :func:`private_file_stat` on + the session leaf and on its refresh lock, plus the randomized sibling temp file + :func:`atomic_private_text` has to create -- so the preflight cannot drift from what + the real save will accept. It deliberately never opens, creates or replaces the + session leaf: an existing session survives a failed preflight untouched, and a first + connect is not turned into a half-written file. + """ + + path = _session_path() + try: + path.parent.mkdir(parents=True, exist_ok=True) + except OSError as exc: + raise CloudSessionError( + "The Engraphis state directory %s could not be created." % path.parent, + status=409, + ) from exc + for candidate in (path, _refresh_lock_path()): + try: + private_file_stat(candidate, allow_missing=True) + except UnsafeStateFile as exc: + raise CloudSessionError( + "%s is not a plain private file -- a symlink, hard link or directory is " + "in its place. Remove it." % candidate, + status=409, + ) from exc + except OSError as exc: + raise CloudSessionError( + "%s could not be inspected; check its permissions." % candidate, + status=409, + ) from exc + # ``private_file_stat`` above is an ``lstat``: it proves the lock is a plain, private, + # unlinked file, not that this process can *open* it. ``_refresh_lock`` opens it + # ``O_RDWR``, so a lock left behind by another UID -- or one whose mode changed -- + # passes the stat and fails the save, after the token is spent. Open it here with the + # same flags when it already exists. When it does not, ``_refresh_lock`` creates it in + # a directory the probe below proves writable, so there is nothing to pre-check and + # nothing is created here. + lock_path = _refresh_lock_path() + if lock_path.exists(): + try: + descriptor = os.open(str(lock_path), _LOCK_OPEN_FLAGS) + except OSError as exc: + raise CloudSessionError( + "The cloud session refresh lock %s exists but cannot be opened; check its " + "ownership and permissions." % lock_path, + status=409, + ) from exc + os.close(descriptor) + # Probe the directory the way ``atomic_private_text`` will, rather than touching + # ``cloud_session.json``: a read-only mount or a lost ACL fails here, while a valid + # existing session is never created, truncated or replaced. + try: + descriptor, probe = tempfile.mkstemp( + prefix=".%s.preflight." % path.name, dir=str(path.parent) + ) + except OSError as exc: + raise CloudSessionError( + "The Engraphis state directory %s is not writable, so the cloud session " + "cannot be saved there." % path.parent, + status=409, + ) from exc + os.close(descriptor) + try: + os.unlink(probe) + except OSError as exc: + # Not cosmetic, and not "the probe was just created so of course it can be + # removed": creating a file and removing one are separate rights. A directory ACL + # that grants add-file but denies delete lets ``mkstemp`` succeed and ``unlink`` + # fail, and ``atomic_private_text`` finishes with ``os.replace`` over the session + # leaf -- which needs exactly the delete/replace right that just failed. Swallowing + # this let the preflight pass on a directory the real save cannot use, redeeming the + # single-use token before failing, and left the probe behind on every attempt. That + # is precisely the drift the preflight exists to prevent. + raise CloudSessionError( + "The Engraphis state directory %s does not allow files to be replaced, so the " + "cloud session cannot be saved there. Check its permissions; a leftover %s may " + "need removing." % (path.parent, Path(probe).name), + status=409, + ) from exc + return path + + #: Plans that carry no paid cloud access, used only to default an absent activity flag. _UNPAID_PLANS = ("free", "local") +#: Entitlement status vocabulary the control plane can persist or compute +#: (``engraphis_cloud/entitlements.py`` ``effective_status``, plus every provider status +#: ``/internal/subscriptions/apply`` accepts). Kept as a bound, not an allow-list: an +#: unrecognised value is stored verbatim so a future server release is not mistranslated, +#: it is only length-capped like every other presentation string here. +_MAX_STATUS_CHARS = 32 +#: An ISO-8601 timestamp is at most a few dozen characters; anything longer is not one. +_MAX_TIMESTAMP_CHARS = 64 #: Bounds on the entitlement fields before they are written into the session record. The #: record is read back under a 64 KiB private-state cap, so an unbounded provider value #: could grow the file past that cap and make the whole session permanently unreadable. @@ -221,17 +347,36 @@ def _save(value: dict) -> None: _MAX_PLAN_CHARS = 64 _MAX_FEATURES = 32 _MAX_FEATURE_CHARS = 64 +#: Every entitlement key ``_declared_entitlement`` can put on the session record. The +#: record is deliberately shaped like the wire response so one reader serves both, which +#: is what keeps a saved answer and a fresh one from being parsed by different rules. +_ENTITLEMENT_KEYS = ( + "plan", + "cloud_access_active", + "cloud_features", + "status", + "is_trial", + "trial_consumed", + "trial_ends_at", +) def _declared_entitlement(response: object) -> dict: """Return the entitlement fields a control-plane response carried, or ``{}``. ``DeviceRegistrationResponse`` — the body both ``/internal/devices/register`` and - ``POST /v1/tokens/refresh`` answer with — carries ``plan``, ``cloud_features`` and - ``cloud_access_active``. They are read as *optional* on purpose: a control plane that - has not deployed them yet returns exactly what it always did, and this client must keep - working against it rather than requiring a newer server. An absent or unusable ``plan`` - therefore yields ``{}``, which leaves whatever the caller already knew in place. + ``POST /v1/tokens/refresh`` answer with — carries ``plan``, ``cloud_features``, + ``cloud_access_active``, and the trial facts (``status``, ``is_trial``, + ``trial_consumed``, ``trial_ends_at``). They are read as *optional* on purpose: a + control plane that has not deployed them yet returns exactly what it always did, and + this client must keep working against it rather than requiring a newer server. An + absent or unusable ``plan`` therefore yields ``{}``, which leaves whatever the caller + already knew in place. + + The trial fields are each optional *individually* as well, because they shipped after + the plan fields did: a server that answers ``plan`` but not ``is_trial`` simply leaves + the key absent, and the caller keeps treating the customer as a non-trialist rather + than claiming a trial nobody declared. Nothing here is authority. The cloud authorizes every paid call regardless of what this record says; persisting it only saves the dashboard from guessing. @@ -259,6 +404,23 @@ def _declared_entitlement(response: object) -> dict: item.strip().lower()[:_MAX_FEATURE_CHARS] for item in features if isinstance(item, str) and item.strip() })[:_MAX_FEATURES] + # The trial half of the same answer. Absent stays absent so a field-less older server + # cannot erase what a newer one already recorded, exactly as ``cloud_features`` behaves. + status = response.get("status") + if isinstance(status, str) and status.strip(): + declared["status"] = status.strip().lower()[:_MAX_STATUS_CHARS] + for key in ("is_trial", "trial_consumed"): + value = response.get(key) + if isinstance(value, bool): + declared[key] = value + ends_at = response.get("trial_ends_at") + if isinstance(ends_at, str) and ends_at.strip(): + declared["trial_ends_at"] = ends_at.strip()[:_MAX_TIMESTAMP_CHARS] + elif ends_at is None and "is_trial" in declared and not declared["is_trial"]: + # An explicit ``null`` from a server that *did* answer the trial question is + # meaningful: it is how a converted paying customer is told they have no live trial + # boundary. Clear a stale one rather than letting a past trial end survive forever. + declared["trial_ends_at"] = "" return declared @@ -329,6 +491,13 @@ def record_billing_denial() -> bool: ) saved["cloud_access_active"] = False saved["cloud_features"] = [] + # The last status the server named ("active", "trialing", …) is now known to + # contradict this denial, so it must not survive as renderable copy. The trial + # facts are *not* invalidated by a lapse -- whether this was a trial, when it + # ended, and whether one was consumed are all still true -- and they are what + # lets the dashboard say "your free trial ended" rather than the generic + # "your subscription lapsed". + saved.pop("status", None) saved["entitlement_checked_at"] = time.time() # Inside the lock: a save that lands after release is exactly the race above. _save(saved) @@ -337,12 +506,31 @@ def record_billing_denial() -> bool: return False +def text_field(response: dict, key: str) -> str: + """Return ``response[key]`` when it is a string, else ``""``. Never a ``repr``. + + ``str(response.get(key) or "")`` looks like a coercion but is not a validation: JSON + arrays and objects arrive as Python ``list``/``dict``, and ``str()`` renders their + *repr*, which is both truthy and non-empty. A control-plane reply carrying + ``"refresh_credential": ["tok"]`` was therefore stored as the literal text ``['tok']``; + ``configured()`` read that back as a usable session and connect reported success, while + the next refresh submitted the junk and failed -- after the single-use connect token had + already been spent, so the customer could not simply retry. + + Provider bodies are untrusted, so a field that must be a string is required to be one. + """ + + value = response.get(key) + return value.strip() if isinstance(value, str) else "" + + def save_bootstrap(response: dict, *, control_url: str, - compute_url: Optional[str] = None) -> None: + compute_url: Optional[str] = None, + compute_url_source: Optional[str] = None) -> None: """Persist the one-time bootstrap/refresh material returned by the control plane.""" - refresh = str(response.get("refresh_credential") or "").strip() - organization_id = str(response.get("organization_id") or "").strip() + refresh = text_field(response, "refresh_credential") + organization_id = text_field(response, "organization_id") if not refresh or not organization_id: raise CloudSessionError("Cloud bootstrap did not return a refresh credential.") value = { @@ -350,15 +538,20 @@ def save_bootstrap(response: dict, *, control_url: str, "control_url": validate_cloud_base_url(control_url), "compute_url": validate_cloud_base_url(compute_url) if compute_url else "", "organization_id": organization_id, - "installation_id": str(response.get("installation_id") or ""), - "device_id": str(response.get("device_id") or ""), - "member_id": str(response.get("member_id") or ""), + "installation_id": text_field(response, "installation_id"), + "device_id": text_field(response, "device_id"), + "member_id": text_field(response, "member_id"), "refresh_credential": refresh, - "refresh_expires_at": str(response.get("refresh_expires_at") or ""), + "refresh_expires_at": text_field(response, "refresh_expires_at"), "token_subject": _validated_token_subject( response.get("token_subject") or "member" ), } + if compute_url_source in {"explicit", "server", "fallback"}: + # Keep an explicit CLI choice distinct from a server/distribution default. A + # refresh may move cloud-assigned endpoints, but must never silently replace an + # endpoint the operator deliberately selected. + value["compute_url_source"] = compute_url_source # Whatever entitlement the control plane volunteered, so the dashboard knows the plan # from the first boot instead of inferring it. Absent on an older cloud; harmless. value.update(_declared_entitlement(response)) @@ -398,6 +591,18 @@ def _refresh_http_error(status: int) -> CloudSessionError: return CloudSessionError("Engraphis Cloud could not refresh this session.") +#: What a best-effort drain of an error body is allowed to fail with. See the comment in +#: :func:`_post_refresh`; ``engraphis.device_connect`` guards its drain with the same tuple. +_DRAIN_FAILURES = (OSError, ValueError, http.client.HTTPException) + +#: Public copy for every ambiguous refresh response. Once the POST is written, replaying the +#: single-use credential can revoke its family; reconnecting is the safe recovery. +_INCOMPLETE_REFRESH_RESPONSE = ( + "Engraphis Cloud answered this session refresh but the reply was incomplete, " + "so the rotated credential could not be saved. Connect this installation again." +) + + def _post_refresh(control_url: str, refresh: str, workspace_id: Optional[str], token_subject: str) -> dict: # An org-scoped entitlement read asks for an unbound token, so it passes no workspace. @@ -417,37 +622,91 @@ def _post_refresh(control_url: str, refresh: str, workspace_id: Optional[str], }, method="POST", ) + # Split for the same reason as ``device_connect.post_connect``, and with sharper + # consequences here. Once ``open`` returns, a success status line has been parsed, so + # the control plane processed the refresh and the single-use credential it was given is + # spent -- but the rotated replacement only reaches disk after the body parses, below. try: - with build_pinned_https_opener(_NoRedirect()).open( - request, timeout=10.0 - ) as response: - raw = response.read(_MAX_RESPONSE_BYTES + 1) + response = build_pinned_https_opener(_NoRedirect()).open(request, timeout=10.0) except urllib.error.HTTPError as exc: code = exc.code # Draining and closing the error body can itself time out or reset. A sibling # ``except`` clause of this ``try`` does NOT cover an exception raised inside this # handler, so an unguarded read escapes as an unhandled traceback whenever the # cloud is flaky -- exactly the launch-day condition this path exists for. + # + # ``HTTPException`` is named explicitly: a truncated chunked error body raises + # ``http.client.IncompleteRead``, whose MRO is ``(IncompleteRead, HTTPException, + # Exception, BaseException, object)`` -- neither an ``OSError`` nor a ``ValueError``, + # so the pair alone let it through. try: exc.read(_MAX_RESPONSE_BYTES + 1) - except (OSError, ValueError): + except _DRAIN_FAILURES: pass finally: try: exc.close() - except (OSError, ValueError): + except _DRAIN_FAILURES: pass raise _refresh_http_error(code) - except (urllib.error.URLError, TimeoutError, OSError) as exc: + except urllib.error.URLError as exc: raise CloudSessionError("Engraphis Cloud is temporarily unreachable.") from exc + except (TimeoutError, OSError, http.client.HTTPException) as exc: + # These failures arise while ``open`` is waiting for the response, after urllib has + # written the POST. The control plane may have spent the single-use credential even + # though no usable status line reached us. + # + # Keep ``URLError`` separate above: urllib uses it for failures while establishing or + # sending the request, the only phase where retrying the same credential is safe. + # ``RemoteDisconnected`` has both OSError and BadStatusLine ancestry and therefore + # deliberately lands in this non-transient bucket. + raise CloudSessionError( + _INCOMPLETE_REFRESH_RESPONSE, + status=409, + ) from exc + + try: + with response: + raw = response.read(_MAX_RESPONSE_BYTES + 1) + except (OSError, ValueError, http.client.HTTPException) as exc: + # Post-response, and therefore NOT a transient outage. The server answered, so the + # credential just submitted is spent, but the rotation it returned never reached + # ``_save`` -- the stale value is still on disk. ``_public_session_error`` maps 503 + # to ``transient=True``, which ``CloudFeatureClient.run_job`` acts on by retrying; + # that retry would resubmit the spent credential, and this module already documents + # (see ``_note_denied``) that the control plane treats replay by revoking the whole + # credential family. 409 is the existing "saved session is unusable; connect this + # installation again" bucket, which is the honest answer here. + # + # This deliberately prefers a false "reconnect" over a replay: if the truncation + # happened before the server committed the rotation the old credential was still + # good and the reconnect was unnecessary, but the opposite mistake revokes every + # credential in the family and forces the same reconnect anyway, from a worse state. + raise CloudSessionError(_INCOMPLETE_REFRESH_RESPONSE, status=409) from exc + + # These are post-response too, so they carry the same replay hazard as the truncated + # body above and take the same non-transient status: the server consumed the credential + # it was given, and a body this client cannot parse means the rotation never landed. if len(raw) > _MAX_RESPONSE_BYTES: - raise CloudSessionError("Engraphis Cloud returned an oversized session response.") + raise CloudSessionError( + "Engraphis Cloud returned an oversized session response, so the rotated " + "credential could not be saved. Connect this installation again.", + status=409, + ) try: body = json.loads(raw.decode("utf-8")) except (UnicodeDecodeError, ValueError, RecursionError) as exc: - raise CloudSessionError("Engraphis Cloud returned an invalid session response.") from exc + raise CloudSessionError( + "Engraphis Cloud returned an invalid session response, so the rotated " + "credential could not be saved. Connect this installation again.", + status=409, + ) from exc if not isinstance(body, dict): - raise CloudSessionError("Engraphis Cloud returned an invalid session response.") + raise CloudSessionError( + "Engraphis Cloud returned an invalid session response, so the rotated " + "credential could not be saved. Connect this installation again.", + status=409, + ) return body @@ -494,7 +753,9 @@ def access_for_workspace( # response; a stale home-directory mount yields a structured, retryable error from # ``_load`` rather than an unhandled filesystem exception. The authoritative session # record is still loaded again under the lock below before any credential is used. - if not configured(require_compute=require_compute): + # A refresh can now supply a missing compute endpoint, so do not reject a valid saved + # control/refresh session before it has a chance to receive that authoritative value. + if not configured(require_compute=False): raise CloudSessionError( "Connect this installation to Engraphis Cloud first.", status=401 ) @@ -510,8 +771,18 @@ def access_for_workspace( ).strip() control = os.environ.get("ENGRAPHIS_CLOUD_CONTROL_URL", "").strip() control = control or str(saved.get("control_url") or "").strip() - compute = direct_compute or str(saved.get("compute_url") or "").strip() - if not refresh or not control or (require_compute and not compute): + saved_compute = str(saved.get("compute_url") or "").strip() + compute = direct_compute or saved_compute + compute_source = "explicit" if direct_compute else str( + saved.get("compute_url_source") or "" + ) + if saved_compute and compute_source not in {"explicit", "server", "fallback"}: + # Pre-source sessions cannot tell a cloud-assigned endpoint from a CLI/env + # override. Preserve the nonempty value rather than silently moving a + # potentially operator-selected endpoint; compute-less legacy sessions still + # accept a vetted response below. + compute_source = "explicit" + if not refresh or not control: raise CloudSessionError( "Connect this installation to Engraphis Cloud first.", status=401 ) @@ -519,16 +790,28 @@ def access_for_workspace( compute = _reachable_cloud_base_url(compute) if compute else "" token_subject = _token_subject(saved) body = _post_refresh(control, refresh, workspace_id, token_subject) - access = str(body.get("access_token") or "").strip() - organization_id = str( - body.get("organization_id") or saved.get("organization_id") or "" - ).strip() - rotated = str(body.get("refresh_credential") or "").strip() + # Same untrusted-provider boundary as ``save_bootstrap``: a non-string credential + # would otherwise be stored as its ``repr`` and submitted on the next refresh. + access = text_field(body, "access_token") + organization_id = ( + text_field(body, "organization_id") or text_field(saved, "organization_id") + ) + rotated = text_field(body, "refresh_credential") if not access or not organization_id or not rotated: - raise CloudSessionError("Engraphis Cloud returned incomplete session credentials.") + # Also post-response: the submitted credential is spent and no rotation was + # saved, so this must not be reported as a retryable outage either. + raise CloudSessionError( + "Engraphis Cloud returned incomplete session credentials, so the rotated " + "credential could not be saved. Connect this installation again.", + status=409, + ) response_subject = _validated_token_subject( body.get("token_subject") or token_subject ) + server_compute = _server_compute_url(body) + if server_compute and compute_source != "explicit": + compute = server_compute + compute_source = "server" updated = dict(saved) updated.update({ "schema": "engraphis-cloud-session/v1", @@ -536,24 +819,35 @@ def access_for_workspace( "compute_url": compute, "organization_id": organization_id, "refresh_credential": rotated, - "refresh_expires_at": str(body.get("refresh_expires_at") or ""), + "refresh_expires_at": text_field(body, "refresh_expires_at"), "token_subject": response_subject, }) + if compute_source in {"explicit", "server", "fallback"}: + updated["compute_url_source"] = compute_source # The refresh response carries the same entitlement fields as registration, so the # plan re-confirms itself on every token rotation. An older cloud omits them and # the previously persisted answer (if any) is left untouched. declared = _declared_entitlement(body) - if declared and "cloud_features" not in declared: - # A *plan change* may never inherit the previous plan's grants. - # ``_declared_entitlement`` omits ``cloud_features`` whenever the body carried - # no feature list, so merging it onto the saved record left a Team feature list - # alive underneath a downgraded Pro plan and kept the Team tab unlocked - # indefinitely. Dropping the stale key hands the answer back to this client's - # own plan table, which is right for the plan the cloud just named. A refresh - # that re-confirms the *same* plan still keeps the richer saved list. + if declared: + # A *plan change* may never inherit the previous plan's state. + # ``_declared_entitlement`` omits any field the body did not carry, so merging + # it onto the saved record left a Team feature list alive underneath a + # downgraded Pro plan and kept the Team tab unlocked indefinitely. Dropping + # every entitlement key the new answer did not restate hands those back to this + # client's own defaults, which are right for the plan the cloud just named -- + # and stops a finished trial's ``is_trial``/``trial_ends_at`` surviving under + # the paid plan it converted into. A refresh that re-confirms the *same* plan + # still keeps the richer saved answer. previous = str(saved.get("plan") or "").strip().lower() if previous != declared["plan"]: - updated.pop("cloud_features", None) + for key in _ENTITLEMENT_KEYS: + if key not in declared: + updated.pop(key, None) updated.update(declared) _save(updated) + if require_compute and not compute: + raise CloudSessionError( + "Engraphis Cloud did not provide a compute endpoint for this installation.", + status=503, + ) return access, organization_id, compute diff --git a/engraphis/commercial_manifest.json b/engraphis/commercial_manifest.json index 42c06ff..cbb4980 100644 --- a/engraphis/commercial_manifest.json +++ b/engraphis/commercial_manifest.json @@ -1,6 +1,6 @@ { "schema": "engraphis-commercial/v2", - "version": "1.0.1", + "version": "1.1.0", "control_plane": "https://api.engraphis.com", "managed_dashboard": "https://team.engraphis.com", "billing": { diff --git a/engraphis/config.py b/engraphis/config.py index 4578837..d3a95f6 100644 --- a/engraphis/config.py +++ b/engraphis/config.py @@ -1,6 +1,7 @@ """Central configuration — all values sourced from env with safe defaults.""" from __future__ import annotations +import errno import json import hashlib import os @@ -8,6 +9,7 @@ import sqlite3 import stat import sys +import time from contextlib import contextmanager import uuid from dataclasses import dataclass, field @@ -35,6 +37,7 @@ _PROJECT_ROOT = Path(__file__).resolve().parent.parent _DEFAULT_DB_NOTICES = set() +_WINDOWS_LOCK_RETRY_SECONDS = 0.05 def _default_db_path(root: Path = _PROJECT_ROOT, *, os_name: Optional[str] = None, @@ -221,6 +224,21 @@ def _cleanup_stale_migration_stages(target: Path) -> None: pass +def _lock_windows_migration_file(handle, msvcrt) -> None: + """Acquire the CRT lock even when its finite internal wait window expires.""" + while True: + try: + msvcrt.locking(handle.fileno(), msvcrt.LK_LOCK, 1) + return + except OSError as exc: + # ``LK_LOCK`` only retries internally for about ten seconds. Another process + # may still be performing a legitimate first-run migration, so keep waiting + # for lock contention but surface all filesystem/programming failures. + if exc.errno not in (errno.EACCES, errno.EDEADLK): + raise + time.sleep(_WINDOWS_LOCK_RETRY_SECONDS) + + @contextmanager def _migration_lock(target: Path): """Serialize first-run migration across processes without a third-party lock.""" @@ -261,9 +279,17 @@ def _migration_lock(target: Path): handle.seek(0, os.SEEK_END) if handle.tell() == 0: handle.write(b"\0") - handle.flush() + # Retry flush on Windows to handle concurrent file access + for attempt in range(10): + try: + handle.flush() + break + except PermissionError: + if attempt == 9: + raise + time.sleep(0.01 * (2 ** attempt)) # Exponential backoff handle.seek(0) - msvcrt.locking(handle.fileno(), msvcrt.LK_LOCK, 1) + _lock_windows_migration_file(handle, msvcrt) else: import fcntl fcntl.flock(handle.fileno(), fcntl.LOCK_EX) @@ -280,7 +306,18 @@ def _migration_lock(target: Path): import fcntl fcntl.flock(handle.fileno(), fcntl.LOCK_UN) finally: - handle.close() + # Retry close on Windows to handle concurrent file access + if os.name == "nt": + for attempt in range(10): + try: + handle.close() + break + except PermissionError: + if attempt == 9: + raise + time.sleep(0.01 * (2 ** attempt)) # Exponential backoff + else: + handle.close() def _prepare_installed_db_default_unlocked(root: Path, target: Path) -> Path: diff --git a/engraphis/dashboard_app.py b/engraphis/dashboard_app.py index 804f1b0..e013d1f 100644 --- a/engraphis/dashboard_app.py +++ b/engraphis/dashboard_app.py @@ -23,6 +23,7 @@ from engraphis.service import MemoryService _STATIC = Path(__file__).resolve().parent / "static" +_V2_ASSETS = Path(__file__).resolve().parent / "dashboard_assets" _INDEX = _STATIC / "index.html" # The public package is a single-user local runtime. Hosted account, Team, trial, and @@ -231,6 +232,10 @@ async def _auth_gate(request: Request, call_next): ) return await call_next(request) + # New dashboard capabilities belong to the v2 application surface. The old ``static`` + # directory remains mounted for the legacy shell and compatibility adapters only. + if _V2_ASSETS.is_dir(): + app.mount("/v2-assets", StaticFiles(directory=str(_V2_ASSETS)), name="v2-assets") if _STATIC.is_dir(): app.mount("/static", StaticFiles(directory=str(_STATIC)), name="static") diff --git a/engraphis/dashboard_assets/__init__.py b/engraphis/dashboard_assets/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/engraphis/dashboard_assets/engraphis-graph.js b/engraphis/dashboard_assets/engraphis-graph.js new file mode 100644 index 0000000..2981256 --- /dev/null +++ b/engraphis/dashboard_assets/engraphis-graph.js @@ -0,0 +1,1074 @@ +/* Engraphis knowledge graph — the unified v2 dashboard's opt-in force-graph engine. + `engraphis.dashboard_app` owns this renderer and serves it from `/v2-assets`; the legacy + server exposes only the compatibility marker in `engraphis/static/engraphis-graph.js`. + + Restores the shipped behaviour: GRAPH_PRESETS, GSTYLE render modes (cyber/galaxy/solar/classic), + STYLE_PAL / STYLE_LAYERS / STYLE_BG, COMMUNITY_PALS, GRAPH_HEAT, colour-by community/type/connections, + GRAPH_PALETTES with per-entity-type overrides, d3 force wiring, directional particles, label ranking, + hover neighbourhood highlight, freeze, fit and reheat. Values copied from dashboard.js. + + The public graph endpoint calls its fields `label`, `from` and `to`; the engine also + accepts the renderer-friendly `name`, `source` and `target` aliases so it can be used + with both the dashboard adapter and standalone scene payloads. */ +(function () { + const PRESETS = { + original: { label: 'Original force', repel: 120, link: 30, gravity: 14, font: 13, size: 3, linkw: 1, labelDensity: 40, curve: 0, particles: 0 }, + compact: { label: 'Compact clusters', repel: 42, link: 20, gravity: 26, font: 12, size: 3, linkw: 0.7, labelDensity: 30, curve: 0.08, particles: 0 }, + communities: { label: 'Community islands', repel: 48, link: 16, gravity: 48, font: 12, size: 3, linkw: 0.72, labelDensity: 24, curve: 0.12, particles: 0 }, + radial: { label: 'Radial orbit', repel: 68, link: 26, gravity: 12, font: 13, size: 3, linkw: 0.75, labelDensity: 55, curve: 0.22, particles: 0 }, + constellation: { label: 'Constellation flow', repel: 34, link: 16, gravity: 38, font: 12, size: 3, linkw: 0.65, labelDensity: 35, curve: 0.32, particles: 2 }, + custom: { label: 'Custom tuning', curve: 0.1, particles: 0 } + }; + + const STYLE_PAL = { + galaxy: { person_or_concept: '#b789ff', mention: '#7bb4ff', hashtag: '#ffcf6b', email: '#8aa2ff', organization: '#66e0d0', location: '#ff7ea8' }, + solar: { person_or_concept: '#ffb454', mention: '#3fd2c7', hashtag: '#ffd68a', email: '#8ea8ff', organization: '#5b9bff', location: '#ff8f6b' }, + cyber: { person_or_concept: '#ff3ea5', mention: '#b6ff3c', hashtag: '#ffe14d', email: '#8b7bff', organization: '#22e0ff', location: '#ff5c7a' } + }; + const STYLE_LAYERS = { + classic: { temporal: '#6f9fd8', entity: '#5aafb3', causal: '#d7a84b', semantic: '#8c83e8' }, + galaxy: { temporal: '#7bb4ff', entity: '#66e0d0', causal: '#ffcf6b', semantic: '#b789ff' }, + solar: { temporal: '#5b9bff', entity: '#3fd2c7', causal: '#ffb454', semantic: '#ffd68a' }, + cyber: { temporal: '#22e0ff', entity: '#b6ff3c', causal: '#ffe14d', semantic: '#ff3ea5' } + }; + /* The per-style pane backgrounds are NOT defined here. `style-src-attr 'none'` forbids + writing them onto the element, so dashboard.css owns them behind + `#graph-net[data-graph-style="galaxy|solar|cyber"]` and this file only sets that + attribute. Keeping a second copy of the gradients in JS would be dead drift. */ + const PALETTES = { + theme: null, + aurora: { person_or_concept: '#8b7cf6', mention: '#2dd4bf', hashtag: '#fbbf24', email: '#60a5fa', organization: '#f472b6', location: '#a3e635' }, + ocean: { person_or_concept: '#38bdf8', mention: '#2dd4bf', hashtag: '#facc15', email: '#818cf8', organization: '#22d3ee', location: '#34d399' }, + ember: { person_or_concept: '#f97316', mention: '#fb7185', hashtag: '#facc15', email: '#a78bfa', organization: '#ef4444', location: '#84cc16' }, + contrast: { person_or_concept: '#0072b2', mention: '#009e73', hashtag: '#e69f00', email: '#56b4e9', organization: '#cc79a7', location: '#d55e00' } + }; + const THEME_ETYPE = { person_or_concept: '#8c83e8', mention: '#5aafb3', hashtag: '#d7a84b', email: '#6f9fd8', organization: '#58b882', location: '#df7478' }; + /* Community colour is the *palette slot*, not the node: `nodeColor` indexes this by the + community id, and communities are numbered by size (largest == 0). The legend beside the + canvas paints its swatches from `.graph-cluster-N` in dashboard.css, which encodes the + Cyber palette — the default style — slot for slot. These arrays must therefore stay + byte-identical to `COMMUNITY_PALS` in dashboard.js, or "Cluster 1" gets one colour in the + legend and another on the canvas. Ordering is load-bearing; this is not free-choice art. */ + const COMMUNITY_PALS = { + classic: ['#8c83e8', '#5aafb3', '#d7a84b', '#6f9fd8', '#58b882', '#df7478', '#b07de0', '#4fb0a0', '#e0894a', '#7c9be0', '#e06a9a', '#9ac25a'], + galaxy: ['#b789ff', '#7bb4ff', '#66e0d0', '#ffcf6b', '#ff7ea8', '#8aa2ff', '#c98bff', '#5ad0e0', '#ffa0d0', '#9d7bff', '#6ad0b0', '#ffb060'], + solar: ['#ffb454', '#5b9bff', '#3fd2c7', '#ffd68a', '#ff8f6b', '#8ea8ff', '#ffc24a', '#6ac0d0', '#ff9f7a', '#7ab0ff', '#e0b050', '#5fd0b0'], + cyber: ['#22e0ff', '#ff3ea5', '#b6ff3c', '#ffe14d', '#8b7bff', '#ff5c7a', '#3affd0', '#ff7be0', '#7affea', '#c0ff4a', '#5c9bff', '#ff9b3c'] + }; + const GRAPH_HEAT = ['#3f7bff', '#6a5cff', '#a24bff', '#e0479f', '#ff6b6b', '#ffc23d']; + + /* Flow particles are per *relation*, and force-graph advances every one of them on every + frame — three particles on a few thousand relations is tens of thousands of animated + objects and a canvas that stops responding. The classic renderer already refuses to draw + them past this many links (`data.links.length>800` in dashboard.js's graphRender); the + opt-in engine uses the same cutoff rather than inventing a second large-graph signal. */ + const PARTICLE_LINK_LIMIT = 800; + + /* The classic renderer's large-graph signal (`GPERF` in dashboard.js, set from the rendered + data as `nodes>600 || links>2400`). Past it the classic path drops the galaxy starfield + outright — `if(GPERF.large)return` in graphStyleBackground — because repainting 110 stars + plus every node and link on every frame is what makes a big store unusable. The opt-in + engine reuses the same thresholds rather than inventing a second signal. */ + const LARGE_NODE_LIMIT = 600; + const LARGE_LINK_LIMIT = 2400; + + /* The classic renderer's *dense* signal (`GPERF.dense`, `links>1500` in dashboard.js). Past + it the classic path turns off the two per-edge costs that scale with the link count and + buy nothing at that density: link curvature (a quadratic bezier per relation instead of a + straight line) and the directional arrowhead (a filled triangle per relation, recomputed + every frame). Relation labels get the same treatment unless one node is highlighted. Same + thresholds and same behaviour here — a second signal would only drift. */ + const DENSE_LINK_LIMIT = 1500; + + /* Relation labels are the noisiest layer on the canvas, so — exactly as the classic + `linkCanvasObject` does — they only appear once the user has zoomed in past this scale. */ + const LINK_LABEL_MIN_SCALE = 2.4; + + function idOf(value) { return value && typeof value === 'object' ? value.id : value; } + function nodeName(node) { return String(node.name || node.label || node.id || ''); } + function linkEndpoint(link, side) { + return idOf(link[side] !== undefined ? link[side] : link[side === 'source' ? 'from' : 'to']); + } + function asOfValue(value) { + if (value instanceof Date) return value.getTime(); + if (typeof value === 'number') return Number.isFinite(value) ? value * (value < 1e11 ? 1000 : 1) : null; + if (typeof value === 'string' && value.trim()) { + const numeric = Number(value); + if (Number.isFinite(numeric)) return asOfValue(numeric); + const parsed = Date.parse(value); + return Number.isFinite(parsed) ? parsed : null; + } + return null; + } + function temporalValue(item, key, fallback) { + const value = item[key] !== undefined ? item[key] : item[key === 'valid_from' ? 'born' : 'closed']; + if (value === undefined || value === null || value === '') return fallback; + return asOfValue(value); + } + + /* Node and link labels come from ingested memories, i.e. untrusted text. force-graph's + tooltip renders a string label through `innerHTML` (see float-tooltip in + vendor/force-graph.min.js), so every label handed to it must already be escaped. */ + function esc(value) { + if (value === undefined || value === null) return ''; + return String(value) + .replace(/&/g, '&').replace(//g, '>') + .replace(/"/g, '"').replace(/'/g, '''); + } + + function hexRgb(c) { + if (!c) return [140, 131, 232]; + if (c[0] === '#') { + const hex = c.length === 4 ? c[1] + c[1] + c[2] + c[2] + c[3] + c[3] : c.slice(1, 7); + const n = parseInt(hex, 16); + if (!Number.isFinite(n)) return [140, 131, 232]; + return [n >> 16 & 255, n >> 8 & 255, n & 255]; + } + const m = c.match(/\d+/g) || [140, 131, 232]; + return [+m[0], +m[1], +m[2]]; + } + function alpha(c, a) { const [r, g, b] = hexRgb(c); return 'rgba(' + r + ',' + g + ',' + b + ',' + a + ')'; } + function lighten(c, amt) { let [r, g, b] = hexRgb(c); r = Math.round(r + (255 - r) * amt); g = Math.round(g + (255 - g) * amt); b = Math.round(b + (255 - b) * amt); return 'rgb(' + r + ',' + g + ',' + b + ')'; } + function contrastOn(c) { const [r, g, b] = hexRgb(c); return (0.2126 * r + 0.7152 * g + 0.0722 * b) > 150 ? '#111827' : '#f8fafc'; } + + function makeStars() { + const a = [], c = ['#dfe6ff', '#dfe6ff', '#c9b6ff', '#a7c6ff', '#ffd9ef']; + for (let i = 0; i < 110; i++) a.push({ x: (Math.random() - 0.5) * 1200, y: (Math.random() - 0.5) * 1200, r: Math.random() * 1.1 + 0.25, a: Math.random() * 0.7 + 0.25, tw: Math.random() * 1.6 + 0.4, ph: Math.random() * 6.28, c: c[i % c.length] }); + return a; + } + const STARS = makeStars(); + + /* Relations that cross topics rather than describe one. The classic renderer keeps them + visible and traversable but builds its *clustering* adjacency without them (`GCOMM_ADJ` + in dashboard.js), because a single sparse `influences` edge otherwise fuses two unrelated + topics into one connected component — one Community-Islands colour and one force centre + for both. Same semantics here. */ + const CLUSTER_EXCLUDED_LABELS = { influences: true }; + function clustersAcross(link) { + return !!(link && CLUSTER_EXCLUDED_LABELS[link.label]); + } + + function communities(nodes, links) { + const adj = {}; + // Traversal adjacency (hover neighbourhood, focus depth, bridges, betweenness) keeps every + // relation; only the community BFS below reads `clusterAdj`. + const clusterAdj = {}; + const nodesById = new Map(nodes.map(node => [node.id, node])); + nodes.forEach(n => { adj[n.id] = []; clusterAdj[n.id] = []; }); + links.forEach(l => { + const s = linkEndpoint(l, 'source'), t = linkEndpoint(l, 'target'); + if (adj[s]) adj[s].push(t); + if (adj[t]) adj[t].push(s); + if (clustersAcross(l)) return; + if (clusterAdj[s]) clusterAdj[s].push(t); + if (clusterAdj[t]) clusterAdj[t].push(s); + }); + // Respect clusters supplied with the data (a store that already knows its topics); + // otherwise fall back to connected-component BFS, as the dashboard does. + if (nodes.length && nodes.every(n => typeof n.community === 'number')) return adj; + const seen = new Set(); + const groups = []; + nodes.forEach(n => { + if (seen.has(n.id)) return; + // Read head instead of Array#shift: shift() is O(n) per pop, which turns this BFS + // quadratic on the large stores the dashboard is expected to open. + const queue = [n.id]; + let head = 0; + seen.add(n.id); + while (head < queue.length) { + const id = queue[head++]; + (clusterAdj[id] || []).forEach(next => { if (!seen.has(next)) { seen.add(next); queue.push(next); } }); + } + // `queue` has accumulated the whole component by now, so it *is* the group. + groups.push(queue); + }); + /* Rank by size before the IDs become visible. `graphRenderLegend()` sorts communities by + size and labels the largest "Cluster 1", while node colour indexes the palette by the + community ID itself (`nodeColor` -> `commPal()[community % n]`). Assigning IDs in raw + node order therefore let the legend describe one component with another's swatch + whenever a smaller component happened to appear first in the payload. The classic + renderer sorts its components the same way (`graphComputeCommunities` in dashboard.js), + so largest == community 0 == palette slot 0 == "Cluster 1" on both paths. */ + groups.sort((a, b) => b.length - a.length); + groups.forEach((group, index) => { + group.forEach(id => { const node = nodesById.get(id); if (node) node.community = index; }); + }); + return adj; + } + + function maxOf(values, floor) { + // Math.max(...array) throws RangeError once the array outgrows the argument limit, + // which a real store reaches long before the renderer gets slow. + let best = floor; + for (let i = 0; i < values.length; i++) if (values[i] > best) best = values[i]; + return best; + } + + /* Brandes betweenness — which entity is the bridge whose loss would split a topic. + Brandes is O(V·E); on a large store that is seconds of blocked main thread, so above + BETWEENNESS_PIVOTS sources we run the standard pivot approximation over a deterministic, + evenly-spaced sample. The score is only ever used as a *relative* size/highlight signal + (it is normalised to the maximum), so a sampled estimate is fit for purpose. */ + const BETWEENNESS_PIVOTS = 220; + const BETWEENNESS_BUDGET = 1.5e6; + function betweenness(nodes, adj) { + const bc = {}; + nodes.forEach(n => { bc[n.id] = 0; }); + // Each pivot costs O(V) just to initialise its bookkeeping, so cap pivots by total work + // as well as by count: without the budget a 60k-entity store blocks the main thread for + // ~25s. This is a relative sizing signal, so fewer pivots degrades quality, not truth. + const pivots = Math.max(1, Math.min( + BETWEENNESS_PIVOTS, + Math.floor(BETWEENNESS_BUDGET / Math.max(1, nodes.length)) + )); + const stride = nodes.length > pivots ? Math.ceil(nodes.length / pivots) : 1; + for (let index = 0; index < nodes.length; index += stride) { + const src = nodes[index]; + const stack = [], pred = {}, sigma = {}, dist = {}, delta = {}; + nodes.forEach(n => { pred[n.id] = []; sigma[n.id] = 0; dist[n.id] = -1; delta[n.id] = 0; }); + sigma[src.id] = 1; dist[src.id] = 0; + const queue = [src.id]; + let head = 0; + while (head < queue.length) { + const v = queue[head++]; + stack.push(v); + (adj[v] || []).forEach(w => { + if (dist[w] < 0) { dist[w] = dist[v] + 1; queue.push(w); } + if (dist[w] === dist[v] + 1) { sigma[w] += sigma[v]; pred[w].push(v); } + }); + } + while (stack.length) { + const w = stack.pop(); + pred[w].forEach(v => { delta[v] += (sigma[v] / sigma[w]) * (1 + delta[w]); }); + if (w !== src.id) bc[w] += delta[w]; + } + } + const max = maxOf(Object.values(bc), 1); + nodes.forEach(n => { n.betweenness = bc[n.id] / max; }); + return bc; + } + + /* Bridge edges (Tarjan): removing one disconnects part of the store. */ + function findBridges(nodes, links, adj) { + const disc = {}, low = {}, parent = {}, bridges = new Set(); + const multiplicity = {}; + links.forEach(link => { + const s = linkEndpoint(link, 'source'), t = linkEndpoint(link, 'target'); + const key = s < t ? s + '|' + t : t + '|' + s; + multiplicity[key] = (multiplicity[key] || 0) + 1; + }); + let timer = 0; + // Iterative Tarjan. The recursive form recurses once per node along a path, so a + // chain-shaped component of a few thousand entities overflows the call stack and takes + // the whole render down with it — an explicit frame stack has no such ceiling. + const visit = root => { + const frames = [{ u: root, i: 0 }]; + disc[root] = low[root] = ++timer; + while (frames.length) { + const frame = frames[frames.length - 1]; + const u = frame.u, neighbors = adj[u] || []; + if (frame.i < neighbors.length) { + const v = neighbors[frame.i++]; + if (!disc[v]) { + parent[v] = u; + disc[v] = low[v] = ++timer; + frames.push({ u: v, i: 0 }); + } else if (v !== parent[u]) { + low[u] = Math.min(low[u], disc[v]); + } + continue; + } + frames.pop(); + const p = parent[u]; + if (p !== undefined) { + low[p] = Math.min(low[p], low[u]); + const key = p < u ? p + '|' + u : u + '|' + p; + if (low[u] > disc[p] && multiplicity[key] === 1) { + bridges.add(p + '|' + u); + bridges.add(u + '|' + p); + } + } + } + }; + nodes.forEach(n => { if (!disc[n.id]) visit(n.id); }); + links.forEach(l => { + const s = linkEndpoint(l, 'source'), t = linkEndpoint(l, 'target'); + l.bridge = bridges.has(s + '|' + t); + }); + return bridges; + } + + function create(el, options) { + if (typeof ForceGraph === 'undefined') throw new Error('force-graph not loaded'); + if (!el || typeof el.getAttribute !== 'function') throw new Error('graph container missing'); + const opts = options || {}; + const state = { + // Named `styleName`, not `style`: scripts/externalize_dashboard_assets.py scans this + // asset for runtime inline-style mutation with a text pattern, and a plain data field + // by the shorter name reads as one. The longer name keeps that gate honest. + styleName: 'cyber', colorBy: 'community', palette: 'theme', overrides: {}, themeColors: {}, + settings: Object.assign({}, PRESETS.communities, { mode: 'communities', labels: false, flow: true, frozen: false }), + minDegree: 1, showUnlinked: false, focusId: null, depth: 2, layers: { temporal: true, entity: true, causal: true, semantic: true, code: false }, + path: null, asOf: null, ghost: true, sizeBy: 'degree', bridges: false, suggestions: false, collapse: 'auto' + }; + let raw = { nodes: [], links: [], suggestions: [] }, adj = {}, hilite = null, hoverSet = null, maxDeg = 1; + // Match the classic renderer's ranked label cap. Computing the selected IDs once per + // render also keeps per-frame canvas work bounded on dense graphs. + let labelIds = new Set(); + let zoom = 1, collapsed = false; + /* Recomputed from the *rendered* data on every render, exactly as the classic path + recomputes GPERF — filters and focus can take a huge store down to a small view. */ + let large = false, dense = false; + /* The node/link arrays last handed to force-graph. Seeding is not free: the vendor copies + the data in and d3 resets the simulation alpha to 1, so a paint-only change would restart + the whole layout. See `sameData`/`render`. */ + let seeded = null; + let destroyed = false, running = true, fitTimer = 0, suspended = 0, pendingRender = null; + let betweennessReady = false; + const fg = ForceGraph()(el); + const api = {}; + + /* The dashboard already honours `prefers-reduced-motion` for the classic renderer; this + engine must not quietly reintroduce perpetual motion for the same user. */ + function reduced() { + if (typeof opts.reducedMotion === 'function') return !!opts.reducedMotion(); + try { + return !!(window.matchMedia && window.matchMedia('(prefers-reduced-motion: reduce)').matches); + } catch (e) { return false; } + } + /* force-graph already keeps redrawing while the simulation runs or any link still has + particles in flight, so `autoPauseRedraw(false)` is only needed for paint this engine + does behind its back: the galaxy starfield lives in onRenderFramePre and is invisible + to that change detection. Everywhere else, letting force-graph park the redraw is what + keeps a settled graph off the CPU. */ + function needsContinuousFrames() { + return !reduced() && state.styleName === 'galaxy' && !large; + } + /* Betweenness is the one analysis that is superlinear in the store size, and nothing in + the default view consumes it — the bridge overlay and betweenness-sizing are both off. + Computing it lazily keeps opening the graph cheap; the first toggle pays for it once. */ + function ensureBetweenness() { + if (betweennessReady) return; + betweennessReady = true; + betweenness(raw.nodes, adj); + } + /* Apply a batch of setters with exactly one render at the end. Each public setter renders + on its own, so a single dashboard sync used to cost six full re-simulations (and six + zoom-to-fit timers). The caller also states the intent explicitly, because the merged + intent of the individual setters is not the caller's: `setSettings` asks for a reheat + whenever the patch carries a physics key, and the dashboard's sync hands it the whole + GSET — so it would reheat even on a `render(false, false)` refresh. */ + function batch(fn, fit, reheat) { + suspended++; + try { fn(api); } finally { + suspended--; + pendingRender = null; + render(!!fit, !!reheat); + } + } + + /* Priority mirrors the classic renderer's graphTypeColor(): an explicit user override wins, + then a non-classic style's own palette, then the *active theme*. The theme tier is the + reason `themeColors` exists — it cannot be folded into `overrides`, which outrank + STYLE_PAL. The dashboard owns the CSS custom properties (`--entity-*`), so it supplies + the resolved values through setThemeColors() on every applyTheme()/graphRecolor(); + THEME_ETYPE stays only as the standalone-embed fallback for a caller that never does. */ + function etypeColor(type) { + if (state.overrides[type]) return state.overrides[type]; + if (state.styleName !== 'classic' && STYLE_PAL[state.styleName] && STYLE_PAL[state.styleName][type]) return STYLE_PAL[state.styleName][type]; + return state.themeColors[type] || THEME_ETYPE[type] || '#8c83e8'; + } + function commPal() { return COMMUNITY_PALS[state.styleName] || COMMUNITY_PALS.classic; } + function heatColor(node) { + const t = (node.rank || 0) / Math.max(1, raw.nodes.length - 1); + return GRAPH_HEAT[Math.min(GRAPH_HEAT.length - 1, Math.floor(t * GRAPH_HEAT.length))]; + } + function nodeColor(node) { + if (state.colorBy === 'community') { const p = commPal(); return p[(node.community || 0) % p.length]; } + if (state.colorBy === 'connections') return heatColor(node); + return etypeColor(node.etype); + } + function layerColor(layer) { return (STYLE_LAYERS[state.styleName] || STYLE_LAYERS.classic)[layer] || '#8c83e8'; } + + function born(item) { return temporalValue(item, 'valid_from', -Infinity); } + function closed(item) { return temporalValue(item, 'valid_to', null); } + function aliveAt(item, date) { + const start = born(item), end = closed(item); + return start <= date && (end === null || end > date); + } + + function collapsedData(nodes, links) { + const groups = {}; + nodes.forEach(n => { + const c = n.community || 0; + if (!groups[c]) groups[c] = { id: 'cluster-' + c, cluster: true, community: c, name: (n.topic || 'Cluster ' + (c + 1)), etype: n.etype, members: 0, degree: 0, betweenness: 0 }; + groups[c].members++; + groups[c].degree += n.degree || 0; + groups[c].betweenness = Math.max(groups[c].betweenness, n.betweenness || 0); + }); + const cnodes = Object.values(groups); + const seen = {}; + const clinks = []; + // Indexed lookup, not Array#find per endpoint: auto-collapse fires on every zoom-out, + // and the scan made that O(nodes x links) — a visible freeze on a real store. + const byId = new Map(raw.nodes.map(n => [n.id, n])); + links.forEach(l => { + const s = byId.get(linkEndpoint(l, 'source')); + const t = byId.get(linkEndpoint(l, 'target')); + if (!s || !t) return; + const a = 'cluster-' + (s.community || 0), b = 'cluster-' + (t.community || 0); + if (a === b) return; + const key = a < b ? a + '|' + b : b + '|' + a; + if (seen[key]) { seen[key].weight++; return; } + const link = { source: a, target: b, layer: l.layer, weight: 1, aggregate: true }; + seen[key] = link; + clinks.push(link); + }); + return { nodes: cnodes, links: clinks }; + } + + function visible() { + const keepLayer = l => state.layers[l.layer] !== false; + let nodes = raw.nodes.filter(n => (state.showUnlinked || n.degree > 0) && n.degree >= state.minDegree); + if (state.repo) nodes = nodes.filter(n => (n.repo || n.topic || '').toLowerCase().includes(state.repo) || nodeName(n).toLowerCase().includes(state.repo)); + if (state.asOf !== null) { + const live = nodes.filter(n => aliveAt(n, state.asOf)); + const ghosts = state.ghost ? nodes.filter(n => !aliveAt(n, state.asOf) && born(n) <= state.asOf).map(n => Object.assign(n, { ghost: true })) : []; + live.forEach(n => { n.ghost = false; }); + nodes = live.concat(ghosts); + } else { + nodes.forEach(n => { n.ghost = false; }); + } + if (state.focusId != null) { + const keep = new Set([state.focusId]); + let frontier = [state.focusId]; + for (let h = 0; h < state.depth; h++) { + const next = []; + frontier.forEach(id => (adj[id] || []).forEach(n => { if (!keep.has(n)) { keep.add(n); next.push(n); } })); + frontier = next; + } + nodes = nodes.filter(n => keep.has(n.id)); + } + const ids = new Set(nodes.map(n => n.id)); + let links = raw.links.filter(l => keepLayer(l) && ids.has(linkEndpoint(l, 'source')) && ids.has(linkEndpoint(l, 'target'))); + if (state.asOf !== null) { + links.forEach(l => { l.ghost = !aliveAt(l, state.asOf); }); + if (!state.ghost) links = links.filter(l => !l.ghost); + links = links.filter(l => born(l) <= state.asOf); + } else { + links.forEach(l => { l.ghost = false; }); + } + if (state.suggestions && raw.suggestions) { + raw.suggestions.forEach(s => { + const source = linkEndpoint(s, 'source'), target = linkEndpoint(s, 'target'); + if (ids.has(source) && ids.has(target)) links = links.concat([Object.assign({}, s, { source, target, layer: 'semantic', suggested: true })]); + }); + } + if (collapsed) return collapsedData(nodes, links.filter(l => !l.suggested)); + return { nodes, links }; + } + + function applyForces() { + const s = state.settings, mode = s.mode || 'compact'; + fg.d3Force('charge').strength(-s.repel); + fg.d3Force('link').distance(s.link); + if (typeof d3 === 'undefined') return; + fg.d3Force('radial', null); + if (mode === 'communities') { + const target = node => { + const c = node.community || 0; + const ring = 240 + (c % 3) * 90; + const a = (c * 2.399); + return { x: Math.cos(a) * ring, y: Math.sin(a) * ring * 0.62 }; + }; + fg.d3Force('x', d3.forceX(n => target(n).x).strength(s.gravity / 100)); + fg.d3Force('y', d3.forceY(n => target(n).y).strength(s.gravity / 100)); + } else { + const centering = mode === 'radial' ? Math.max(0.04, s.gravity / 300) : s.gravity / 100; + fg.d3Force('x', d3.forceX(0).strength(centering)); + fg.d3Force('y', d3.forceY(0).strength(centering)); + if (mode === 'radial' && d3.forceRadial) fg.d3Force('radial', d3.forceRadial(n => Math.max(0, 5 - Math.min(5, n.degree || 0)) * Math.max(8, s.link * 0.72)).strength(0.32)); + } + /* One collision pass on a large graph, two otherwise — the classic path's + `.iterations(GPERF.large?1:2)`. The second pass costs another full quadtree traversal + per node on every tick, and a large store pays that on the initial layout and on every + reheat, which is exactly where it is least affordable. */ + if (d3.forceCollide) fg.d3Force('collide', d3.forceCollide(n => n.radius + 1.5).iterations(large ? 1 : 2)); + } + + function styleBackground(ctx, scale) { + if (state.styleName === 'galaxy') { + /* Matches the classic path's `if(GPERF.large)return`. Paired with the `large` term in + needsContinuousFrames(), this is what lets a big galaxy graph settle: the starfield + is the only paint force-graph cannot see, so once it is skipped there is nothing + left that requires a frame the vendor would not have scheduled itself. */ + if (large) return; + const t = performance.now() / 1000; + ctx.save(); + ctx.globalCompositeOperation = 'lighter'; + for (let i = 0; i < STARS.length; i++) { + const s = STARS[i], al = s.a * (0.5 + 0.5 * Math.sin(t * s.tw + s.ph)); + if (al <= 0.02) continue; + ctx.globalAlpha = al; + ctx.beginPath(); + ctx.arc(s.x, s.y, s.r, 0, 6.2832); + ctx.fillStyle = s.c; + ctx.fill(); + } + ctx.restore(); + } else if (state.styleName === 'solar') { + ctx.save(); + const g = ctx.createRadialGradient(0, 0, 2, 0, 0, 130); + g.addColorStop(0, 'rgba(255,192,112,.20)'); + g.addColorStop(0.6, 'rgba(255,150,80,.05)'); + g.addColorStop(1, 'rgba(255,150,80,0)'); + ctx.fillStyle = g; + ctx.beginPath(); + ctx.arc(0, 0, 130, 0, 6.2832); + ctx.fill(); + ctx.strokeStyle = 'rgba(255,190,120,.10)'; + ctx.lineWidth = 1 / scale; + [72, 132, 200, 286, 384].forEach(r => { ctx.beginPath(); ctx.ellipse(0, 0, r, r * 0.66, 0, 0, 6.2832); ctx.stroke(); }); + ctx.restore(); + } + } + + function styleNode(node, ctx, scale) { + if (!Number.isFinite(node.x) || !Number.isFinite(node.y)) return; + const focus = hoverSet && hoverSet.size > 1, neighbor = focus && hoverSet.has(node.id), dim = focus && !neighbor; + let r = node.radius; + const col = node.color, rich = node.id === hilite || neighbor || node.hub || node.degree >= 3; + ctx.globalAlpha = node.ghost ? 0.22 : (dim ? 0.12 : 1); + if (node.ghost) { + ctx.lineWidth = 1.1 / scale; + ctx.strokeStyle = col; + ctx.beginPath(); ctx.arc(node.x, node.y, r, 0, 6.2832); ctx.stroke(); + ctx.globalAlpha = 1; + return; + } + if (node.cluster) { + const g = ctx.createRadialGradient(node.x, node.y, r * 0.2, node.x, node.y, r * 1.5); + g.addColorStop(0, alpha(col, 0.9)); + g.addColorStop(0.7, alpha(col, 0.35)); + g.addColorStop(1, alpha(col, 0)); + ctx.fillStyle = g; + ctx.beginPath(); ctx.arc(node.x, node.y, r * 1.5, 0, 6.2832); ctx.fill(); + ctx.fillStyle = contrastOn(col); + ctx.font = '600 ' + Math.max(3, r * 0.55) + 'px system-ui, sans-serif'; + ctx.textAlign = 'center'; + ctx.textBaseline = 'middle'; + ctx.fillText(String(node.members), node.x, node.y); + ctx.font = '500 ' + Math.max(2.6, r * 0.4) + 'px system-ui, sans-serif'; + // Cluster names sit outside the coloured bubble. They therefore need the active + // theme's text colour, not the dark-theme near-white that disappears on light canvas. + ctx.fillStyle = state.themeColors.label || '#e7e9ee'; + ctx.fillText(nodeName(node), node.x, node.y + r * 1.5 + r * 0.5); + ctx.textAlign = 'left'; + ctx.globalAlpha = 1; + return; + } + if (state.bridges && node.betweenness > 0.35) { + ctx.save(); + ctx.strokeStyle = alpha('#ff5c7a', 0.75); + ctx.lineWidth = 1.2 / scale; + ctx.setLineDash([2 / scale, 2 / scale]); + ctx.beginPath(); ctx.arc(node.x, node.y, r + 3 / scale, 0, 6.2832); ctx.stroke(); + ctx.restore(); + } + /* Every per-node glow below is gated on `!large`, matching the classic path's + `if(rich&&!GPERF.large)` / `if(!GPERF.large)` pairs. A radial gradient or a shadow blur + is per-node, per-frame canvas work: at the >600-node cutoff that is hundreds of + gradients rebuilt on every tick of the layout, which is what makes a big store crawl. + The flat fill in the `else` is the classic fallback, not a missing branch. */ + if (state.styleName === 'galaxy') { + if (rich && !large) { + ctx.save(); + ctx.globalCompositeOperation = 'lighter'; + const R = r * (node.id === hilite ? 4.4 : 3.0); + const g = ctx.createRadialGradient(node.x, node.y, 0, node.x, node.y, R); + g.addColorStop(0, alpha(col, dim ? 0.15 : 0.6)); + g.addColorStop(0.42, alpha(col, dim ? 0.05 : 0.16)); + g.addColorStop(1, alpha(col, 0)); + ctx.fillStyle = g; + ctx.beginPath(); + ctx.arc(node.x, node.y, R, 0, 6.2832); + ctx.fill(); + ctx.restore(); + } + ctx.beginPath(); ctx.arc(node.x, node.y, r, 0, 6.2832); ctx.fillStyle = col; ctx.fill(); + ctx.beginPath(); ctx.arc(node.x, node.y, Math.max(0.4, r * 0.4), 0, 6.2832); ctx.fillStyle = 'rgba(255,255,255,.9)'; ctx.fill(); + } else if (state.styleName === 'solar') { + const sun = node.rank === 0; + if (sun) r *= 1.7; + if (rich && !large) { + ctx.save(); + ctx.globalCompositeOperation = 'lighter'; + const cc = sun ? '#ffcf6b' : col, R2 = r * (sun ? 3.4 : 2.1); + const g2 = ctx.createRadialGradient(node.x, node.y, 0, node.x, node.y, R2); + g2.addColorStop(0, alpha(cc, dim ? 0.1 : (sun ? 0.6 : 0.3))); + g2.addColorStop(1, alpha(cc, 0)); + ctx.fillStyle = g2; + ctx.beginPath(); ctx.arc(node.x, node.y, R2, 0, 6.2832); ctx.fill(); + ctx.restore(); + } + if (large) { + ctx.fillStyle = col; + } else { + const sg = ctx.createRadialGradient(node.x - r * 0.4, node.y - r * 0.4, Math.max(0.1, r * 0.12), node.x, node.y, r); + sg.addColorStop(0, lighten(sun ? '#ffe4ad' : col, 0.5)); + sg.addColorStop(1, sun ? '#e08a25' : col); + ctx.fillStyle = sg; + } + ctx.beginPath(); ctx.arc(node.x, node.y, r, 0, 6.2832); ctx.fill(); + } else if (state.styleName === 'cyber') { + ctx.save(); + if (rich && !large) { ctx.shadowColor = col; ctx.shadowBlur = dim ? 2 : r * 2.6; } + ctx.beginPath(); ctx.arc(node.x, node.y, r, 0, 6.2832); ctx.fillStyle = col; ctx.fill(); + ctx.restore(); + ctx.beginPath(); ctx.arc(node.x, node.y, Math.max(0.4, r * 0.42), 0, 6.2832); ctx.fillStyle = '#eafcff'; ctx.fill(); + } else { + ctx.beginPath(); ctx.arc(node.x, node.y, r, 0, 6.2832); ctx.fillStyle = col; ctx.fill(); + if (node.hub) { ctx.lineWidth = 0.8 / scale; ctx.strokeStyle = node.stroke; ctx.stroke(); } + } + if (node.id === hilite) { + ctx.lineWidth = 1.3 / scale; + ctx.strokeStyle = state.styleName === 'cyber' ? '#ffffff' : 'rgba(255,255,255,.9)'; + ctx.beginPath(); ctx.arc(node.x, node.y, r + 1.4 / scale, 0, 6.2832); ctx.stroke(); + } + const showLabel = (state.settings.labels && labelIds.has(node.id)) || node.id === hilite || neighbor; + if (showLabel && scale > 0.35) { + // The dashboard setting is a screen-space font size. As on the classic renderer, + // compensate only for graph zoom; an extra artistic divisor makes a configured 12px + // label unreadable at normal zoom and makes the Font size control misleading. + const size = state.settings.font / scale; + ctx.font = '500 ' + size + 'px system-ui, sans-serif'; + ctx.textBaseline = 'middle'; + ctx.fillStyle = 'rgba(0,0,0,.5)'; + ctx.fillText(nodeName(node), node.x + r + 1.6 + 0.3, node.y + 0.3); + // The canvas sits on both light and dark themes. A hard-coded pale label disappears + // on the light canvas, including for a highlighted node. + ctx.fillStyle = state.themeColors.label || '#e7e9ee'; + ctx.fillText(nodeName(node), node.x + r + 1.6, node.y); + } + ctx.globalAlpha = 1; + } + + function applyChrome() { + // Keep the asset compatible with `style-src-attr 'none'`: the CSP-safe dashboard + // stylesheet owns the visual backgrounds, while the canvas owns the data-driven paint. + el.setAttribute('data-graph-style', state.styleName); + } + + /* force-graph parks its redraw loop as soon as the simulation settles and no particle is in + flight (`autoPauseRedraw`), and it has no way to know that `hilite`/`hoverSet` — plain + closure state read by the paint callbacks — changed. Re-setting an accessor to its own + value is the vendor's own invalidation hook, so highlight changes still paint with + reduced motion on, flow off, or a settled graph. */ + function invalidate() { + if (destroyed) return; + fg.nodeCanvasObject(fg.nodeCanvasObject()); + } + + function refreshColors() { + const nodes = fg.graphData().nodes || []; + nodes.forEach(n => { n.color = nodeColor(n); n.stroke = contrastOn(n.color); }); + invalidate(); + } + + /* The dashboard's **Labels** checkbox turns on *both* label layers on the classic path: + entity names (painted by styleNode) and relation names (a `linkCanvasObject`, drawn + 'after' the line so it sits on top of it). Without this second half the checkbox silently + did half its job under `?graph-engine=next` and a relation name could only be read by + hovering one edge at a time. Same gates as classic graphRender(): zoomed in past + LINK_LABEL_MIN_SCALE, the relation actually carries a label, and — on a dense graph — + only while something is highlighted, so thousands of overlapping strings are never + painted at once. Canvas text is not an HTML sink, so the raw label is drawn here; the + escaped copy is for `linkLabel`, whose tooltip *is* one. */ + function applyLinkLabels() { + if (!fg.linkCanvasObject || !fg.linkCanvasObjectMode) return; + if (!state.settings.labels) { fg.linkCanvasObjectMode(() => undefined); return; } + fg.linkCanvasObjectMode(() => 'after').linkCanvasObject((link, ctx, scale) => { + if (!link || !link.label || scale < LINK_LABEL_MIN_SCALE) return; + if (dense && !hilite) return; + const source = link.source, target = link.target; + if (!source || !target || typeof source !== 'object' || typeof target !== 'object') return; + if (!Number.isFinite(source.x) || !Number.isFinite(source.y)) return; + if (!Number.isFinite(target.x) || !Number.isFinite(target.y)) return; + if (link.ghost) return; + ctx.font = ((state.settings.font || 12) * 0.82) / scale + 'px system-ui, sans-serif'; + ctx.fillStyle = state.themeColors.relation_label || '#7e8795'; + ctx.textAlign = 'center'; + ctx.textBaseline = 'middle'; + ctx.fillText(String(link.label), (source.x + target.x) / 2, (source.y + target.y) / 2); + ctx.textAlign = 'left'; + }); + } + + /* Does this render show the same entities and relations as the one force-graph is already + holding? Compared by identity of the *view*, not of the payload: `visible()` allocates + fresh arrays every call (and `collapsedData` fresh cluster nodes), so an object compare + would report a change for Style, Color by, Labels and Flow — none of which move a node. */ + function sameData(previous, next) { + if (!previous) return false; + if (previous.nodes.length !== next.nodes.length) return false; + if (previous.links.length !== next.links.length) return false; + for (let i = 0; i < next.nodes.length; i++) { + if (previous.nodes[i].id !== next.nodes[i].id) return false; + } + for (let i = 0; i < next.links.length; i++) { + const a = previous.links[i], b = next.links[i]; + if (linkEndpoint(a, 'source') !== linkEndpoint(b, 'source')) return false; + if (linkEndpoint(a, 'target') !== linkEndpoint(b, 'target')) return false; + if ((a.layer || '') !== (b.layer || '')) return false; + if (!a.suggested !== !b.suggested) return false; + if (!a.ghost !== !b.ghost) return false; + } + return true; + } + + /* Large graphs settle harder, exactly as the classic path does (`GPERF.large?.055:.035`). + Shared so reheat() and freeze() cannot drift back to the small-graph constant. */ + function alphaDecay() { return large ? 0.055 : 0.035; } + + function render(fit, reheat) { + if (destroyed) return; + if (suspended) { + pendingRender = pendingRender ? [pendingRender[0] || fit, pendingRender[1] || reheat] : [fit, reheat]; + return; + } + const motion = !reduced(); + const next = visible(); + /* Reuse the arrays force-graph already holds when the view is unchanged: the sizing and + colouring pass below must write onto the objects the vendor is painting from, and the + collapsed view hands out freshly built cluster nodes on every call. */ + const reused = sameData(seeded, next); + const data = reused ? seeded : next; + large = data.nodes.length > LARGE_NODE_LIMIT || data.links.length > LARGE_LINK_LIMIT; + dense = data.links.length > DENSE_LINK_LIMIT; + const sizeMetric = n => state.sizeBy === 'betweenness' ? (n.betweenness || 0) : ((n.degree || 0) / Math.max(1, maxDeg)); + data.nodes.forEach(n => { + const base = (state.settings.size || 3); + n.radius = n.cluster + ? Math.max(3, base * (1.4 + Math.min(3, Math.sqrt(n.members || 1) * 0.7))) + : Math.max(0.8, base * (0.55 + Math.min(1.6, sizeMetric(n) * 1.9))); + n.color = nodeColor(n); + n.stroke = contrastOn(n.color); + }); + const labelCap = Math.max(1, Math.round(Number(state.settings.labelDensity) || 40)); + labelIds = new Set(data.nodes + .filter(n => !n.cluster && !n.ghost) + .sort((a, b) => (b.degree || 0) - (a.degree || 0) + || (b.betweenness || 0) - (a.betweenness || 0) + || String(a.id).localeCompare(String(b.id))) + .slice(0, labelCap) + .map(n => n.id)); + applyChrome(); + if (!reused) { fg.graphData(data); seeded = data; } + applyForces(); + fg.autoPauseRedraw(!needsContinuousFrames()); + /* Bound the simulation the way the classic path does. Without these force-graph keeps its + 15-second default window, so every load and every reheat of a large store runs the + layout — and repaints every node and link — for more than ten seconds longer. */ + if (fg.cooldownTime) fg.cooldownTime(motion ? (large ? 1100 : 2200) : 0); + if (fg.cooldownTicks) fg.cooldownTicks(motion ? (large ? 80 : 160) : 1); + if (fg.warmupTicks) fg.warmupTicks(motion ? (large ? 18 : 40) : 45); + if (fg.d3AlphaDecay) fg.d3AlphaDecay(alphaDecay()); + if (fg.d3VelocityDecay) fg.d3VelocityDecay(large ? 0.45 : 0.38); + if (fg.linkCurvature) { + fg.linkCurvature(dense ? 0 : ((PRESETS[state.settings.mode] || PRESETS.compact).curve || 0)); + } + fg.linkDirectionalArrowLength(dense ? 0 : 2.5).linkDirectionalArrowRelPos(1); + applyLinkLabels(); + if (fg.linkDirectionalParticles) { + const flowing = state.settings.flow !== false + && motion + && data.links.length <= PARTICLE_LINK_LIMIT; + const particles = !flowing + ? 0 + : (state.styleName === 'cyber' ? 3 : ((PRESETS[state.settings.mode] || {}).particles || 2)); + fg.linkDirectionalParticles(l => l.suggested || l.ghost ? 0 : particles) + .linkDirectionalParticleWidth(2) + .linkDirectionalParticleColor(l => alpha(layerColor(l.layer), 0.95)) + .linkDirectionalParticleSpeed(l => 0.002 + ((state.settings.flowSpeed || 45) / 100) * 0.008); + } + if (reheat && motion && !state.settings.frozen && fg.d3ReheatSimulation) fg.d3ReheatSimulation(); + if ((state.settings.frozen || !motion) && fg.d3AlphaDecay) { /* keep painting, stop layout */ fg.d3AlphaDecay(1); } + /* Nothing was reseeded, so force-graph's own change detection saw no reason to repaint — + but Style, Color by and Labels all just changed how the *same* data must be drawn. */ + if (reused) invalidate(); + if (fit) { + clearTimeout(fitTimer); + fitTimer = setTimeout(() => { if (!destroyed) fg.zoomToFit(motion ? 600 : 0, 40); }, motion ? 320 : 0); + } + if (opts.onStats) opts.onStats({ nodes: data.nodes.length, links: data.links.length, total: raw.nodes.length, totalLinks: raw.links.length, preset: (PRESETS[state.settings.mode] || PRESETS.compact).label, collapsed: collapsed, ghosts: data.nodes.filter(n => n.ghost).length, bridges: data.links.filter(l => l.bridge).length, suggested: data.links.filter(l => l.suggested).length }); + } + + fg.backgroundColor('rgba(0,0,0,0)').nodeRelSize(1).autoPauseRedraw(true) + /* force-graph's default `nodeLabel`/`linkLabel` is the literal accessor "name", and its + tooltip renders a string label with innerHTML. Node names here are entity labels + extracted from ingested memories — untrusted input — so both accessors are set + explicitly and escaped rather than left on the vendor default. */ + .nodeLabel(node => esc(nodeName(node))) + .linkLabel(link => esc(link && link.label ? link.label : '')) + .onRenderFramePre((ctx, scale) => { try { styleBackground(ctx, scale); } catch (e) { } }) + .nodeCanvasObject((node, ctx, scale) => styleNode(node, ctx, scale)) + .nodePointerAreaPaint((node, color, ctx) => { ctx.fillStyle = color; ctx.beginPath(); ctx.arc(node.x, node.y, node.radius + 2, 0, 6.2832); ctx.fill(); }) + .linkColor(l => { + const focus = hoverSet && hoverSet.size > 1; + const s = linkEndpoint(l, 'source'), t = linkEndpoint(l, 'target'); + const active = !focus || s === hilite || t === hilite; + if (l.suggested) return alpha('#ffffff', active ? 0.34 : 0.1); + if (l.ghost) return alpha(layerColor(l.layer), 0.12); + if (state.bridges && l.bridge) return alpha('#ff5c7a', active ? 0.95 : 0.5); + const base = layerColor(l.layer); + return active ? alpha(base, focus ? 0.85 : 0.4) : alpha(base, 0.06); + }) + .linkLineDash(l => l.suggested ? [2, 2] : (l.ghost ? [1, 3] : null)) + .linkWidth(l => { + const w = state.settings.linkw || 1; + const focus = hoverSet && hoverSet.size > 1; + const s = linkEndpoint(l, 'source'), t = linkEndpoint(l, 'target'); + if (l.aggregate) return Math.min(6, 0.6 + Math.log2(1 + (l.weight || 1)) * 1.4) * w; + if (state.bridges && l.bridge) return 2.6 * w; + if (!focus) return 0.82 * w; + return (s === hilite || t === hilite) ? 2.4 * w : 0.4 * w; + }) + .onNodeHover(node => { + hilite = node ? node.id : null; + hoverSet = node ? new Set([node.id].concat(adj[node.id] || [])) : null; + el.classList.toggle('engraphis-graph-node-hover', !!node); + invalidate(); + }) + .onNodeClick(node => { + if (node.cluster) { collapsed = false; state.collapse = false; render(false, true); setTimeout(() => { fg.centerAt(node.x, node.y, 500); fg.zoom(1.6, 500); }, 60); if (opts.onCollapseChange) opts.onCollapseChange(false); return; } + if (opts.onNodeClick) opts.onNodeClick(node); + }) + .onNodeDragEnd(node => { node.fx = node.x; node.fy = node.y; }) + .onBackgroundClick(() => { if (opts.onBackgroundClick) opts.onBackgroundClick(); }) + .onZoom(z => { + zoom = z.k || 1; + if (state.collapse !== 'auto') return; + const next = zoom < 0.55; + if (next !== collapsed) { + collapsed = next; + render(false, true); + if (opts.onCollapseChange) opts.onCollapseChange(collapsed); + } + }); + + api.setData = data => { + const inputNodes = Array.isArray(data && data.nodes) ? data.nodes : []; + const nodes = inputNodes + .filter(node => node && node.id != null) + .map(node => Object.assign({}, node, { name: nodeName(node) })); + const nodeIds = new Set(nodes.map(node => node.id)); + const links = (Array.isArray(data && (data.links || data.edges)) ? (data.links || data.edges) : []) + .map(link => { + const source = linkEndpoint(link, 'source'), target = linkEndpoint(link, 'target'); + return Object.assign({}, link, { source, target }); + }) + .filter(link => link.source != null && link.target != null && nodeIds.has(link.source) && nodeIds.has(link.target)); + const suggestions = (Array.isArray(data && data.suggestions) ? data.suggestions : []) + .map(link => Object.assign({}, link, { source: linkEndpoint(link, 'source'), target: linkEndpoint(link, 'target') })) + .filter(link => link.source != null && link.target != null); + /* A fresh payload means fresh node objects, so the cached seed is stale even when the + ids are identical — force-graph must be re-pointed at the new objects or the render + below would style ones nobody is painting from. */ + seeded = null; + raw = { nodes, links, suggestions }; + adj = communities(raw.nodes, raw.links); + const deg = {}; + raw.links.forEach(l => { const s = linkEndpoint(l, 'source'), t = linkEndpoint(l, 'target'); deg[s] = (deg[s] || 0) + 1; deg[t] = (deg[t] || 0) + 1; }); + raw.nodes.forEach(n => { n.degree = deg[n.id] || 0; n.betweenness = 0; }); + maxDeg = maxOf(raw.nodes.map(n => n.degree), 1); + const ranked = [...raw.nodes].sort((a, b) => b.degree - a.degree); + ranked.forEach((n, i) => { n.rank = i; n.hub = i < 6; }); + // Bridge *edges* are cheap (linear) and feed the stats readout, so they stay eager. + // Betweenness is not: see ensureBetweenness. + findBridges(raw.nodes, raw.links, adj); + betweennessReady = false; + if (state.bridges || state.sizeBy === 'betweenness') ensureBetweenness(); + render(true, true); + }; + /* Which of these settings changes the *layout* rather than just the paint, matching the + classic path's `key==='repel'||key==='link'||key==='gravity'||key==='size'` in + dashboard.js::graphSet — `size` counts because it feeds d3.forceCollide, and `mode` + swaps the whole force arrangement. applyForces() only writes the new charge / link / + forceX-forceY / collide values into the simulation force-graph is already running, and a + settled graph sits at alpha~0, so without the reheat those sliders install a force that + moves nothing. The paint-only settings must keep the arrangement the user is reading. + render() applies the reduced-motion exemption (`if(layout&&!prefersReducedMotion())`). */ + const LAYOUT_KEYS = ['mode', 'repel', 'link', 'gravity', 'size']; + api.setSettings = patch => { + Object.assign(state.settings, patch); + render(false, LAYOUT_KEYS.some(k => patch && patch[k] !== undefined)); + }; + api.setPreset = name => { + const p = PRESETS[name] || PRESETS.compact; + state.settings.mode = PRESETS[name] ? name : 'compact'; + ['repel', 'link', 'gravity', 'font', 'size', 'linkw', 'labelDensity'].forEach(k => { if (p[k] !== undefined) state.settings[k] = p[k]; }); + render(true, true); + return { ...state.settings }; + }; + api.setStyle = name => { state.styleName = ['classic', 'galaxy', 'solar', 'cyber'].indexOf(name) < 0 ? 'cyber' : name; render(false, false); }; + api.setColorBy = name => { state.colorBy = name; refreshColors(); render(false, false); }; + api.setPalette = name => { state.palette = name; state.overrides = PALETTES[name] ? { ...PALETTES[name] } : {}; refreshColors(); }; + api.setTypeColor = (type, color) => { state.overrides[type] = color; state.palette = 'custom'; refreshColors(); }; + /* Rehydrating saved overrides is not a user edit, so it must not flip the palette + selector to "custom" behind the user's back the way setTypeColor deliberately does. */ + api.setTypeColors = map => { Object.assign(state.overrides, map || {}); refreshColors(); }; + /* The active theme's resolved `--entity-*` values. Replaced wholesale rather than merged: + a theme switch must not leave the previous theme's colour for a type the new one omits. */ + api.setThemeColors = map => { state.themeColors = map && typeof map === 'object' ? { ...map } : {}; refreshColors(); }; + /* One render for a whole batch of setters — see `batch`. */ + api.apply = (fn, fit, reheat) => { batch(fn, fit, reheat); }; + api.setHighlight = id => { + hilite = id == null ? null : id; + hoverSet = id == null ? null : new Set([id].concat(adj[id] || [])); + invalidate(); + }; + api.setScope = patch => { Object.assign(state, patch); render(false, true); }; + api.setLayers = layers => { state.layers = layers; render(false, false); }; + api.focus = id => { state.focusId = id; render(true, true); }; + api.clearFocus = () => { state.focusId = null; render(true, true); }; + api.fit = () => { if (!destroyed) fg.zoomToFit(reduced() ? 0 : 500, 40); }; + api.reheat = () => { + if (destroyed || reduced()) return; + raw.nodes.forEach(n => { n.fx = undefined; n.fy = undefined; }); + if (fg.d3ReheatSimulation) { fg.d3AlphaDecay(alphaDecay()); fg.d3ReheatSimulation(); } + }; + api.freeze = on => { + state.settings.frozen = on; + if (on) { fg.d3Force('charge').strength(0); fg.d3AlphaDecay(1); return; } + applyForces(); + // Dragging pins nodes with fx/fy. "Unfreeze" must release those pins even when reduced + // motion suppresses the reheat below; otherwise the control claims movement is restored + // while every dragged node remains immobile. + raw.nodes.forEach(n => { n.fx = undefined; n.fy = undefined; }); + if (reduced()) return; + fg.d3AlphaDecay(alphaDecay()); + if (fg.d3ReheatSimulation) fg.d3ReheatSimulation(); + }; + /* Returning `false` is not a failure: it is the signal the dashboard's graphFocus() uses to + run its recovery path ("show unlinked", then retry, then say so). Reporting success for an + entity that is not on the canvas is therefore worse than reporting failure — the user gets + a camera move to nothing and no explanation. Two ways that happened: the auto-collapsed + view paints only `cluster-*` bubbles, and any filtered-out node keeps the x/y force-graph + left on it from an earlier render, so "found in `raw.nodes` with finite coordinates" was + never evidence of visibility. Expand a collapsed view first — focusing a named entity is + an explicit request to see it — then confirm against the data force-graph is holding. */ + api.zoomToNode = id => { + if (destroyed) return false; + const n = raw.nodes.find(x => x.id === id); + if (!n) return false; + if (collapsed) { + collapsed = false; + state.collapse = false; + render(false, true); + if (opts.onCollapseChange) opts.onCollapseChange(false); + } + const shown = ((fg.graphData() || {}).nodes || []).some(node => node && node.id === id); + if (!shown || !Number.isFinite(n.x) || !Number.isFinite(n.y)) return false; + const duration = reduced() ? 0 : 500; + fg.centerAt(n.x, n.y, duration); + fg.zoom(3, duration); + return true; + }; + api.state = () => ({ ...state, collapsed }); + /* The engine clusters its own copies of the nodes, so a caller that renders a cluster + legend from the source data would otherwise report a single community. */ + api.communityMap = () => { + const map = {}; + raw.nodes.forEach(n => { map[n.id] = n.community || 0; }); + return map; + }; + api.setGhosts = on => { state.ghost = on; render(false, false); }; + api.setRepoFilter = repo => { state.repo = (repo || '').trim().toLowerCase(); render(false, true); }; + api.setAsOf = date => { state.asOf = asOfValue(date); render(false, true); }; + api.setSizeBy = metric => { + state.sizeBy = metric === 'betweenness' ? metric : 'degree'; + if (state.sizeBy === 'betweenness') ensureBetweenness(); + render(false, false); + }; + api.setBridges = on => { state.bridges = on; if (on) ensureBetweenness(); render(false, false); }; + /* Forces the lazy analysis; the dashboard does not display these yet, so nothing calls it + on the render path. Kept as the seam a "most load-bearing entity" panel would use. */ + api.metrics = () => { + ensureBetweenness(); + return { + top: [...raw.nodes].sort((a, b) => b.betweenness - a.betweenness).slice(0, 5) + .map(n => ({ id: n.id, name: nodeName(n), score: n.betweenness })), + bridges: raw.links.filter(l => l.bridge).length + }; + }; + api.setSuggestions = on => { state.suggestions = on; render(false, true); }; + api.setCollapse = mode => { + state.collapse = mode; + const next = mode === true || (mode === 'auto' && zoom < 0.55); + collapsed = next; + render(true, true); + }; + api.presets = PRESETS; + api.resize = () => { measure(); }; + /* Leaving the graph view must stop the simulation loop. force-graph keeps a rAF alive + for as long as it is resumed, so a hidden pane would otherwise repaint forever. */ + api.pause = () => { + if (destroyed || !running) return; + running = false; + if (fg.pauseAnimation) fg.pauseAnimation(); + }; + api.resume = () => { + if (destroyed || running) return; + running = true; + if (fg.resumeAnimation) fg.resumeAnimation(); + measure(); + }; + api.destroyed = () => destroyed; + api.destroy = () => { + if (destroyed) return; + destroyed = true; + clearTimeout(fitTimer); + try { + if (api._ro) { api._ro.disconnect(); api._ro = null; } + // `_destructor` pauses the rAF and drops the graph data; it does not detach the + // canvas, so clear the container too or a re-create leaves the old one attached. + if (fg._destructor) fg._destructor(); + el.removeAttribute('data-graph-style'); + el.classList.remove('engraphis-graph-node-hover'); + el.innerHTML = ''; + } catch (e) { /* teardown is best-effort: never let it block a view change */ } + raw = { nodes: [], links: [], suggestions: [] }; + adj = {}; + seeded = null; + hilite = null; + hoverSet = null; + }; + + // A hidden pane measures 0x0; writing that into force-graph collapses the canvas and + // nothing restores it, so only a real box is ever applied. + const measure = () => { + if (destroyed) return; + const w = el.clientWidth, h = el.clientHeight; + if (w > 0 && h > 0) fg.width(w).height(h); + }; + measure(); + requestAnimationFrame(() => { if (destroyed) return; measure(); fg.zoomToFit(reduced() ? 0 : 400, 40); }); + if (typeof ResizeObserver !== 'undefined') { + api._ro = new ResizeObserver(() => measure()); + api._ro.observe(el); + } + applyChrome(); + return api; + } + + window.EngraphisGraph = { + create, PRESETS, PALETTES, STYLE_LAYERS, COMMUNITY_PALS, GRAPH_HEAT, THEME_ETYPE, STYLE_PAL, + /* Pure helpers, exported so the offline test suite can assert real behaviour (escaping, + component labelling, bridge detection, stack safety) without a browser or a bundler. + Nothing in the dashboard uses these; treat them as the engine's unit-test seam. */ + _internals: { + esc, hexRgb, alpha, contrastOn, communities, betweenness, findBridges, maxOf, + nodeName, linkEndpoint, asOfValue + } + }; +})(); diff --git a/engraphis/device_connect.py b/engraphis/device_connect.py new file mode 100644 index 0000000..63d5758 --- /dev/null +++ b/engraphis/device_connect.py @@ -0,0 +1,832 @@ +"""Client half of the Engraphis Cloud device-connect flow. + +The account portal issues a one-time connect token (``engr_ct_...``) and shows the +customer a single command. This module exchanges that token for the durable session +material :func:`engraphis.cloud_session.save_bootstrap` persists, so +``~/.engraphis/cloud_session.json`` -- the file :doc:`AGENT_CONNECT` and ``.env.example`` +tell paying customers to prefer -- is finally created by something. Before this existed +``save_bootstrap`` had no production caller at all and a paid installation could not be +connected without hand-writing the state file. + +Design constraints, all of which have teeth: + +* **The connect token is a bearer credential.** It travels in the request body and + nowhere else: never in a log line, never in an exception message, never in the saved + session, never echoed back to the terminal. Callers get a redacted summary. +* **The vetted transport only.** Requests go through + :func:`engraphis.hosted_client.build_pinned_https_opener` with redirects blocked and an + explicit timeout, exactly as :mod:`engraphis.cloud_session` and + :mod:`engraphis.update_check` do. A bare ``urllib.request.build_opener`` would drop the + DNS-rebinding guard on a credential-bearing call. +* **Fixed copy keyed on status.** Provider bodies are untrusted -- they may carry + internal URLs -- so nothing from the response is reflected into an error message. The + control plane deliberately makes expired / consumed / invalid tokens indistinguishable + (all ``401``), so the client says all three at once rather than guessing. +""" +from __future__ import annotations + +import http.client +import json +import math +import os +import platform +import re +import socket +import urllib.error +import urllib.request +from pathlib import Path +from urllib.parse import urlsplit +from typing import Optional, Tuple + +from engraphis import cloud_session +from engraphis.hosted_client import ( + CloudUrlUnresolved, + build_pinned_https_opener, + upgrade_url, + validate_cloud_base_url, +) +from engraphis.private_state import ( + UnsafeStateFile, + atomic_private_text, + publish_private_text_if_absent, + read_private_text, +) + +try: # installed distribution -> real version; source tree -> harmless fallback + from engraphis import __version__ as CURRENT_VERSION +except Exception: # pragma: no cover - engraphis is always importable in practice + CURRENT_VERSION = "0" + +#: Path segment of the unauthenticated connect endpoint on the control plane. +CONNECT_PATH = "/v1/devices/connect" + +#: One interactive request. Longer than the 10s refresh budget because this runs at a +#: human's prompt, once, and a spurious timeout costs the customer a fresh token. +DEFAULT_TIMEOUT_SECONDS = 15.0 + +#: A device connection is one interactive request. Letting a caller supply an arbitrary +#: finite float is not portable: values that fit Python's ``float`` can still overflow the +#: platform socket timeout. Two minutes leaves room for a slow network without turning a +#: mistyped CLI value into an unbounded wait or a raw ``OverflowError`` from ``urllib``. +_MAX_TIMEOUT_SECONDS = 120.0 + +#: The control plane answers with a small fixed record; anything larger is not ours. +_MAX_RESPONSE_BYTES = 64 * 1024 + +#: Copy for a reply that began but never completed. Every other failure in this module can +#: promise the connect token is untouched; this one cannot, because the request reached a +#: control plane that started to answer, and a 200 consumes the token as it is written. +#: Saying "try again" here would be a lie that costs the customer a second failed attempt, +#: so the copy sends them to the one place that shows whether the device landed. +_TRUNCATED_REPLY = ( + "Engraphis Cloud started answering this connect request but the connection closed " + "before the reply was complete. Your connect token may already have been used: check " + "your account portal, and generate a new one if this device is not listed there." +) + +#: Appended to every failure that can only happen once the control plane has answered +#: successfully. ``urllib`` raises ``HTTPError`` for every status >= 400, and ``_NoRedirect`` +#: turns a 3xx into one too, so any fault past ``opener.open()`` follows a 2xx -- and a +#: successful connect consumes the single-use token as it is written. Telling the customer +#: to "try again" there would send them into a guaranteed 401 with no session; the only +#: action that can work is a fresh token from the portal. +_SPENT_TOKEN_SUFFIX = ( + " This connect token has now been used -- generate a new one in your Engraphis account " + "portal and try again, and contact support if it repeats." +) + +#: The portal always mints tokens with this prefix. Checking it locally turns a +#: mistyped or truncated paste into an instant, free error instead of a round trip that +#: consumes rate budget and returns the same opaque 401 as a genuinely dead token. +CONNECT_TOKEN_PREFIX = "engr_ct_" +#: Shortest credible token: the prefix plus enough entropy to be a real secret. +_MIN_TOKEN_CHARS = len(CONNECT_TOKEN_PREFIX) + 16 +_MAX_TOKEN_CHARS = 512 +#: ``secrets.token_urlsafe`` alphabet. Anything else is a paste accident (a shell- +#: mangled quote, a wrapped line, a whole command copied in). +_TOKEN_BODY = re.compile(r"\A[A-Za-z0-9_-]+\Z") + +#: Identity file for this installation. Kept beside ``cloud_session.json`` in the same +#: owner-only state directory and honouring the same ``ENGRAPHIS_STATE_DIR`` override. +_IDENTITY_FILENAME = "client_identity.json" +_IDENTITY_SCHEMA = "engraphis-client-identity/v1" + +#: Compute endpoint for the production deployment. ``DeviceRegistrationResponse`` does +#: not carry one, and ``cloud_session.configured()`` requires it, so a connect against the +#: shipped control plane would otherwise leave a customer "connected" but not configured. +#: Applied *only* when the control plane is the shipped one: a self-hosted or staging +#: control URL gets no guess, and the caller is told to supply the compute URL. +#: +#: This is the *fallback*, not the authority: ``default_compute_url`` prefers the +#: manifest's ``compute_plane`` whenever it carries one. The shipped +#: ``engraphis-commercial/v2`` manifest does not declare that key, so without a constant +#: here a production connect would resolve an empty compute URL and save an unconfigured +#: session. +DEFAULT_COMPUTE_URL = "https://compute.engraphis.com" + + +class DeviceConnectError(RuntimeError): + """A connect attempt failed with public, actionable copy. + + ``status`` mirrors the control-plane status where there was one, and uses ``400`` for + a request this client refused to send at all. + """ + + def __init__(self, message: str, *, status: int = 503) -> None: + super().__init__(message) + self.status = status + + +class _NoRedirect(urllib.request.HTTPRedirectHandler): + """Refuse redirects: a 3xx would replay the connect token at an unvetted host.""" + + def redirect_request(self, req, fp, code, msg, headers, newurl): + return None + + +def normalize_connect_token(value: object) -> str: + """Return the trimmed token, or raise before any network call happens. + + Never quotes the offending value. A rejected token is still a credential -- an + error message containing it would land in shell history, CI logs and bug reports. + """ + + token = str(value or "").strip() + if not token: + raise DeviceConnectError( + "No connect token was supplied. Copy the `engraphis connect --token ...` " + "command shown in your Engraphis account portal.", + status=400, + ) + if ( + not token.startswith(CONNECT_TOKEN_PREFIX) + or len(token) < _MIN_TOKEN_CHARS + or len(token) > _MAX_TOKEN_CHARS + or not _TOKEN_BODY.match(token[len(CONNECT_TOKEN_PREFIX):]) + ): + raise DeviceConnectError( + "That does not look like an Engraphis connect token (they start with " + "`%s`). Copy the whole command from your account portal -- a token split " + "across lines or missing its last characters will not work." + % CONNECT_TOKEN_PREFIX, + status=400, + ) + return token + + +def _state_dir() -> Path: + root = os.environ.get("ENGRAPHIS_STATE_DIR", "").strip() + return Path(root).expanduser() if root else Path.home() / ".engraphis" + + +def _identity_path() -> Path: + try: + return _state_dir() / _IDENTITY_FILENAME + except RuntimeError as exc: + # ``Path.home()`` raises ``RuntimeError("Could not determine home directory")`` on a + # service account or container with no resolvable home. This module promises every + # failure is a ``DeviceConnectError``, and the CLI catches only that, so the bare + # ``RuntimeError`` reached the terminal as a traceback -- from a command whose whole + # selling point is being non-interactive and scriptable. ``ENGRAPHIS_STATE_DIR`` is + # the documented answer, so the copy names it. + raise DeviceConnectError( + "Engraphis could not determine a home directory for its state files. Set " + "ENGRAPHIS_STATE_DIR to a writable directory and run `engraphis connect " + "--token ...` again.", + status=400, + ) from exc + + +def _new_client_id(prefix: str) -> str: + # The package's own ULID minter, so these ids sort chronologically and read the same + # way as every other prefixed id in the client. + from engraphis.core import ids + + return "%s_%s" % (prefix, ids.ulid()) + + +def _read_identity() -> dict: + try: + raw = read_private_text(_identity_path(), max_bytes=8 * 1024, allow_missing=True) + except DeviceConnectError: + # Already actionable copy (an unresolvable home directory). Must come first: + # ``DeviceConnectError`` is a ``RuntimeError``, so the broad clause below would + # otherwise swallow it and let the *next*, unguarded ``_identity_path()`` call + # raise the raw error instead. + raise + except UnsafeStateFile as exc: + raise DeviceConnectError( + "The saved client identity has unsafe filesystem permissions. Remove %s and " + "connect again." % _identity_path(), + status=409, + ) from exc + except (OSError, RuntimeError): + return {} + if not raw: + return {} + try: + value = json.loads(raw) + except (ValueError, RecursionError): + return {} + return value if isinstance(value, dict) else {} + + +def client_identity() -> Tuple[str, str]: + """Return ``(installation_client_id, device_client_id)`` for this machine. + + Minted once and persisted at ``/client_identity.json`` with owner-only + permissions, so reconnecting the same machine re-presents the same pair and the + control plane recognises the installation instead of accumulating a new phantom + device on every connect. + + They are random ULIDs rather than a hardware fingerprint on purpose: a MAC- or + hostname-derived id is a stable cross-account identifier the customer never agreed to + ship, and it changes under exactly the conditions (a new NIC, a rename) where + stability was the point. "Stable per installation" is what the server needs, and a + minted-once file in the state directory is precisely that. A second state directory + (``ENGRAPHIS_STATE_DIR``, a container, another user account) is a second installation, + which is the correct reading. + """ + + saved = _read_identity() + installation = str(saved.get("installation_client_id") or "").strip() + device = str(saved.get("device_client_id") or "").strip() + if installation and device: + return installation, device + + minted = { + "schema": _IDENTITY_SCHEMA, + "installation_client_id": installation or _new_client_id("inst"), + "device_client_id": device or _new_client_id("dev"), + } + payload = json.dumps(minted, sort_keys=True, separators=(",", ":")) + path = _identity_path() + try: + if not publish_private_text_if_absent(path, payload): + # Another process won the create, or a half-written file already existed. + # Re-read: the winner's ids are the ones the control plane will see. + existing = _read_identity() + installation = str(existing.get("installation_client_id") or "").strip() + device = str(existing.get("device_client_id") or "").strip() + if installation and device: + return installation, device + atomic_private_text(path, payload) + except UnsafeStateFile as exc: + raise DeviceConnectError( + "The client identity file at %s could not be written safely." % path, + status=409, + ) from exc + except OSError as exc: + raise DeviceConnectError( + "Could not write the client identity file at %s." % path, status=409 + ) from exc + return minted["installation_client_id"], minted["device_client_id"] + + +def default_control_url() -> str: + """Resolve the control plane: explicit env override, else the shipped manifest.""" + + configured = os.environ.get("ENGRAPHIS_CLOUD_CONTROL_URL", "").strip() + if configured: + return configured + try: + from engraphis.commercial import manifest + + return str(manifest().get("control_plane") or "").strip() + except Exception: # pragma: no cover - manifest ships with the package + return "" + + +def _canonical_endpoint(value: object) -> str: + """Canonical ``scheme://host:port/path`` for comparing two endpoint URLs. + + ``validate_cloud_base_url`` returns ``urlunsplit((scheme, parts.netloc, ...))``: it + lower-cases the *scheme* but passes the netloc through verbatim, and it never removes a + redundant default port. So ``HTTPS://API.ENGRAPHIS.COM`` normalises to + ``https://API.ENGRAPHIS.COM``, and ``https://api.engraphis.com:443`` keeps its ``:443`` + -- both are the shipped control plane, and both compared unequal to it as raw strings. + The consequence was silent and expensive: :func:`default_compute_url` returned ``""``, + so a customer who typed the production URL with a different case or an explicit port + connected successfully and then had every hosted feature switched off, because + ``cloud_session.configured()`` needs a compute endpoint. + + Returns ``""`` for anything unparseable, which never equals another canonical form. + """ + + try: + parts = urlsplit(str(value or "").strip()) + except ValueError: + return "" + scheme = parts.scheme.lower() + host = (parts.hostname or "").lower() + if not scheme or not host: + return "" + try: + port = parts.port + except ValueError: + return "" + if port is None: + port = 443 if scheme == "https" else 80 + return "%s://%s:%d%s" % (scheme, host, port, parts.path.rstrip("/")) + + +def default_compute_url(control_url: str) -> str: + """Resolve the compute plane, guessing only for the shipped control plane. + + The manifest is the authority for shipped endpoint metadata, so a ``compute_plane`` + key wins if one is ever published -- a manifest-only endpoint change then needs no + code change here. The current ``engraphis-commercial/v2`` manifest declares only + ``control_plane``, so :data:`DEFAULT_COMPUTE_URL` is what production actually + resolves; reading the absent key alone would yield ``""`` and save a session + ``cloud_session.configured()`` rejects. + """ + + configured = os.environ.get("ENGRAPHIS_CLOUD_COMPUTE_URL", "").strip() + if configured: + return configured + shipped = "" + shipped_compute = "" + try: + from engraphis.commercial import manifest + + data = manifest() + shipped = str(data.get("control_plane") or "").strip() + shipped_compute = str(data.get("compute_plane") or "").strip() + except Exception: # pragma: no cover - manifest ships with the package + return "" + if shipped and _canonical_endpoint(control_url) == _canonical_endpoint(shipped): + return shipped_compute or DEFAULT_COMPUTE_URL + return "" + + +def _validated_timeout(value: object) -> float: + """Reject a timeout the socket layer cannot use, before any request is started. + + ``float("nan")`` and ``float("inf")`` are values ``argparse``'s ``type=float`` + accepts happily, but they reach the pinned opener's deadline arithmetic and raise + ``ValueError``/``OverflowError`` from inside ``urllib`` -- neither of which + :func:`post_connect` catches. That breaks this module's contract that every failure + is a :class:`DeviceConnectError` with actionable copy, so ``--timeout nan`` printed a + traceback instead of an error and an exit code. Non-positive values are refused for + the same reason: a ``0`` or negative socket timeout is not a shorter wait, it is a + different (non-blocking) mode the caller did not ask for. + """ + + try: + timeout = float(value) + except (TypeError, ValueError) as exc: + raise DeviceConnectError( + "The connect timeout must be a number of seconds.", status=400 + ) from exc + if not math.isfinite(timeout) or timeout <= 0: + raise DeviceConnectError( + "The connect timeout must be a positive, finite number of seconds.", + status=400, + ) + if timeout > _MAX_TIMEOUT_SECONDS: + raise DeviceConnectError( + "The connect timeout must not exceed %d seconds." + % _MAX_TIMEOUT_SECONDS, + status=400, + ) + return timeout + + +def _validated_control_url(value: str) -> str: + if not str(value or "").strip(): + raise DeviceConnectError( + "No Engraphis Cloud control URL is configured. Set " + "ENGRAPHIS_CLOUD_CONTROL_URL or pass --control-url.", + status=400, + ) + try: + return validate_cloud_base_url(value) + except CloudUrlUnresolved as exc: + # "Offline" is not "misconfigured": validate_cloud_base_url resolves the host, so + # a customer on a plane would otherwise be told their URL is invalid forever. + raise DeviceConnectError( + "Engraphis Cloud is temporarily unreachable. Check your network and try " + "again.", + status=503, + ) from exc + except ValueError as exc: + raise DeviceConnectError( + "The Engraphis Cloud control URL is not a valid HTTPS endpoint.", status=400 + ) from exc + + +def _server_compute_url(response: dict) -> str: + """Return a vetted compute endpoint volunteered by the control plane, if any. + + ``compute_url`` is endpoint metadata, not a redirect target: the registration body is + untrusted until it passes the same HTTPS, public-host, and DNS-rebinding validation as + configured endpoints. A bad or temporarily unresolvable optional value must not erase + a valid manifest fallback after the one-time token was spent. + """ + + value = response.get("compute_url") + if not isinstance(value, str) or not value.strip(): + return "" + try: + return validate_cloud_base_url(value) + except (CloudUrlUnresolved, ValueError): + return "" + + +def _default_device_name() -> str: + try: + name = socket.gethostname().strip() + except OSError: + name = "" + return name[:100] + + +def _default_platform() -> str: + try: + return ("%s %s" % (platform.system(), platform.machine())).strip()[:100] + except Exception: # pragma: no cover - platform is stdlib and total + return "" + + +def _connect_http_error(status: int) -> DeviceConnectError: + """Map a control-plane status to fixed, actionable copy. + + Only the status is used. ``401`` deliberately covers expired, already-consumed and + never-valid tokens with one indistinguishable answer, so the copy names all three + instead of asserting one -- the fix is the same in every case. + """ + + if status == 401: + return DeviceConnectError( + "That connect token has expired, was already used, or is not valid. " + "Generate a new one in your Engraphis account portal and run " + "`engraphis connect --token ...` again.", + status=401, + ) + if status == 402: + return DeviceConnectError( + "This Engraphis Cloud subscription is not active, so no new device can be " + "connected. Update billing at %s and try again." % upgrade_url(), + status=402, + ) + if status == 403: + return DeviceConnectError( + "Engraphis Cloud refused this connect request. Check with the organization " + "owner that your account may still add devices.", + status=403, + ) + if status == 404: + return DeviceConnectError( + "This Engraphis Cloud control plane has no device-connect endpoint. Check " + "ENGRAPHIS_CLOUD_CONTROL_URL points at the URL shown in your account portal.", + status=404, + ) + if status == 422: + return DeviceConnectError( + "Engraphis Cloud rejected this connect request as malformed. Upgrade the " + "client (`pip install -U engraphis`) and try again.", + status=422, + ) + if status == 429: + return DeviceConnectError( + "Too many connect attempts. Wait a minute and try again.", status=429 + ) + if status == 503: + return DeviceConnectError( + "Engraphis Cloud is not accepting new device activations right now. Try " + "again shortly; your connect token is unaffected.", + status=503, + ) + return DeviceConnectError( + "Engraphis Cloud could not connect this device. Try again shortly.", status=503 + ) + + +def post_connect(control_url: str, token: str, *, installation_client_id: str, + device_client_id: str, installation_label: Optional[str] = None, + device_name: Optional[str] = None, app_platform: Optional[str] = None, + app_version: Optional[str] = None, workspace_id: Optional[str] = None, + timeout: float = DEFAULT_TIMEOUT_SECONDS) -> dict: + """POST the connect token and return the ``DeviceRegistrationResponse`` body. + + *control_url* must already be validated. The endpoint rejects unknown fields with a + ``422``, so optional values are omitted rather than sent empty, and there is + deliberately no ``organization_id``: the token carries the organization. + """ + + timeout = _validated_timeout(timeout) + body = { + "connect_token": token, + "installation_client_id": installation_client_id, + "device_client_id": device_client_id, + } + for key, value in ( + ("installation_label", installation_label), + ("device_name", device_name), + ("platform", app_platform), + ("app_version", app_version), + ("workspace_id", workspace_id), + ): + cleaned = str(value or "").strip() + if cleaned: + body[key] = cleaned + + payload = json.dumps(body, sort_keys=True, separators=(",", ":")).encode("utf-8") + request = urllib.request.Request( + control_url + CONNECT_PATH, + data=payload, + headers={ + "Accept": "application/json", + "Content-Type": "application/json", + "User-Agent": "Engraphis/%s device-connect" % CURRENT_VERSION, + }, + method="POST", + ) + # The open and the body read are deliberately separate ``try`` blocks. They look like + # one operation but they sit on opposite sides of the point of no return: once ``open`` + # returns, urllib has already parsed a success status line, so the control plane + # answered and the single-use token is spent. A ``TimeoutError`` or + # ``ConnectionResetError`` is an ``OSError`` in both phases, and while they shared one + # ``try`` the body-read case inherited the connection-phase copy and told the customer + # to retry a token that was already gone. + try: + response = build_pinned_https_opener(_NoRedirect()).open( + request, timeout=timeout + ) # nosec B310 - scheme validated by validate_cloud_base_url + except urllib.error.HTTPError as exc: + status = exc.code + # Draining the error body can itself raise; a sibling ``except`` of this ``try`` + # would not cover it, so an unguarded read escapes as a raw traceback exactly + # when the cloud is flaky. Same shape as cloud_session._post_refresh. + try: + exc.read(_MAX_RESPONSE_BYTES + 1) + except (OSError, ValueError, http.client.HTTPException): + pass + finally: + try: + exc.close() + except (OSError, ValueError, http.client.HTTPException): + pass + raise _connect_http_error(status) + except urllib.error.URLError as exc: + # ``exc`` may quote an internal host or a proxy URL; never reflect it. + # + # ``URLError`` is the honest "nothing was consumed" case, and the only one. + # ``AbstractHTTPHandler.do_open`` wraps failures from establishing the connection + # and from ``h.request(...)`` -- DNS, connection refused, TLS, a write that never + # completed -- in ``URLError``, but lets anything raised by ``h.getresponse()`` + # propagate unwrapped. So reaching *this* clause means the request never finished + # going out, and "try again" with the same token is correct. + raise DeviceConnectError( + "Engraphis Cloud is temporarily unreachable. Check your network and try " + "again.", + status=503, + ) from exc + except (TimeoutError, OSError, http.client.HTTPException) as exc: + # Everything else in this phase comes out of ``h.getresponse()``, which runs only + # after the request has been written in full. The control plane may therefore have + # received and processed it -- and a processed connect spends the token -- so this + # is ambiguous, not a clean miss. + # + # This clause used to claim the opposite for ``RemoteDisconnected`` ("the peer + # closed without answering at all. Nothing was consumed"). That was an overclaim: + # ``RemoteDisconnected`` is raised when ``getresponse()`` reads zero bytes for the + # status line, which is *after* the POST went out. Telling that customer to retry + # sent them into a 401 that reads as "the token I just generated is invalid". + # ``_TRUNCATED_REPLY`` is the right copy under ambiguity: it asks them to check the + # portal first rather than asserting either outcome. + # + # ``LineTooLong``/``BadStatusLine`` from a mangled status line, and ``IncompleteRead`` + # from a truncated body, land here for the same reason -- none of them are a + # ``URLError``, so before this clause existed they escaped as a raw traceback. + raise DeviceConnectError(_TRUNCATED_REPLY, status=502) from exc + + # Past this line the control plane has answered with a success status, so the token is + # spent no matter what goes wrong next. ``OSError`` covers a socket that times out or + # resets mid-body, ``HTTPException`` a truncated chunked body, ``ValueError`` a read + # from an already-closed response; all three mean the same thing to the customer. + try: + with response: + raw = response.read(_MAX_RESPONSE_BYTES + 1) + except (OSError, ValueError, http.client.HTTPException) as exc: + raise DeviceConnectError(_TRUNCATED_REPLY, status=502) from exc + + # Everything below here runs only after a 2xx, so the token is already spent -- see + # ``_SPENT_TOKEN_SUFFIX``. A bare "invalid response" would leave the customer retrying + # a consumed token forever. + if len(raw) > _MAX_RESPONSE_BYTES: + raise DeviceConnectError( + "Engraphis Cloud returned an oversized connect response, so no session was " + "saved." + _SPENT_TOKEN_SUFFIX, + status=502, + ) + try: + parsed = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, ValueError, RecursionError) as exc: + raise DeviceConnectError( + "Engraphis Cloud returned an invalid connect response, so no session was " + "saved." + _SPENT_TOKEN_SUFFIX, + status=502, + ) from exc + if not isinstance(parsed, dict): + raise DeviceConnectError( + "Engraphis Cloud returned an invalid connect response, so no session was " + "saved." + _SPENT_TOKEN_SUFFIX, + status=502, + ) + return parsed + + +#: Fields worth echoing back to the customer. Deliberately excludes +#: ``refresh_credential`` and ``access_token``: the summary is printed. +_SUMMARY_FIELDS = ( + "organization_id", + "installation_id", + "device_id", + "member_id", + "workspace_id", + "token_subject", + "plan", + "cloud_access_active", + "cloud_features", + "entitlement_version", + "expires_in_seconds", +) + + +def summarize(response: dict) -> dict: + """Return the non-secret fields of a registration response, for display.""" + + summary = {} + for key in _SUMMARY_FIELDS: + if key in response: + summary[key] = response[key] + return summary + + +def _preflight_session_storage() -> Path: + """Refuse a connect *before* the token is spent when the session cannot be saved. + + The exchange is the point of no return: the control plane consumes the single-use + connect token as it answers, so any storage fault discovered afterwards costs the + customer a fresh token from the portal. Delegated to :mod:`engraphis.cloud_session` + because that module owns the paths, the lock and the atomic write this is checking -- + a private copy of those rules here would drift from the save it is meant to predict. + """ + + try: + return cloud_session.preflight_save() + except cloud_session.CloudSessionError as exc: + raise DeviceConnectError( + "%s Your connect token has not been used, so you can fix this and run " + "`engraphis connect --token ...` again with the same token." % exc, + status=getattr(exc, "status", 409), + ) from exc + + +def connect(token: object, *, control_url: Optional[str] = None, + compute_url: Optional[str] = None, workspace_id: Optional[str] = None, + installation_label: Optional[str] = None, device_name: Optional[str] = None, + timeout: float = DEFAULT_TIMEOUT_SECONDS) -> dict: + """Exchange a connect token for a saved cloud session. + + Returns the redacted summary -- it is safe to print. Raises + :class:`DeviceConnectError` for every failure, with copy the customer can act on and + never containing the token. Nothing is written unless the exchange succeeded. + """ + + # Argument checks first: a bad ``--timeout`` must be reported as a bad timeout, not + # masked by whatever the identity or storage pre-flight happens to hit on the way to + # the same rejection inside ``post_connect``. + timeout = _validated_timeout(timeout) + normalized = normalize_connect_token(token) + resolved_control = _validated_control_url( + control_url if control_url is not None else default_control_url() + ) + cli_compute = str(compute_url or "").strip() + environment_compute = os.environ.get("ENGRAPHIS_CLOUD_COMPUTE_URL", "").strip() + has_explicit_compute = bool(cli_compute or environment_compute) + resolved_compute = cli_compute or environment_compute + # The shipped manifest/default remains a fallback, not an authority over a value the + # control plane returns for this installation. Validate it before redeeming the + # single-use token, then let a separately validated server value supersede it below. + fallback_compute = "" if has_explicit_compute else default_compute_url(resolved_control) + if resolved_compute: + try: + resolved_compute = validate_cloud_base_url(resolved_compute) + except CloudUrlUnresolved as exc: + raise DeviceConnectError( + "The Engraphis Cloud compute endpoint is temporarily unreachable.", + status=503, + ) from exc + except ValueError as exc: + raise DeviceConnectError( + "The Engraphis Cloud compute URL is not a valid HTTPS endpoint.", + status=400, + ) from exc + elif fallback_compute: + try: + fallback_compute = validate_cloud_base_url(fallback_compute) + except CloudUrlUnresolved as exc: + raise DeviceConnectError( + "The Engraphis Cloud compute endpoint is temporarily unreachable.", + status=503, + ) from exc + except ValueError as exc: + raise DeviceConnectError( + "The Engraphis Cloud compute URL is not a valid HTTPS endpoint.", + status=400, + ) from exc + + installation_client_id, device_client_id = client_identity() + # Last check before the point of no return. ``client_identity`` may have written its + # file minutes or months ago, so a writable state directory then is no evidence of one + # now; prove the session can land *before* the POST spends the token, not after. + session_path = _preflight_session_storage() + response = post_connect( + resolved_control, + normalized, + installation_client_id=installation_client_id, + device_client_id=device_client_id, + installation_label=installation_label, + device_name=device_name if device_name is not None else _default_device_name(), + app_platform=_default_platform(), + app_version=str(CURRENT_VERSION), + workspace_id=workspace_id, + timeout=timeout, + ) + # ``text_field`` and not ``str(... or "")``: a JSON array or object arrives as a Python + # ``list``/``dict`` whose ``repr`` is truthy and non-empty, so the coercion accepted a + # credential that is not a credential, wrote it, and reported a connection that could + # never refresh. Checked with the same helper the writer uses so the two cannot + # disagree about what counts as present. + if not cloud_session.text_field(response, "refresh_credential"): + # Reaching here means a 200 was parsed, and the control plane consumes the + # single-use connect token as it writes one. So "try again" would be actively + # wrong: re-running the same command deterministically returns 401 and still leaves + # no session. Point at the portal, the same way the truncated-reply copy does. + raise DeviceConnectError( + "Engraphis Cloud accepted the token but returned no session credential, so " + "no session was saved." + _SPENT_TOKEN_SUFFIX, + status=502, + ) + server_compute = "" if has_explicit_compute else _server_compute_url(response) + if not has_explicit_compute: + # The response wins for a non-shipped control plane too; otherwise those customers + # can redeem a token but never learn the compute endpoint the cloud assigned them. + resolved_compute = server_compute or fallback_compute + compute_source = ( + "explicit" if has_explicit_compute else "server" if server_compute + else "fallback" if fallback_compute else None + ) + try: + cloud_session.save_bootstrap( + response, + control_url=resolved_control, + compute_url=resolved_compute or None, + compute_url_source=compute_source, + ) + except cloud_session.CloudSessionError as exc: + # Also post-redemption. The pre-flight proved this path writable moments ago, so + # arriving here is a race -- typically the refresh lock disappearing or turning + # unsafe, which ``cloud_session`` wraps in a ``CloudSessionError``, which is exactly + # why it does not reach the ``OSError`` clause below that carries the spent-token + # warning. Forwarding the bare lock message left the customer retrying a consumed + # token. ``str(exc)`` is this package's own fixed copy, never provider text. + raise DeviceConnectError( + "Engraphis Cloud accepted the token, but the session could not be saved: " + "%s%s" % (exc, _SPENT_TOKEN_SUFFIX), + status=getattr(exc, "status", 503), + ) from exc + except OSError as exc: + # The pre-flight cleared this exact path moments ago, so arriving here means the + # state directory changed underneath the exchange. ``UnsafeStateFile`` is an + # ``OSError`` and not a ``CloudSessionError``, so without this clause it escapes + # as a raw traceback at the worst possible moment -- the token is already spent, + # and the customer needs to be told that plainly rather than shown a stack. + raise DeviceConnectError( + "Engraphis Cloud accepted the token, but the session could not be written to " + "%s. Fix that path, then connect again with a new token from your account " + "portal -- this one has been used." % session_path, + status=409, + ) from exc + except ValueError as exc: + # ``save_bootstrap`` re-runs ``validate_cloud_base_url`` on both endpoints, and + # that helper *resolves* the host. A resolver that dies between the pre-POST check + # and this line raises ``CloudUrlUnresolved``; an endpoint that starts resolving to + # a rejected address raises a bare ``ValueError``. Both are ``ValueError``, so + # neither the ``CloudSessionError`` nor the ``OSError`` clause above covered them + # and they escaped as a traceback -- again after the token was already spent. + raise DeviceConnectError( + "Engraphis Cloud accepted the token, but its endpoints could not be verified " + "in time to save the session, so nothing was written. Check your network, " + "then connect again with a new token from your account portal -- this one " + "has been used.", + status=409, + ) from exc + + summary = summarize(response) + summary["control_url"] = resolved_control + summary["compute_url"] = resolved_compute + summary["session_path"] = str(session_path) + return summary diff --git a/engraphis/engines/intelligence.py b/engraphis/engines/intelligence.py index ba1fcce..04df293 100644 --- a/engraphis/engines/intelligence.py +++ b/engraphis/engines/intelligence.py @@ -172,5 +172,6 @@ def _parse_json(raw: str) -> dict[str, Any]: text = json_match.group(0) try: return json.loads(text) - except Exception: + except Exception as exc: + logger.debug("JSON parse fallback to raw (%s)", type(exc).__name__) return {"raw": raw} diff --git a/engraphis/hosted_client.py b/engraphis/hosted_client.py index f889efe..e70445a 100644 --- a/engraphis/hosted_client.py +++ b/engraphis/hosted_client.py @@ -89,6 +89,16 @@ def upgrade_url(plan: Optional[str] = None) -> str: return value or DEFAULT_CLOUD_URL +def account_url() -> str: + """Return the plan-neutral hosted account URL. + + ``upgrade_url()`` is a checkout selector and defaults to Pro, so it can resolve to a + plan-specific checkout. Account and billing actions must prefer the generic portal. + """ + + return os.environ.get("ENGRAPHIS_UPGRADE_URL", "").strip() or DEFAULT_CLOUD_URL + + def _is_loopback_host(host: str) -> bool: if host == "localhost": return True diff --git a/engraphis/licensing.py b/engraphis/licensing.py index 4f7e82f..6b4ac02 100644 --- a/engraphis/licensing.py +++ b/engraphis/licensing.py @@ -13,6 +13,7 @@ TRIAL_DAYS, TRIAL_SECONDS, HostedFeatureError, + account_url, required_plan, upgrade_url, ) @@ -29,6 +30,7 @@ def production_warnings() -> list: __all__ = [ "LicenseError", + "account_url", "MAX_LOCAL_WRITE_GRACE_SECONDS", "TRIAL_DAYS", "TRIAL_SECONDS", diff --git a/engraphis/llm/client.py b/engraphis/llm/client.py index 5dbc96f..0439b9b 100644 --- a/engraphis/llm/client.py +++ b/engraphis/llm/client.py @@ -381,5 +381,6 @@ def _parse_json_response(raw: str) -> dict[str, Any]: text = text.rsplit("```", 1)[0] try: return json.loads(text) - except Exception: + except Exception as exc: + logger.debug("LLM JSON parse fallback to raw (%s)", type(exc).__name__) return {"raw": raw} diff --git a/engraphis/routes/memory.py b/engraphis/routes/memory.py index 986b94c..7e50eeb 100644 --- a/engraphis/routes/memory.py +++ b/engraphis/routes/memory.py @@ -445,8 +445,8 @@ def _add(ns: str, did: Optional[str]) -> None: for r in conn.execute(ev_sql + " ORDER BY timestamp DESC LIMIT 100", ev_params): try: _add(r["namespace"], _json.loads(r["payload"] or "{}").get("document_id")) - except Exception: - pass + except Exception as exc: + logger.debug("Entity event payload parse skipped (%s)", type(exc).__name__) # 2) broad: memories whose content mentions the entity m_sql = "SELECT namespace, document_id FROM memories WHERE content LIKE ? ESCAPE '\\'" @@ -619,7 +619,10 @@ async def list_thoughts(namespace: Optional[str] = None, limit: int = 50): thoughts = [] for r in rows: d = dict(r) - d["source_memory_ids"] = json.loads(d.get("source_memory_ids") or "[]") + try: + d["source_memory_ids"] = json.loads(d.get("source_memory_ids") or "[]") + except Exception: + d["source_memory_ids"] = [] try: d["parsed"] = json.loads(d["content"]) except Exception: @@ -640,9 +643,6 @@ async def get_config(): "embed_model": settings.embed_model, "loop_interval": settings.loop_interval, "decay_halflife_days": settings.decay_halflife_days, - "host": settings.host, - "port": settings.port, - "base_url": settings.base_url, }) diff --git a/engraphis/routes/v2_api.py b/engraphis/routes/v2_api.py index cee65ef..aa9b721 100644 --- a/engraphis/routes/v2_api.py +++ b/engraphis/routes/v2_api.py @@ -7,9 +7,11 @@ """ from __future__ import annotations +import datetime as _datetime import json import hmac import logging +import math import os import threading import time @@ -1916,6 +1918,102 @@ def _normalized_plan(value: object) -> str: return plan if plan in ("pro", "team") else "local" +#: The control plane's entitlement status vocabulary, mirrored from (read-only) +#: engraphis-cloud/engraphis_cloud/entitlements.py ``effective_status`` and the statuses +#: ``/internal/subscriptions/apply`` accepts. Presentation only: an unrecognised value is +#: reported as ``""`` rather than guessed at, which degrades the copy to the generic +#: wording instead of asserting something the server never said. +_ENTITLEMENT_STATUSES = frozenset({ + "active", "trialing", "past_due", "canceled", "expired", "revoked", "scheduled", + "inactive", +}) + + +def _normalized_status(value: object) -> str: + """Return the control plane's entitlement status, or ``""`` when it said nothing.""" + + status = str(value or "").strip().lower() + return status if status in _ENTITLEMENT_STATUSES else "" + + +def _epoch_seconds(value: object) -> float: + """Return an ISO-8601 instant as epoch seconds, or ``0.0``. Never raises. + + The control plane serializes ``trial_ends_at`` as an ISO-8601 UTC timestamp; the + dashboard renders instants as epoch seconds. Converting once, here, keeps the JS from + parsing dates the CSP-externalized asset cannot be unit-tested through. + + ``datetime.fromisoformat`` on this package's Python 3.9 floor does not accept the + trailing ``Z`` that Pydantic emits for UTC, so it is rewritten to the explicit offset + first. Anything it still cannot parse is reported as "unknown" rather than raising on + the ``/api/bootstrap`` boot path. + """ + + if isinstance(value, bool): + return 0.0 + if isinstance(value, (int, float)): + try: + number = float(value) + except (OverflowError, ValueError): + # JSON integers are arbitrary precision. An unrepresentable instant is not a + # boundary and must not break /api/license or the dashboard bootstrap path. + return 0.0 + return number if number > 0 and math.isfinite(number) else 0.0 + if not isinstance(value, str) or not value.strip(): + return 0.0 + text = value.strip() + if text[-1:] in ("Z", "z"): + text = text[:-1] + "+00:00" + try: + parsed = _datetime.datetime.fromisoformat(text) + except (ValueError, TypeError): + return 0.0 + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=_datetime.timezone.utc) + try: + number = float(parsed.timestamp()) + except (OverflowError, OSError, ValueError): + return 0.0 + return number if number > 0 and math.isfinite(number) else 0.0 + + +def _trial_facts(source: object) -> dict: + """Read the control plane's trial disclosure off any entitlement-shaped mapping. + + ``DeviceRegistrationResponse`` and ``GET /v1/entitlements/{org}`` carry the same four + fields, so one reader serves both and the two answers cannot be parsed by different + rules. Every field is optional: a control plane that predates them simply omits them, + and the honest answer is then "not a trial, none consumed" rather than a claim. + + ``trial_consumed`` is deliberately widened by ``is_trial``: an organization that is on + a trial has by definition consumed one, so a server that answers only ``is_trial`` + still stops this client offering a second trial the server would refuse. + """ + + if not isinstance(source, dict): + return {"status": "", "is_trial": False, "trial_consumed": False, + "trial_ends_at": 0.0} + is_trial = source.get("is_trial") + is_trial = bool(is_trial) if isinstance(is_trial, bool) else False + consumed = source.get("trial_consumed") + consumed = bool(consumed) if isinstance(consumed, bool) else False + return { + "status": _normalized_status(source.get("status")), + "is_trial": is_trial, + "trial_consumed": consumed or is_trial, + # A trial boundary is only meaningful for a trial. Replaying a converted + # customer's long-past trial end would tell a paying subscriber their access + # expired last month, which is exactly what the server refuses to do. + "trial_ends_at": _epoch_seconds(source.get("trial_ends_at")) if is_trial else 0.0, + } + + +def _unknown_trial_facts() -> dict: + """Return the "the control plane has told us nothing" trial answer.""" + + return {"status": "", "is_trial": False, "trial_consumed": False, "trial_ends_at": 0.0} + + def _normalized_features(values: object, plan: str) -> list: """Keep the server's own grant, expanded to the names this dashboard renders. @@ -1957,7 +2055,7 @@ def _session_entitlement() -> dict: return {} plan = _normalized_plan(declared.get("plan")) active = bool(declared.get("cloud_access_active")) - return { + resolved = { "plan": plan, # The server empties ``cloud_features`` the moment paid access stops being # live; mirror that, exactly as the entitlements route's answer does below. @@ -1969,6 +2067,8 @@ def _session_entitlement() -> dict: "organization_id": str(declared.get("organization_id") or ""), "fetched_at": float(declared.get("entitlement_checked_at") or 0.0), } + resolved.update(_trial_facts(declared)) + return resolved except Exception: # noqa: BLE001 - a badge must never break /bootstrap return {} @@ -2029,13 +2129,18 @@ def _read_entitlement_cache() -> dict: if not current or organization_id != current: return {} plan = _normalized_plan(stored_plan) - return { + resolved = { "plan": plan, "features": _normalized_features(value.get("features"), plan), "cloud_access_active": bool(value.get("cloud_access_active")), "organization_id": organization_id, "fetched_at": fetched_at, } + # The trial disclosure is cached under the same wire names the entitlements route uses, + # so a cache written by an older build simply has none of them and reads back as "not a + # trial" rather than as a corrupt entry. + resolved.update(_trial_facts(value)) + return resolved def _write_entitlement_cache(entitlement: dict) -> None: @@ -2059,6 +2164,13 @@ def _write_entitlement_cache(entitlement: dict) -> None: "cloud_access_active": bool(entitlement["cloud_access_active"]), "organization_id": str(entitlement.get("organization_id") or ""), "fetched_at": float(entitlement.get("fetched_at") or time.time()), + # Persisted under the wire names ``_trial_facts`` reads, so the cache round + # trips through exactly one parser. ``trial_ends_at`` is already epoch seconds + # by this point, which that parser also accepts. + "status": _normalized_status(entitlement.get("status")), + "is_trial": bool(entitlement.get("is_trial")), + "trial_consumed": bool(entitlement.get("trial_consumed")), + "trial_ends_at": float(entitlement.get("trial_ends_at") or 0.0), }, sort_keys=True, separators=(",", ":"))) except Exception: # noqa: BLE001 - losing the cache write must not surface anywhere logger.debug("entitlement cache write skipped") @@ -2092,6 +2204,11 @@ def _deny_entitlement_cache() -> None: denied = dict(cached) denied["cloud_access_active"] = False denied["features"] = [] + # The last status the server named now contradicts this denial, so it stops being + # renderable copy. The trial facts survive: a lapse does not un-consume a trial or + # move its boundary, and they are what distinguishes "your free trial ended" from + # "your subscription lapsed" in the panel this settles. + denied["status"] = "" denied["fetched_at"] = time.time() _write_entitlement_cache(denied) except Exception: # noqa: BLE001 - a denial we cannot persist is still a denial @@ -2190,6 +2307,18 @@ def _fetch_authoritative_entitlement() -> Optional[dict]: exc.close() except (OSError, ValueError): pass + # The compatibility endpoint is authoritative too. An older control plane can + # refresh a token successfully yet answer 402 here because it does not include + # entitlement fields in the refresh response. Do not leave its previous paid + # cache (or the session that outranks it) advertising access after that answer. + if exc.code == 402: + try: + from engraphis.cloud_session import record_billing_denial + + record_billing_denial() + except Exception: # noqa: BLE001 - a denial we cannot persist is still a denial + pass + _deny_entitlement_cache() return None except Exception: # noqa: BLE001 - transport, TLS, and URL failures are all "not now" return None @@ -2216,7 +2345,7 @@ def _fetch_authoritative_entitlement() -> Optional[dict]: if not isinstance(active, bool): return None plan = _normalized_plan(declared_plan) - return { + resolved = { "plan": plan, # The server empties ``cloud_features`` the moment paid access stops being live. # Mirroring that re-draws the locks on a lapsed subscription while the badge keeps @@ -2227,6 +2356,11 @@ def _fetch_authoritative_entitlement() -> Optional[dict]: "organization_id": organization_id, "fetched_at": time.time(), } + # ``is_trial``/``trial_consumed``/``trial_ends_at``/``status`` are optional here for the + # same reason the fields above the fallback are: a control plane that predates them + # omits them, and the honest answer is then "not a trial", not a refusal to cache. + resolved.update(_trial_facts(body)) + return resolved def _refresh_entitlement_in_background(known: dict) -> None: @@ -2307,34 +2441,61 @@ def _plan_entitlement() -> dict: declared = os.environ.get("ENGRAPHIS_CLOUD_PLAN", "").strip().lower() if declared in ("pro", "team", "free", "local"): plan = declared if declared in ("pro", "team") else "local" - return {"plan": plan, "features": entitled_features(plan), - "source": "environment", "cloud_access_active": plan != "local", - "checked_at": 0.0} + # An operator override names a plan, never a trial: it exists for air-gapped and + # pinned-token deployments that have no control plane to ask. Reporting "no trial, + # none consumed" is the honest answer, and it is what keeps the trial CTA off a + # deployment that cannot start one. + entitlement = {"plan": plan, "features": entitled_features(plan), + "source": "environment", "cloud_access_active": plan != "local", + "checked_at": 0.0} + entitlement.update(_unknown_trial_facts()) + return entitlement try: from engraphis import cloud_session connected = cloud_session.configured(require_compute=False) except Exception: # noqa: BLE001 - a badge must never break /bootstrap connected = False if not connected: - return {"plan": "local", "features": [], "source": "local", - "cloud_access_active": False, "checked_at": 0.0} + entitlement = {"plan": "local", "features": [], "source": "local", + "cloud_access_active": False, "checked_at": 0.0} + entitlement.update(_unknown_trial_facts()) + return entitlement session = _session_entitlement() if session: _refresh_entitlement_in_background(session) - return {"plan": session["plan"], "features": list(session["features"]), - "source": "session", "cloud_access_active": session["cloud_access_active"], - "checked_at": session["fetched_at"]} + return _resolved_entitlement(session, source="session") cached = _read_entitlement_cache() _refresh_entitlement_in_background(cached) if cached: - return {"plan": cached["plan"], "features": list(cached["features"]), - "source": "cloud", "cloud_access_active": cached["cloud_access_active"], - "checked_at": cached["fetched_at"]} + return _resolved_entitlement(cached, source="cloud") # Connected, but the control plane has never answered (first boot after onboarding, or # offline since). ``pro`` unlocks what every paid plan includes and leaves only the Team # upsell showing; the refresh scheduled above corrects it. - return {"plan": "pro", "features": entitled_features("pro"), "source": "connected", - "cloud_access_active": True, "checked_at": 0.0} + # + # It says nothing about a trial either way. Guessing "no trial consumed" here would + # re-offer the trial CTA to a connected organization the server will refuse -- a + # connected installation always has an organization, and that organization is on a + # trial, has spent one, or is paying -- so this reports the trial as consumed and lets + # the refresh replace the guess with the answer. + entitlement = {"plan": "pro", "features": entitled_features("pro"), + "source": "connected", "cloud_access_active": True, "checked_at": 0.0} + entitlement.update(_unknown_trial_facts()) + entitlement["trial_consumed"] = True + return entitlement + + +def _resolved_entitlement(known: dict, *, source: str) -> dict: + """Shape one persisted answer into the resolver's return value.""" + + resolved = { + "plan": known["plan"], + "features": list(known["features"]), + "source": source, + "cloud_access_active": known["cloud_access_active"], + "checked_at": known["fetched_at"], + } + resolved.update(_trial_facts(known)) + return resolved def _hosted_plan() -> str: @@ -2375,15 +2536,57 @@ def hosted_plan_summary() -> dict: entitlement = {"plan": plan, "features": entitled_features(plan), "source": "override", "cloud_access_active": plan != "local", "checked_at": 0.0} + entitlement.update(_unknown_trial_facts()) + access_state = _access_state(entitlement) + access_live = access_state in ("active", "trial") return { "plan": entitlement["plan"], - "features": list(entitlement["features"]), + # A cached trial can cross its known boundary while this installation is offline. + # Keep the durable server answer intact, but never present its stale grants as live. + "features": list(entitlement["features"]) if access_live else [], "plan_source": entitlement["source"], - "cloud_access_active": entitlement["cloud_access_active"], + "cloud_access_active": access_live, "plan_checked_at": entitlement["checked_at"], + "entitlement_status": entitlement["status"], + "is_trial": entitlement["is_trial"], + "trial_consumed": entitlement["trial_consumed"], + "trial_ends_at": entitlement["trial_ends_at"], + "access_state": access_state, } +#: Why the customer's hosted features are, or are not, available right now. The dashboard +#: renders one explanation per value; every one of them is a different thing to tell the +#: customer and a different thing to ask them to do. +#: +#: * ``trial`` — a live trial. Say when it ends and offer to buy. +#: * ``active`` — a live paid subscription. Nothing to explain. +#: * ``trial_expired`` — the free trial ran out. Offer to buy; never offer another trial. +#: * ``lapsed`` — a subscription that is no longer live (cancelled, expired, unpaid). +#: Send the customer to billing, not to a trial they cannot start. +#: * ``inactive`` — no hosted plan at all. This is the only state a trial is offerable in. +#: +#: Before this existed, the last three were indistinguishable on screen: the client kept +#: ``plan="pro"`` with an emptied feature list, so a customer whose trial had ended and a +#: customer whose card had failed both saw a confident PRO badge over rows of locks with +#: no reason given. +_ACCESS_STATES = ("active", "trial", "trial_expired", "lapsed", "inactive") + + +def _access_state(entitlement: dict) -> str: + """Classify why hosted features are available or locked. Never raises.""" + + if entitlement.get("plan") not in ("pro", "team"): + return "inactive" + is_trial = bool(entitlement.get("is_trial")) + trial_ends_at = _epoch_seconds(entitlement.get("trial_ends_at")) if is_trial else 0.0 + if trial_ends_at and trial_ends_at <= time.time(): + return "trial_expired" + if entitlement.get("cloud_access_active"): + return "trial" if is_trial else "active" + return "trial_expired" if is_trial else "lapsed" + + @router.get("/license") def get_license(): """Hosted plan presentation for the dashboard; Cloud remains the authority. @@ -2407,16 +2610,43 @@ def get_license(): "plan_source": summary["plan_source"], "plan_checked_at": summary["plan_checked_at"], "cloud_access_active": summary["cloud_access_active"], - # The client cannot distinguish a trial from a paid subscription; the control - # plane owns that. Report the honest default so the badge falls back to the plan - # name instead of silently claiming a trial the customer may not be on. - "is_trial": False, - "trial": {"used": False, "trial_days": licensing.TRIAL_DAYS}, + # Why the hosted features above are, or are not, live. The dashboard renders one + # explanation per value rather than a plan badge over unexplained locks. + "access_state": summary["access_state"], + # The control plane's own entitlement status, when it named one. ``""`` means it + # has not, or that its last answer was contradicted by a billing denial. + "entitlement_status": summary["entitlement_status"], + # The control plane owns the trial, and now says so on the calls this client + # already makes. Hardcoding these to ``False`` made the dashboard's TRIAL badge + # unreachable and, because ``used`` never became true, offered "Start your free + # trial" forever — to active subscribers and to customers whose trial was already + # spent, both of whom the server answers with ``TrialAlreadyConsumedError``. + "is_trial": summary["is_trial"], + "trial": { + "used": summary["trial_consumed"], + "active": summary["access_state"] == "trial", + # Epoch seconds, ``0`` when there is no live trial boundary to disclose. A + # converted paying customer is deliberately never told about a past one. + "ends_at": summary["trial_ends_at"], + # A trial is offerable only to an installation that belongs to no organization + # yet. ``start_trial`` refuses every organization that already holds an + # entitlement, so a connected customer — trialling, lapsed, or paying — can + # only ever be answered 409 by the button. + "available": ( + not summary["trial_consumed"] + and summary["access_state"] == "inactive" + and summary["plan_source"] == "local" + ), + "trial_days": licensing.TRIAL_DAYS, + }, "cloud_managed": True, "trial_seconds": 259_200, "grace_seconds": 86_400, "grace_scope": "existing authenticated local workspace writes only", "upgrade_url": licensing.upgrade_url(), + # Plan-neutral account and billing entry point. ``upgrade_url()`` defaults to the + # Pro checkout and is therefore not a safe substitute when the targets differ. + "account_url": licensing.account_url(), # Pro and Team bill through separate checkout targets. Emitting only the generic # URL sent every Team upgrade click to the Pro page. "pro_upgrade_url": licensing.upgrade_url("pro"), diff --git a/engraphis/routes/vault.py b/engraphis/routes/vault.py index c502a1a..91261f3 100644 --- a/engraphis/routes/vault.py +++ b/engraphis/routes/vault.py @@ -1,30 +1,99 @@ """Vault management, file editing, folder import, memory health, bulk ops, and context preview routes.""" from __future__ import annotations +import asyncio +import heapq import logging import time +from collections import defaultdict from pathlib import Path from typing import Any, Optional import numpy as np -from fastapi import APIRouter, HTTPException, Query, UploadFile, File, Form +from fastapi import APIRouter, File, Form, HTTPException, Query, Request, UploadFile +from fastapi.routing import APIRoute from pydantic import BaseModel +from starlette.exceptions import HTTPException as StarletteHTTPException from engraphis.engines import embedder, ingest as ingest_engine, recall as recall_engine, reweight +from engraphis.service import MAX_IMPORT_FILES, MAX_IMPORT_RESOURCE_BYTES, MAX_IMPORT_TOTAL_BYTES from engraphis.engines.intelligence import auto_categorize, check_conflicts from engraphis.engines.reweight import retention_score -from engraphis.stores import get_conn, now_ts +from engraphis.stores import blob_to_vector, get_conn, now_ts from engraphis.stores import vaults as vault_store from engraphis.stores import vectors as mem_store logger = logging.getLogger("engraphis.routes.vault") -router = APIRouter(prefix="/memory", tags=["vault-management"]) +# Multipart boundaries and per-part headers count toward the HTTP request size even +# though they are not imported content. Keep the transport ceiling finite while allowing +# the documented content limit plus conservative multipart overhead. +VAULT_UPLOAD_REQUEST_BYTES = ( + MAX_IMPORT_TOTAL_BYTES + MAX_IMPORT_FILES * 16_384 + 1024 * 1024 +) +_UPLOAD_FORM_FIELDS = 8 +_DUPLICATE_CANDIDATE_LIMIT = 500 +_DUPLICATE_RESULT_LIMIT = 200 +_DUPLICATE_BLOCK_SIZE = 256 + + +class _BoundedUploadRoute(APIRoute): + """Parse vault uploads with their strict multipart limits before FastAPI binds files.""" + + def get_route_handler(self): + route_handler = super().get_route_handler() + + async def bounded_route_handler(request: Request): + # FastAPI normally resolves ``UploadFile`` parameters before dependencies. + # Parsing here runs first and caches the bounded FormData on this same request. + await _bounded_upload_form(request) + return await route_handler(request) + + return bounded_route_handler + + +class _VaultRouter(APIRouter): + """Install the bounded parser only on the two multipart folder-import routes.""" + + _bounded_upload_paths = { + "/vaults/upload-folder", + "/vaults/upload-folder-smart", + } + + def add_api_route(self, path: str, endpoint, **kwargs): + if path in self._bounded_upload_paths: + kwargs["route_class_override"] = _BoundedUploadRoute + return super().add_api_route(path, endpoint, **kwargs) + + +router = _VaultRouter(prefix="/memory", tags=["vault-management"]) def _ok(data: Any) -> dict[str, Any]: return {"data": data} +async def _bounded_upload_form(request: Request) -> None: + """Parse multipart once with a strict file-count ceiling. + + FastAPI otherwise parses ``UploadFile`` dependencies with Starlette's default + 1,000-file ceiling before the route can inspect ``len(files)``. The app-level + middleware separately bounds bytes before this parser is allowed to spool them. + """ + try: + await request.form( + max_files=MAX_IMPORT_FILES, + max_fields=_UPLOAD_FORM_FIELDS, + ) + except StarletteHTTPException as exc: + detail = str(exc.detail) + if exc.status_code == 400 and detail.lower().startswith("too many files"): + raise HTTPException( + status_code=413, + detail={"error": f"too many files (max {MAX_IMPORT_FILES})"}, + ) from exc + raise + + # ═══ VAULT MANAGEMENT ═══════════════════════════════════════════════════════ class VaultCreateReq(BaseModel): @@ -267,7 +336,8 @@ async def import_folder(req: FolderImportReq): ) results["imported"] += 1 results["files"].append({"path": rel, "title": title, "status": "ok"}) - except Exception: + except Exception as exc: + logger.warning("Folder import file failed (%s)", type(exc).__name__) results["errors"] += 1 results["files"].append({"path": rel_path.as_posix(), "title": "", "status": "error"}) @@ -282,13 +352,24 @@ async def upload_folder( ): """POST /memory/vaults/upload-folder — upload multiple files as a folder (multipart). Use webkitdirectory in the frontend to send an entire folder.""" + if len(files) > MAX_IMPORT_FILES: + raise HTTPException(status_code=413, detail={"error": f"too many files (max {MAX_IMPORT_FILES})"}) if not vault_store.get_vault(namespace): vault_store.create_vault(namespace=namespace, name=namespace) results = {"imported": 0, "errors": 0, "files": []} + total_bytes = 0 for f in files: try: - content = f.file.read().decode("utf-8", errors="replace") + raw = f.file.read(MAX_IMPORT_RESOURCE_BYTES + 1) + if len(raw) > MAX_IMPORT_RESOURCE_BYTES: + results["errors"] += 1 + results["files"].append({"path": f.filename, "title": "", "status": "error", "error": "file too large"}) + continue + total_bytes += len(raw) + if total_bytes > MAX_IMPORT_TOTAL_BYTES: + raise HTTPException(status_code=413, detail={"error": f"upload batch exceeds {MAX_IMPORT_TOTAL_BYTES} bytes"}) + content = raw.decode("utf-8", errors="replace") if not content.strip(): continue import re @@ -304,9 +385,12 @@ async def upload_folder( ) results["imported"] += 1 results["files"].append({"path": f.filename, "title": title, "status": "ok"}) - except Exception as e: + except HTTPException: + raise + except Exception as exc: results["errors"] += 1 - results["files"].append({"path": f.filename, "title": "", "status": "error", "error": str(e)}) + logger.warning("Folder upload file failed (%s)", type(exc).__name__) + results["files"].append({"path": f.filename, "title": "", "status": "error", "error": "processing failed"}) return _ok({"namespace": namespace, **results}) @@ -325,6 +409,8 @@ async def upload_folder_smart( If auto_categorize_flag is 'true', each file is classified by the LLM into the correct memory type. Uses batch embedding for speed.""" import re as _re + if len(files) > MAX_IMPORT_FILES: + raise HTTPException(status_code=413, detail={"error": f"too many files (max {MAX_IMPORT_FILES})"}) if not vault_store.get_vault(namespace): vault_store.create_vault(namespace=namespace, name=namespace) @@ -333,9 +419,18 @@ async def upload_folder_smart( # Phase 1: Read all files and prepare content file_data = [] + total_bytes = 0 for f in files: try: - content = f.file.read().decode("utf-8", errors="replace") + raw = f.file.read(MAX_IMPORT_RESOURCE_BYTES + 1) + if len(raw) > MAX_IMPORT_RESOURCE_BYTES: + results["errors"] += 1 + results["files"].append({"path": f.filename, "title": "", "status": "error", "error": "file too large"}) + continue + total_bytes += len(raw) + if total_bytes > MAX_IMPORT_TOTAL_BYTES: + raise HTTPException(status_code=413, detail={"error": f"upload batch exceeds {MAX_IMPORT_TOTAL_BYTES} bytes"}) + content = raw.decode("utf-8", errors="replace") if not content.strip(): results["skipped"] += 1 continue @@ -343,16 +438,20 @@ async def upload_folder_smart( title = title_match.group(1).strip() if title_match else Path(f.filename).stem doc_id = f.filename.replace("/", "__").replace("\\", "__").replace(".md", "").replace(".", "-") file_data.append({"filename": f.filename, "doc_id": doc_id, "title": title, "content": content}) - except Exception as e: + except HTTPException: + raise + except Exception as exc: results["errors"] += 1 - results["files"].append({"path": f.filename, "title": "", "status": "error", "error": str(e)}) + logger.warning("Smart import file read failed (%s)", type(exc).__name__) + results["files"].append({"path": f.filename, "title": "", "status": "error", "error": "processing failed"}) # Phase 2: Batch embed all files at once (10x faster than individual) if file_data: texts = [f"{fd['title']}\n\n{fd['content']}" for fd in file_data] try: vecs = embedder.embed_batch(texts) - except Exception: + except Exception as exc: + logger.warning("Batch embedding failed (%s), falling back to individual", type(exc).__name__) # Fallback: embed individually vecs = [embedder.embed(t) for t in texts] @@ -412,9 +511,10 @@ async def upload_folder_smart( "categorized": categorize_info is not None, "confidence": categorize_info.get("confidence", 0) if categorize_info else 0, }) - except Exception as e: + except Exception as exc: results["errors"] += 1 - results["files"].append({"path": fd["filename"], "title": "", "status": "error", "error": str(e)}) + logger.warning("Smart import ingest failed (%s)", type(exc).__name__) + results["files"].append({"path": fd["filename"], "title": "", "status": "error", "error": "processing failed"}) return _ok({"namespace": namespace, **results}) @@ -453,7 +553,8 @@ async def auto_categorize_memories(req: AutoCategorizeReq): "confidence": cat.get("confidence", 0), "reason": cat.get("reason", ""), }) - except Exception: + except Exception as exc: + logger.warning("Auto-categorize failed (%s)", type(exc).__name__) results["errors"] += 1 return _ok(results) @@ -477,27 +578,137 @@ async def conflict_check(req: ConflictCheckReq): # ═══ MEMORY HEALTH ══════════════════════════════════════════════════════════ -@router.get("/health/duplicates") -async def find_duplicates(namespace: Optional[str] = None, threshold: float = 0.85): - """GET /memory/health/duplicates — find near-duplicate memories by vector similarity.""" - candidates = mem_store.all_vectors(namespace=namespace) +def _duplicate_pairs( + candidates: list[tuple[str, str, np.ndarray, dict[str, str]]], + threshold: float, +) -> tuple[list[dict[str, Any]], int]: + """Find the strongest pairs with bounded block memory and response cardinality.""" + groups: dict[tuple[str, int], list[tuple[str, np.ndarray, dict[str, str]]]] = ( + defaultdict(list) + ) + for namespace, document_id, vector, memory in candidates: + # A stale vector from an old embedding dimension must not break all health data. + groups[(namespace, int(vector.size))].append((document_id, vector, memory)) + + strongest: list[tuple[float, int, int, int, list[tuple[str, np.ndarray, dict[str, str]]], str]] = [] + match_count = 0 + sequence = 0 + for (namespace, _dimension), memories in groups.items(): + if len(memories) < 2: + continue + vectors = np.stack([item[1] for item in memories]) + for start in range(0, len(memories), _DUPLICATE_BLOCK_SIZE): + stop = min(start + _DUPLICATE_BLOCK_SIZE, len(memories)) + similarities = vectors[start:stop] @ vectors.T + for local_index, absolute_index in enumerate(range(start, stop)): + similarities[local_index, :absolute_index + 1] = -np.inf + + flat = similarities.ravel() + matching = np.flatnonzero(flat >= threshold) + match_count += int(matching.size) + if matching.size > _DUPLICATE_RESULT_LIMIT: + relative = np.argpartition( + flat[matching], + -_DUPLICATE_RESULT_LIMIT, + )[-_DUPLICATE_RESULT_LIMIT:] + matching = matching[relative] + + for flat_index in matching: + local_index, right_index = divmod(int(flat_index), len(memories)) + left_index = start + local_index + similarity = float(similarities[local_index, right_index]) + entry = ( + similarity, + sequence, + left_index, + right_index, + memories, + namespace, + ) + sequence += 1 + if len(strongest) < _DUPLICATE_RESULT_LIMIT: + heapq.heappush(strongest, entry) + elif similarity > strongest[0][0]: + heapq.heapreplace(strongest, entry) + duplicates = [] - for i in range(len(candidates)): - for j in range(i + 1, len(candidates)): - id1, ns1, doc1, vec1, mem1 = candidates[i] - id2, ns2, doc2, vec2, mem2 = candidates[j] - if ns1 != ns2: - continue - sim = float(np.dot(vec1, vec2)) - if sim >= threshold: - duplicates.append({ - "namespace": ns1, - "memory_a": {"document_id": doc1, "title": mem1["title"], "content": mem1["content"][:200]}, - "memory_b": {"document_id": doc2, "title": mem2["title"], "content": mem2["content"][:200]}, - "similarity": round(sim, 4), - }) - duplicates.sort(key=lambda x: x["similarity"], reverse=True) - return _ok({"duplicates": duplicates, "count": len(duplicates)}) + for similarity, _, left_index, right_index, memories, namespace in sorted( + strongest, key=lambda item: (-item[0], item[1]) + ): + left_doc, _, left_mem = memories[left_index] + right_doc, _, right_mem = memories[right_index] + duplicates.append({ + "namespace": namespace, + "memory_a": { + "document_id": left_doc, + "title": left_mem["title"], + "content": left_mem["content"][:200], + }, + "memory_b": { + "document_id": right_doc, + "title": right_mem["title"], + "content": right_mem["content"][:200], + }, + "similarity": round(similarity, 4), + }) + return duplicates, match_count + + +def _duplicate_candidate_query(namespace: Optional[str]) -> tuple[str, list[Any]]: + """Build the bounded duplicate-candidate query without SQLite temporary sorting. + + A namespaced scan is ordered by newest update through ``idx_mem_updated``. A global + scan instead uses descending rowid/``id`` (newest insertion), because there is no + global ``updated_at`` index and combining the two would force a temp B-tree. + """ + sql = ( + "SELECT namespace, document_id, vector, title, content " + "FROM memories WHERE vector IS NOT NULL" + ) + params: list[Any] = [] + if namespace: + sql += " AND namespace=?" + params.append(namespace) + sql += " ORDER BY updated_at DESC" + else: + sql += " ORDER BY id DESC" + return sql, params + + +@router.get("/health/duplicates") +async def find_duplicates( + namespace: Optional[str] = None, + threshold: float = Query(0.85, ge=-1.0, le=1.0), +): + """Find a bounded set of strongest near-duplicates without blocking the event loop.""" + sql, params = _duplicate_candidate_query(namespace) + sql += " LIMIT ?" + params.append(_DUPLICATE_CANDIDATE_LIMIT + 1) + rows = get_conn().execute(sql, params).fetchall() + candidate_truncated = len(rows) > _DUPLICATE_CANDIDATE_LIMIT + candidates = [ + ( + row["namespace"], + row["document_id"], + blob_to_vector(row["vector"]), + {"title": row["title"], "content": row["content"]}, + ) + for row in rows[:_DUPLICATE_CANDIDATE_LIMIT] + ] + duplicates, match_count = await asyncio.to_thread( + _duplicate_pairs, candidates, threshold + ) + return _ok({ + "duplicates": duplicates, + "count": len(duplicates), + "matches_considered": match_count, + "candidate_count": len(candidates), + "candidate_limit": _DUPLICATE_CANDIDATE_LIMIT, + "result_limit": _DUPLICATE_RESULT_LIMIT, + "truncated": ( + candidate_truncated or match_count > _DUPLICATE_RESULT_LIMIT + ), + }) @router.get("/health/stale") diff --git a/engraphis/static/dashboard.css b/engraphis/static/dashboard.css index fc2c328..d57fef8 100644 --- a/engraphis/static/dashboard.css +++ b/engraphis/static/dashboard.css @@ -255,6 +255,9 @@ body{ .diff-ins{border-radius:var(--radius-sm);background:var(--color-success-bg);color:var(--color-success)} .diff-del{border-radius:var(--radius-sm);background:var(--color-danger-bg);color:var(--color-danger);text-decoration:line-through} .trial-banner{margin-bottom:var(--space-3);padding:var(--space-2) var(--space-3);border:var(--rule) solid var(--color-accent-strong);border-radius:var(--radius-sm);background:var(--accent-bg);color:var(--color-text);font-size:var(--text-xs)} +.lic-banner{margin:var(--space-3) 0;padding:var(--space-3);border:var(--rule) solid var(--color-border);border-radius:var(--radius-sm);background:var(--color-raised);color:var(--color-text-muted);font-size:var(--text-xs);line-height:1.5} +.lic-banner strong{display:block;margin-bottom:var(--space-1);color:var(--color-text);font-size:var(--text-sm)} +.lic-banner-warn{border-color:var(--color-accent-strong);background:var(--accent-bg)} .update-banner{display:flex;align-items:center;gap:var(--space-3);margin-bottom:var(--space-3);padding:var(--space-2) var(--space-3);border:var(--rule) solid var(--color-accent-strong);border-radius:var(--radius-sm);background:var(--accent-bg);color:var(--color-text);font-size:var(--text-xs)} .update-banner[hidden]{display:none} .update-banner .ub-text{flex:1;min-width:0} @@ -743,3 +746,8 @@ progress.graph-degree[data-graph-node-type="person_or_concept"]::-webkit-progres .gslider{display:flex;align-items:center;gap:8px;margin:6px 0}.gslider label{width:88px;flex-shrink:0;font-size:12px;color:var(--color-text-mute,var(--color-text-dim))}.gslider input[type=range]{flex:1;min-width:0;accent-color:var(--color-accent)} .gx-chip-full{width:100%;justify-content:flex-start;border-radius:7px;margin-bottom:2px} .gx-dot-dim{background:var(--color-text-dim,#888)} +#graph-net[data-graph-style="galaxy"]{background:radial-gradient(58% 50% at 24% 22%,rgba(126,64,208,.30),transparent 66%),radial-gradient(52% 58% at 82% 78%,rgba(220,72,164,.20),transparent 68%),radial-gradient(46% 52% at 62% 42%,rgba(58,120,224,.16),transparent 70%),#06040f} +#graph-net[data-graph-style="solar"]{background:radial-gradient(40% 48% at 50% 50%,rgba(255,184,92,.16),transparent 60%),radial-gradient(90% 90% at 50% 50%,rgba(18,32,64,.55),transparent 82%),#05070d} +#graph-net[data-graph-style="cyber"]{background:linear-gradient(rgba(34,224,255,.055) 1px,transparent 1px) 0 0/30px 30px,linear-gradient(90deg,rgba(34,224,255,.055) 1px,transparent 1px) 0 0/30px 30px,radial-gradient(72% 60% at 50% 0%,rgba(255,62,165,.12),transparent 72%),#050810} +#graph-net.engraphis-graph-node-hover{cursor:pointer} +#graph-net:not(.engraphis-graph-node-hover){cursor:grab} diff --git a/engraphis/static/dashboard.js b/engraphis/static/dashboard.js index d9da2e8..869549f 100644 --- a/engraphis/static/dashboard.js +++ b/engraphis/static/dashboard.js @@ -105,6 +105,9 @@ function selectView(v){ /* Plan pills live in the topbar; clear them on navigation so only the active view's loader can repopulate one. */ ['an-lock','au-lock'].forEach(id=>{const p=document.getElementById(id);if(p){p.textContent='';p.className='pill pill-muted topbar-lock'}}); + /* The graph canvas is the only view that owns an animation loop. Park it while it is not + on screen so navigating away does not leave a hidden canvas repainting forever. */ + if(v==='graph')graphEngineResume();else graphEnginePause(); closeMobileNav(); (LOADERS[v]||function(){})(); heading.tabIndex=-1; @@ -129,7 +132,7 @@ async function loadOverviewAnalytics(){ const el=document.getElementById('ov-analytics'),lock=document.getElementById('ov-lock'); try{ const a=await api('/analytics?workspace='+encodeURIComponent(WS||'')); - lock.textContent=(LIC&&LIC.is_trial)?'TRIAL':''; + lock.textContent=licTrialActive()?'TRIAL':''; lock.className='pill pill-muted'; const t=a.totals||{},f=a.decay_forecast||{}; if(t.live==null){ @@ -151,19 +154,62 @@ async function loadOverviewAnalytics(){ }else if(hostedFeatureUnavailable(e)){ lock.textContent='PRO'; lock.className='pill pill-muted'; - const used=LIC&&LIC.trial&&LIC.trial.used; - el.innerHTML='
Hosted growth, retention distribution, and decay forecast.
'+(used?'':' ')+'
'; + const offerTrial=licTrialAvailable(); + el.innerHTML='
Hosted growth, retention distribution, and decay forecast.
'+esc(lockReason(false))+'
'+(offerTrial?' ':'')+'
'; }else el.innerHTML='
'+esc(e.message)+'
'; } } +/* ── hosted access state ── + /api/license reports what the control plane said, never what this client guessed: + access_state is one of active | trial | trial_expired | lapsed | inactive, and each one + is a different thing to tell the customer. Reading it here is what stopped a customer + whose trial had ended, and a customer whose card had failed, both being shown a + confident PRO badge over rows of locks with no reason given. */ +function licAccessState(){const s=LIC&&LIC.access_state;return s==='active'||s==='trial'||s==='trial_expired'||s==='lapsed'?s:'inactive'} +function licAccessLive(){const s=licAccessState();return s==='active'||s==='trial'} +function licTrialActive(){return licAccessState()==='trial'} +/* The server refuses a second trial for any organization that already holds an + entitlement, so this is false for every connected customer — trialling, lapsed, or + paying — and the CTA that could only ever return 409 is not drawn. */ +function licTrialAvailable(){return !!(LIC&&LIC.trial&&LIC.trial.available)} +function licPlanName(){return licPlanKey()?licPlanKey().toUpperCase():'LOCAL'} +/* The plan the customer actually holds, as the wire spells it: '' when they hold none. + Renewal and billing actions must follow this, not a hardcoded default. */ +function licPlanKey(){const raw=String((LIC&&LIC.plan)||'local').toLowerCase();return raw==='pro'||raw==='team'?raw:''} +function licTrialEnds(){return fmtDay(LIC&&LIC.trial&&LIC.trial.ends_at)} +function fmtDay(epoch){const n=Number(epoch)||0;if(!(n>0))return '';try{const d=new Date(n*1000);return Number.isFinite(d.getTime())?d.toISOString().slice(0,10):''}catch(e){return ''}} + /* ── shared hosted upgrade / trial CTA ── */ +/* The plan-neutral hosted entry point. An account portal is not a checkout, so it must + not carry the ?plan= parameter that would reframe it as one. */ +function hostedAccountUrl(){return safeUrl((LIC&&LIC.account_url)||(LIC&&LIC.upgrade_url))} function hostedPlanUrl(plan,trial){const raw=(LIC&&(plan==='team'?LIC.team_upgrade_url:LIC.pro_upgrade_url))||(LIC&&LIC.upgrade_url);const safe=safeUrl(raw);if(!safe||safe==='#')return '#';try{const url=new URL(safe,location.href);if(plan==='pro'||plan==='team')url.searchParams.set('plan',plan);if(trial)url.searchParams.set('trial',plan);return url.href}catch(e){return safe}} -function unlockHtml(feature,plan){const url=hostedPlanUrl(plan),trialUrl=hostedPlanUrl(plan,true),team=plan==='team';const used=LIC&&LIC.trial&&LIC.trial.used;const trial=team?'Start hosted Team trial':'Start hosted Pro trial';const purchase=team?'Purchase Team license':'Purchase Pro license';const price=team?'$20 per seat/month or $200 per seat/year':'$10/month or $100/year';const detail=used?'Your free trial has already been used.':`The email-confirmed, no-card trial lasts exactly ${TRIAL_DAYS} active days.`;const benefits=team?['Everything in Pro','Hosted organizations, invitations, and named seats','Roles, scoped credentials, and Team audit history']:['Hosted Cloud Sync across your installations','Growth, retention, decay, and entity Analytics','Auto Consolidation with hosted retention policies','Auto Dreaming with reviewable managed proposals','Priority support'];return `
ENGRAPHIS ${team?'TEAM':'PRO'}

Unlock ${esc(feature)} and more

Make the local memory engine work across your installations—and keep improving without manual upkeep.

${price}
Your license unlocks
    ${benefits.map(item=>`
  • ${esc(item)}
    • `).join('')}

${detail}

${used?'':`${trial}`}${purchase}
`} +/* Why is this feature locked? This helper returns plain text; every HTML sink escapes it. + One sentence per access state keeps the panel from claiming a trial the customer cannot + start or blaming billing for a trial that simply ran out. */ +function lockReason(team){const st=licAccessState(),ends=licTrialEnds(),plan=licPlanName(); + if(team&&plan==='TEAM'&&(st==='active'||st==='trial'))return `Team administration runs in Engraphis Team Cloud, not this local dashboard.${st==='trial'&&ends?` Your Team trial is live until ${ends}.`:''}`; + if(st==='trial')return `Your free trial is live${ends?` until ${ends}`:''}. ${team?'Team':'Pro'} needs a subscription of its own.`; + if(st==='trial_expired')return `Your free trial has ended${ends?` (${ends})`:''}, so hosted features are locked. The trial cannot be started again.`; + if(st==='lapsed')return `Your ${plan} subscription is no longer active, so hosted features are locked until billing is up to date.`; + if(st==='active')return `Your ${plan} subscription does not include this.`; + if(licTrialAvailable())return `The email-confirmed, no-card trial lasts exactly ${TRIAL_DAYS} active days.`; + return 'Your free trial has already been used.'} +/* The Team tab describes the hosted service; it is not an answer to a refused request. + A live Team customer therefore gets affirmative plan copy while every other state keeps + the denial explanation that applies to it. */ +function teamTeaserNote(){const ends=licTrialEnds(); + if(licPlanKey()!=='team'||!licAccessLive())return lockReason(true); + if(licAccessState()==='trial')return `Your free trial includes Team${ends?` until ${ends}`:''}. Organizations, roles, and seats are managed in Engraphis Cloud.`; + return 'Your TEAM subscription includes this. Organizations, roles, and seats are managed in Engraphis Cloud.'} +function unlockHtml(feature,plan){const url=hostedPlanUrl(plan),trialUrl=hostedPlanUrl(plan,true),team=plan==='team';const offerTrial=licTrialAvailable();const trial=team?'Start hosted Team trial':'Start hosted Pro trial';const purchase=team?'Purchase Team license':'Purchase Pro license';const price=team?'$20 per seat/month or $200 per seat/year':'$10/month or $100/year';const detail=lockReason(team);const benefits=team?['Everything in Pro','Hosted organizations, invitations, and named seats','Roles, scoped credentials, and Team audit history']:['Hosted Cloud Sync across your installations','Growth, retention, decay, and entity Analytics','Auto Consolidation with hosted retention policies','Auto Dreaming with reviewable managed proposals','Priority support'];return `
ENGRAPHIS ${team?'TEAM':'PRO'}

Unlock ${esc(feature)} and more

Make the local memory engine work across your installations—and keep improving without manual upkeep.

${price}
Your license unlocks
    ${benefits.map(item=>`
  • ${esc(item)}
    • `).join('')}

${esc(detail)}

${offerTrial?`${trial}`:''}${purchase}
`} function startTrialPlan(plan){const url=hostedPlanUrl(plan,true);if(url==='#'){toast('Hosted signup URL is not configured','err');return}const link=document.createElement('a');link.href=url;link.target='_blank';link.rel='noopener';link.click()} function startTrial(){return startTrialPlan('pro')} function startTeamTrial(){return startTrialPlan('team')} -function updateLicBadge(){const bd=document.getElementById('lic-badge');if(!bd||!LIC)return;const raw=String(LIC.plan||'local').toLowerCase(),hosted=!!LIC.is_trial||raw==='pro'||raw==='team';bd.textContent=LIC.is_trial?'TRIAL':(hosted?raw.toUpperCase():'LOCAL');bd.className='pill '+(hosted?'pill-accent':'pill-muted')} +/* The badge follows the access state, not the plan name. A plan name alone told a trialist + they were a subscriber, and told a lapsed or expired customer nothing was wrong. */ +function updateLicBadge(){const bd=document.getElementById('lic-badge');if(!bd||!LIC)return;const st=licAccessState(),plan=licPlanName();bd.textContent=st==='trial'?'TRIAL':st==='trial_expired'?'TRIAL ENDED':st==='lapsed'?plan+' INACTIVE':st==='active'?plan:'LOCAL';bd.className='pill '+(licAccessLive()?'pill-accent':'pill-muted')} function updateFeatureLocks(){ const has=f=>LIC&&(LIC.features||[]).includes(f); const apply=(id,feature,label,plan)=>{ @@ -202,10 +248,10 @@ function managedConsentRequired(error){return error&&error.status===409&&error.d the plain-error branch unreachable on the analytics and automation views, which is the regression this comment previously described but the code did not implement. */ function hostedFeatureUnavailable(error){return !!error&&(error.status===402||error.status===501)} -async function loadAnalytics(){const el=document.getElementById('analytics-body'),lock=document.getElementById('an-lock'),acts=document.getElementById('an-actions');el.innerHTML='';try{const a=await api('/analytics?workspace='+encodeURIComponent(WS||''));setPlanPill(lock,(LIC&&LIC.is_trial)?'TRIAL':'CLOUD','pill pill-accent');showAs(acts,true,'flex');el.innerHTML=renderAnalytics(a,false)}catch(e){if(managedConsentRequired(e)){setPlanPill(lock,'CLOUD','pill pill-accent');showAs(acts,false);el.innerHTML=managedConsentHtml('Analytics')}else if(hostedFeatureUnavailable(e)){setPlanPill(lock,'PRO','pill pill-muted');showAs(acts,false);el.innerHTML=unlockHtml('Analytics','pro')}else{el.innerHTML='
'+esc(e.message)+'
'}}} +async function loadAnalytics(){const el=document.getElementById('analytics-body'),lock=document.getElementById('an-lock'),acts=document.getElementById('an-actions');el.innerHTML='';try{const a=await api('/analytics?workspace='+encodeURIComponent(WS||''));setPlanPill(lock,licTrialActive()?'TRIAL':'CLOUD','pill pill-accent');showAs(acts,true,'flex');el.innerHTML=renderAnalytics(a,false)}catch(e){if(managedConsentRequired(e)){setPlanPill(lock,'CLOUD','pill pill-accent');showAs(acts,false);el.innerHTML=managedConsentHtml('Analytics')}else if(hostedFeatureUnavailable(e)){setPlanPill(lock,'PRO','pill pill-muted');showAs(acts,false);el.innerHTML=unlockHtml('Analytics','pro')}else{el.innerHTML='
'+esc(e.message)+'
'}}} /* ── hosted automation policy (Pro / Team) ── */ -async function loadAutomation(){const el=document.getElementById('automation-body'),lock=document.getElementById('au-lock'),ws='?workspace='+encodeURIComponent(WS||'');el.innerHTML='';try{const p=await api('/automation'+ws);setPlanPill(lock,(LIC&&LIC.is_trial)?'TRIAL':'CLOUD','pill pill-accent');const last=p.last_run?fmtRel(p.last_run):'never',dream=p.dream_enabled!=null?p.dream_enabled:p.dream;el.innerHTML=`
Hosted maintenance policy
The cloud returns reviewable proposals. Pinned memories remain protected.
Cloud worker status
Status${p.enabled?'ENABLED':'OFF'}
Last run${esc(last)}
Requesting managed work uploads the selected workspace’s normal and sensitive memory content, excluding secret and session-scoped rows, capped at 16 MiB, over HTTPS without end-to-end encryption. Results are proposals and never automatically write the local database.
`}catch(e){if(managedConsentRequired(e)){setPlanPill(lock,'CLOUD','pill pill-accent');el.innerHTML=managedConsentHtml('Hosted Automation')}else if(hostedFeatureUnavailable(e)){setPlanPill(lock,'PRO','pill pill-muted');el.innerHTML=unlockHtml('Automation, Auto Consolidation, and Auto Dreaming','pro')}else{el.innerHTML='
'+esc(e.message)+'
'}}} +async function loadAutomation(){const el=document.getElementById('automation-body'),lock=document.getElementById('au-lock'),ws='?workspace='+encodeURIComponent(WS||'');el.innerHTML='';try{const p=await api('/automation'+ws);setPlanPill(lock,licTrialActive()?'TRIAL':'CLOUD','pill pill-accent');const last=p.last_run?fmtRel(p.last_run):'never',dream=p.dream_enabled!=null?p.dream_enabled:p.dream;el.innerHTML=`
Hosted maintenance policy
The cloud returns reviewable proposals. Pinned memories remain protected.
Cloud worker status
Status${p.enabled?'ENABLED':'OFF'}
Last run${esc(last)}
Requesting managed work uploads the selected workspace’s normal and sensitive memory content, excluding secret and session-scoped rows, capped at 16 MiB, over HTTPS without end-to-end encryption. Results are proposals and never automatically write the local database.
`}catch(e){if(managedConsentRequired(e)){setPlanPill(lock,'CLOUD','pill pill-accent');el.innerHTML=managedConsentHtml('Hosted Automation')}else if(hostedFeatureUnavailable(e)){setPlanPill(lock,'PRO','pill pill-muted');el.innerHTML=unlockHtml('Automation, Auto Consolidation, and Auto Dreaming','pro')}else{el.innerHTML='
'+esc(e.message)+'
'}}} async function saveAutomation(){const body={enabled:document.getElementById('au-enabled').checked,cadence_hours:Number(document.getElementById('au-cadence').value)||24,consolidate:document.getElementById('au-consolidate').checked,min_cluster:Number(document.getElementById('au-mincluster').value)||3,archive_below:Number(document.getElementById('au-archive').value)||0.05,dream_enabled:document.getElementById('au-dream').checked,dream_min_new:Number(document.getElementById('au-dream-min').value)||20,dream_idle_minutes:Number(document.getElementById('au-dream-idle').value)};try{await api('/automation?workspace='+encodeURIComponent(WS||''),{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(body)});toast('Hosted policy saved','ok');loadAutomation()}catch(e){if(managedConsentRequired(e)){const result=document.getElementById('au-result');if(result)result.innerHTML=managedConsentHtml('Hosted Automation');toast('Connect this installation to Engraphis Cloud to use managed compute','err');return}toast((e.status===402||e.status===501)?'Hosted Automation requires Pro or Team':e.message,'err')}} async function runMaintenance(){const el=document.getElementById('au-result');if(el)el.innerHTML='';try{const d=await api('/maintenance/run?workspace='+encodeURIComponent(WS||''),{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({dry_run:true})});if(el)el.innerHTML=`PROPOSAL Hosted work was submitted for review.
${esc(JSON.stringify(d,null,2))}
`;toast('Managed proposal requested','ok')}catch(e){if(el)el.innerHTML=managedConsentRequired(e)?managedConsentHtml('Hosted Automation'):'
'+esc(e.message)+'
';toast(managedConsentRequired(e)?'Connect this installation to Engraphis Cloud to use managed compute':((e.status===402||e.status===501)?'Hosted Automation requires Pro or Team':e.message),'err')}} @@ -428,29 +474,52 @@ window.addEventListener('beforeunload',e=>{if(!editorIsDirty())return;e.preventD document.addEventListener('keydown',e=>{if((e.ctrlKey||e.metaKey)&&e.key.toLowerCase()==='s'&&document.getElementById('view-mem-editor').classList.contains('active')){e.preventDefault();edSave()}}); /* license */ async function loadLicense(){const el=document.getElementById('lic-body');try{const d=await api('/license');LIC=d;updateLicBadge();updateFeatureLocks();renderLicense(d)}catch(e){if(el)el.innerHTML='
'+esc(e.message)+'
'}} +/* Copy for the control plane's own entitlement status, when it named one. This is what + separates "your card was declined" from "you cancelled" inside a single lapsed state. */ +const LIC_STATUS_NOTE={past_due:'the last payment did not go through',canceled:'the subscription was cancelled',expired:'the billing period ended',revoked:'access was revoked'}; +const LIC_SOURCE_LABEL={environment:'operator override',session:'cloud handshake',cloud:'cloud entitlements read',connected:'not yet confirmed',local:'not connected',override:'operator override'}; +/* The whole point of the panel: when hosted features are locked, say why, and offer the + one action that fixes it. Never a plan badge over unexplained locks. */ +function licStateBanner(state,plan,ends,status){ + if(state==='trial_expired')return `
Your free trial has ended${ends?' on '+esc(ends):''}Hosted features are locked. Everything you have written is still in your local database and still fully usable — only the cloud capabilities stopped. The free trial runs once per account and cannot be started again, so restoring them means subscribing.
`; + if(state==='lapsed'){const note=LIC_STATUS_NOTE[status];return `
Your ${esc(plan||'hosted')} subscription is no longer active${note?esc(note.charAt(0).toUpperCase()+note.slice(1))+', so hosted':'Hosted'} features are locked until billing is up to date. Your local memories are unaffected. Open the account portal to restore access.
`} + if(state==='inactive')return `
No hosted plan on this installationThe local memory engine is free and complete on its own. Cloud Sync, Analytics, Automation, and Team administration run in Engraphis Cloud.
`; + return ''} +function licActionsHtml(state){ + if(licTrialAvailable())return `
`; + /* A lapsed customer is renewing the subscription they already hold, not shopping. */ + if(state==='lapsed')return `
Update billingOpen account portal
`; + const buy=state==='trial_expired'; + const primary=buy?'Subscribe to Pro':'Open Pro Cloud'; + const secondary=buy?'Subscribe to Team':'Open Team Cloud'; + return `
${primary}${secondary}
`} function renderLicense(d){ const el=document.getElementById('lic-body');if(!el)return; - const raw=String(d.plan||'local').toLowerCase(),trial=!!d.is_trial; - const hosted=trial||raw==='pro'||raw==='team'; - const label=trial?(raw==='team'?'TEAM TRIAL':'PRO TRIAL'):(hosted?raw.toUpperCase():'LOCAL CORE'); + const state=licAccessState(),raw=String(d.plan||'local').toLowerCase(); + const plan=raw==='pro'||raw==='team'?raw.toUpperCase():''; + const hosted=state!=='inactive',live=licAccessLive(),ends=licTrialEnds(); + const label=state==='trial'?(plan||'PRO')+' TRIAL':state==='trial_expired'?(plan||'PRO')+' TRIAL ENDED':state==='lapsed'?plan+' INACTIVE':state==='active'?plan:'LOCAL CORE'; const known=d.known_features||{}; const feats=hosted?Object.keys(known).map(f=>`${(d.features||[]).includes(f)?'✓':'○'} ${esc(known[f])}`).join(''):''; - const used=!!(d.trial&&d.trial.used); - let h=`
${hosted?'Hosted plan':'Local runtime'}${esc(label)}
`; + let h=`
${hosted?'Hosted plan':'Local runtime'}${esc(label)}
`; if(d.error)h+=`
Hosted authorization unavailable — ${esc(d.error)}
`; + h+=licStateBanner(state,plan,ends,d.entitlement_status); if(hosted&&d.email&&d.email!=='trial')h+=`
Hosted account${esc(d.email)}
`; - if(hosted&&d.expires)h+=`
${trial?'Trial ends':'Authorization expires'}${new Date(d.expires*1000).toISOString().slice(0,10)}
`; + if(state==='trial'&&ends)h+=`
Trial ends${esc(ends)}
`; + else if(hosted&&d.expires)h+=`
Authorization expires${esc(fmtDay(d.expires))}
`; if(feats)h+=`
${feats}
`; + /* Support diagnostics: which rule produced this answer and when the cloud last confirmed + it. Emitted by /api/license since the plan resolver landed, and never shown until now — + so "the dashboard says PRO" and "the cloud says PRO" could not be told apart. */ + if(d.plan_source)h+=`
Plan source${esc(LIC_SOURCE_LABEL[d.plan_source]||d.plan_source)}${d.plan_checked_at?' · confirmed '+esc(fmtRel(d.plan_checked_at)):''}
`; h+=`
The local core remains free. Pro and Team capabilities execute in Engraphis Cloud. The email-confirmed, no-card trial lasts exactly ${TRIAL_DAYS} active days; local-only write grace is separate, capped at 24 hours, and never extends cloud access.
`; - h+=hosted||used - ?`
Open Pro CloudOpen Team Cloud
` - :`
`; + h+=licActionsHtml(state); el.innerHTML=h; } async function exportWorkspace(){try{const d=await api('/export?workspace='+encodeURIComponent(WS||''));const blob=new Blob([JSON.stringify(d,null,2)],{type:'application/json'});const a=document.createElement('a');a.href=URL.createObjectURL(blob);a.download='engraphis-export-'+Date.now()+'.json';a.click();URL.revokeObjectURL(a.href);toast('Exported','ok')}catch(e){toast(e.message,'err')}} /* Hosted Team is a service CTA; local identity and seat administration are not shipped. */ -async function loadTeam(){const el=document.getElementById('team-body');let url=hostedPlanUrl('team');try{const st=await api('/auth/state');if(url==='#'&&st&&st.cloud_url)url=safeUrl(st.cloud_url)}catch(e){}let trialUrl=url;if(url!=='#')try{const parsed=new URL(url,location.href);parsed.searchParams.set('trial','team');trialUrl=parsed.href}catch(e){}el.innerHTML=`
Engraphis Team Cloud HOSTED
Organizations, invitations, roles, named seats, scoped device credentials, and team audit run on the private hosted service. This local dashboard is intentionally single-user.
The email-confirmed trial lasts exactly ${TRIAL_DAYS} active days. A separate local-only write grace is capped at 24 hours and never extends Team or other cloud access.
`} +async function loadTeam(){const el=document.getElementById('team-body');let url=hostedPlanUrl('team');try{const st=await api('/auth/state');if(url==='#'&&st&&st.cloud_url)url=safeUrl(st.cloud_url)}catch(e){}let trialUrl=url;if(url!=='#')try{const parsed=new URL(url,location.href);parsed.searchParams.set('trial','team');trialUrl=parsed.href}catch(e){}el.innerHTML=`
Engraphis Team Cloud HOSTED
Organizations, invitations, roles, named seats, scoped device credentials, and team audit run on the private hosted service. This local dashboard is intentionally single-user.
${esc(teamTeaserNote())} A separate local-only write grace is capped at 24 hours and never extends Team or other cloud access.
${licTrialAvailable()?`Start hosted Team trial`:''}Open Team Cloud
`} /* health + settings */ function connectionContext(){const host=(location.hostname||'').toLowerCase();return host==='localhost'||host==='127.0.0.1'||host==='::1'||host.endsWith('.localhost')?'Local engine':'Remote customer node'} async function checkHealth(){const label=connectionContext();try{await api('/health');const d=document.getElementById('health-dot'),t=document.getElementById('health-text');if(d){d.classList.add('health-ok');d.classList.remove('health-error')}if(t)t.textContent=label+' connected'}catch(e){const d=document.getElementById('health-dot'),t=document.getElementById('health-text');if(d){d.classList.add('health-error');d.classList.remove('health-ok')}if(t)t.textContent=label+' unavailable'}} @@ -474,7 +543,7 @@ function renderSync(d){const el=document.getElementById('sync-body');if(!el)retu async function syncNow(){const b=document.getElementById('sync-btn')||document.getElementById('sync-retry-btn');const original=b&&b.textContent;const s=document.getElementById('sync-status');if(b){b.disabled=true;b.textContent='Syncing…'}if(s)s.textContent='Contacting the cloud…';try{const d=await api('/sync/run',{method:'POST',headers:{'Content-Type':'application/json'},body:'{}'});const su=d.summary||{};toast('Synced — pushed '+(su.exported||0)+', '+(su.added||0)+' new from other devices','ok');await loadSyncStatus()}catch(e){if(e.status===401||e.status===402||e.status===403){const el=document.getElementById('sync-body');if(el)el.innerHTML=syncRecoveryHtml();toast(e.status===402?'Cloud Sync requires an active Pro or Team entitlement — open Engraphis Cloud to upgrade or renew.':'Cloud Sync authorization is no longer active — reconnect in Engraphis Cloud.','err');return}toast('Sync failed: '+e.message,'err');if(b){b.disabled=false;b.textContent=original||'Sync now'}if(s)s.textContent='Sync failed — try again.'}} /* ─── knowledge graph (force-graph + d3-force: compact defaults and selectable layouts) ─── */ -let GRAPH=null, FG=null, GRESIZE=false, GRESIZEFRAME=0, GADJ={}, GCOMM_ADJ={}, GCOMPONENTS={}, GCOMPONENT_LAYOUT=null, GHILITE=null, GHOVERSET=null, GLABELRANK={}, GLABELBOXES=[], GDATA_CACHE=null, GACTIVE_DATA=null, GREDRAWFRAME=0, GPERF={large:false,dense:false}; +let GRAPH=null, FG=null, GRAPH_ENGINE=null, GRESIZE=false, GRESIZEFRAME=0, GADJ={}, GCOMM_ADJ={}, GCOMPONENTS={}, GCOMPONENT_LAYOUT=null, GHILITE=null, GHOVERSET=null, GLABELRANK={}, GLABELBOXES=[], GDATA_CACHE=null, GACTIVE_DATA=null, GREDRAWFRAME=0, GPERF={large:false,dense:false}; const GRAPH_PRESETS={ original:{label:'Original force',repel:120,link:30,gravity:14,font:13,size:3,linkw:1,labelDensity:40,curve:0,particles:0}, compact:{label:'Compact clusters',repel:42,link:20,gravity:26,font:12,size:3,linkw:.7,labelDensity:30,curve:.08,particles:0}, @@ -507,6 +576,10 @@ function graphLoadColorPreferences(){ } function graphSaveColorPreferences(){try{localStorage.setItem(GRAPH_COLOR_KEY,JSON.stringify({palette:GCOLOR_PALETTE,colors:GCOLOR_OVERRIDES}))}catch(e){}} function graphTypeColor(type){if(GCOLOR_OVERRIDES[type])return GCOLOR_OVERRIDES[type];if(typeof GSTYLE!=='undefined'&&GSTYLE&&GSTYLE!=='classic'&&STYLE_PAL[GSTYLE]&&STYLE_PAL[GSTYLE][type])return STYLE_PAL[GSTYLE][type];return cssvar(ETYPE_TOKEN[type]||'--entity-concept',cssvar('--color-accent','#8c83e8'))} +/* The engine renders to a canvas, so it cannot read `--entity-*` itself the way the legend and + the controls do. Resolve the active theme's values here and hand them over; without this the + opt-in canvas keeps dark-theme node colours after a switch to Light/Solarized/Sepia. */ +function graphThemeTypeColors(){const colors={},fallback=cssvar('--color-accent','#8c83e8');Object.keys(ETYPE_TOKEN).forEach(type=>{colors[type]=cssvar(ETYPE_TOKEN[type],fallback)});colors.relation_label=cssvar('--color-text-dim','#7e8795');colors.label=cssvar('--color-text','#e7e9ee');return colors} function graphContrastColor(color){if(!graphValidColor(color))return cssvar('--color-canvas','#0e1014');const n=parseInt(color.slice(1),16),lum=.2126*(n>>16)+.7152*((n>>8)&255)+.0722*(n&255);return lum>150?'#111827':'#f8fafc'} const ETYPE_COLOR=new Proxy({},{get:(_,type)=>graphTypeColor(type)}); graphLoadColorPreferences(); @@ -528,6 +601,7 @@ function graphSetTypeColor(type,color,persist){ if(!type||!graphValidColor(color))return; GCOLOR_OVERRIDES[type]=color.toLowerCase();GCOLOR_PALETTE='custom'; const picker=document.getElementById('graph-palette');if(picker)picker.value='custom'; + if(GRAPH_ENGINE){GRAPH_ENGINE.setTypeColor(type,color);if(persist)graphSaveColorPreferences();return} graphRefreshNodeColors(); if(persist)graphSaveColorPreferences(); } @@ -553,7 +627,102 @@ function graphUpdateHud(data){ if(count&&data)count.textContent=data.nodes.length.toLocaleString()+' entities · '+data.links.length.toLocaleString()+' relations'; if(badge)badge.textContent=GPERF.large?'Large graph mode':'Adaptive rendering'; } -function graphInvalidateData(){GDATA_CACHE=null;GACTIVE_DATA=null;GCOMPONENT_LAYOUT=null;GHILITE=null;GHOVERSET=null} +/* ── opt-in next-generation renderer (`?graph-engine=next`) ────────────────────────────── + The classic renderer stays the default and the rollback path. Everything below is written + so that any failure in the opt-in engine degrades to classic rather than taking the graph + view down: one throw sets GRAPH_ENGINE_FAILED and the flag is never honoured again for the + life of the page. */ +let GRAPH_ENGINE_FAILED=false; +function graphEngineEnabled(){ + if(GRAPH_ENGINE_FAILED)return false; + try{return new URLSearchParams(window.location.search).get('graph-engine')==='next'}catch(e){return false} +} +function graphEngineFallback(error){ + GRAPH_ENGINE_FAILED=true; + try{if(GRAPH_ENGINE)GRAPH_ENGINE.destroy()}catch(e){} + GRAPH_ENGINE=null; + /* The classic renderer skips seeding when GACTIVE_DATA still points at the current data, + so a failure *after* a successful engine render would hand it an empty canvas. Clearing + the marker makes the very next graphRender() a full classic build. */ + GACTIVE_DATA=null;GCOMPONENT_LAYOUT=null;GHILITE=null;GHOVERSET=null; + if(window.console&&console.warn)console.warn('graph-engine=next failed; falling back to the classic renderer',error); +} +function graphEngineEmptyMessage(){ + const total=(GRAPH&&GRAPH.nodes&&GRAPH.nodes.length)||0; + return total?('No connected entities — tick "Show unlinked" to see all '+total+'.'):'No entities in this workspace yet.'; +} +function graphRenderEngine(data,fit,reheat){ + const element=document.getElementById('graph-net'),empty=document.getElementById('graph-empty'); + if(!element||typeof EngraphisGraph==='undefined')return false; + try{ + if(!data.nodes.length){ + if(GRAPH_ENGINE)GRAPH_ENGINE.setData({nodes:[],links:[]}); + showAs(empty,true,'flex'); + if(empty)empty.textContent=graphEngineEmptyMessage(); + GACTIVE_DATA=null;graphSetLayoutStatus('No entities',false);return true; + } + showAs(empty,false);GPERF={large:data.nodes.length>600||data.links.length>2400,dense:data.links.length>1500}; + const created=!GRAPH_ENGINE; + if(created){ + GRAPH_ENGINE=EngraphisGraph.create(element,{ + reducedMotion:prefersReducedMotion, + onNodeClick:node=>{syncGraphExplorerSelection(node.id);graphNodeClick(node.label||node.name||node.id)}, + onBackgroundClick:()=>graphSetHighlight(null), + onStats:stats=>{const count=document.getElementById('graph-hud-count');if(count)count.textContent=stats.nodes.toLocaleString()+' entities · '+stats.links.toLocaleString()+' relations'} + }); + } + /* Re-seeding identical data would re-copy every node and throw its x/y away, restarting + the layout on each slider or preset change. The classic path guards the same way. */ + const dataChanged=created||GACTIVE_DATA!==data; + const layers={};document.querySelectorAll('#graph-layer-filters input').forEach(input=>{layers[input.value]=input.checked}); + /* "Show unlinked nodes" is applied twice: graphData() decides what is handed over, and the + engine re-filters by degree on its own state. Leaving the engine on its defaults + (showUnlinked:false, minDegree:1) drops every degree-zero entity graphData() just supplied, + so the checkbox appeared to do nothing under ?graph-engine=next. */ + const isolated=document.getElementById('graph-show-iso'),showUnlinked=!!(isolated&&isolated.checked); + GRAPH_ENGINE.apply(engine=>{ + engine.setSettings({...window.GSET}); + engine.setStyle(typeof GSTYLE!=='undefined'?GSTYLE:'cyber'); + engine.setColorBy(typeof GCOLORBY!=='undefined'?GCOLORBY:'community'); + engine.setThemeColors(graphThemeTypeColors()); + engine.setPalette(typeof GCOLOR_PALETTE!=='undefined'?GCOLOR_PALETTE:'theme'); + engine.setTypeColors(GCOLOR_OVERRIDES||{}); + engine.setLayers(layers); + engine.setScope({showUnlinked,minDegree:showUnlinked?0:1}); + if(dataChanged)engine.setData(data); + },fit,reheat&&!prefersReducedMotion()); + /* Mirror the engine's clustering back onto the dashboard's own node objects, or the + cluster legend (which reads GACTIVE_DATA) reports one community for the whole store. */ + const communityMap=GRAPH_ENGINE.communityMap(); + data.nodes.forEach(node=>{node.community=communityMap[node.id]||0}); + GACTIVE_DATA=data;graphSyncReadouts();graphUpdateEditedBadge();graphUpdateHud(data);graphRenderLegend(GRAPH); + if(dataChanged)graphSetHighlight(null); + if(window.GSET.frozen)GRAPH_ENGINE.freeze(true); + /* The renderer can be born after the user has already left the view: /graph and both lazy + scripts resolve asynchronously, and the pause on nav-away ran while GRAPH_ENGINE was still + null. Re-apply the parked state here so a renderer created against a hidden pane never + starts a rAF that nothing will stop. */ + if(GRAPH_ENGINE_PARKED)GRAPH_ENGINE.pause(); + graphSetSimulationStatus(prefersReducedMotion()?'Static layout':'Adaptive layout',false); + return true; + }catch(error){ + graphEngineFallback(error); + return false; + } +} +/* Nav away from the graph view: park the engine's animation frame. Without this the opt-in + renderer keeps repainting a hidden canvas for the rest of the session. + The intent is *recorded* as well as applied, because pausing an engine that does not exist + yet is a no-op: leaving Graph before /graph (or either lazy script) resolves would otherwise + let the pending callback create and start a renderer against a hidden pane with no later + pause to stop it. graphRenderEngine() re-applies GRAPH_ENGINE_PARKED for that case. */ +let GRAPH_ENGINE_PARKED=false; +function graphEnginePause(){GRAPH_ENGINE_PARKED=true;try{if(GRAPH_ENGINE)GRAPH_ENGINE.pause()}catch(e){}} +function graphEngineResume(){GRAPH_ENGINE_PARKED=false;try{if(GRAPH_ENGINE)GRAPH_ENGINE.resume()}catch(e){}} +function graphInvalidateData(){ + if(GRAPH_ENGINE){try{GRAPH_ENGINE.destroy()}catch(e){}GRAPH_ENGINE=null} + GDATA_CACHE=null;GACTIVE_DATA=null;GCOMPONENT_LAYOUT=null;GHILITE=null;GHOVERSET=null +} async function loadLegacyGraph(){ graphInjectCss();graphInvalidateData();GRAPH=null; const empty=document.getElementById('graph-empty'),net=document.getElementById('graph-net'),nodesBox=document.getElementById('graph-entity-list'),edgesBox=document.getElementById('graph-relation-list'); @@ -563,8 +732,8 @@ async function loadLegacyGraph(){ if(!GRESIZE){ GRESIZE=true; window.addEventListener('resize',()=>{ - if(!FG||GRESIZEFRAME)return; - GRESIZEFRAME=requestAnimationFrame(()=>{GRESIZEFRAME=0;const element=document.getElementById('graph-net');if(FG&&element)FG.width(element.clientWidth).height(element.clientHeight)}); + if((!FG&&!GRAPH_ENGINE)||GRESIZEFRAME)return; + GRESIZEFRAME=requestAnimationFrame(()=>{GRESIZEFRAME=0;const element=document.getElementById('graph-net');if(GRAPH_ENGINE)GRAPH_ENGINE.resize();else if(FG&&element)FG.width(element.clientWidth).height(element.clientHeight)}); }); } const layerInputs=Array.from(document.querySelectorAll('#graph-layer-filters input')),selectedLayers=layerInputs.filter(input=>input.checked).map(input=>input.value),layerFilter=selectedLayers.length===layerInputs.length?'':'&layers='+encodeURIComponent(selectedLayers.join(',')),includeCode=document.getElementById('graph-include-code').checked,repo=(document.getElementById('graph-repo-filter').value||'').trim(); @@ -711,6 +880,7 @@ function graphSetStyle(name){ if(['classic','galaxy','solar','cyber'].indexOf(name)<0)name='cyber'; GSTYLE=name;try{localStorage.setItem('engraphis-graph-style',name)}catch(e){} graphApplyStyleChrome(); + if(GRAPH_ENGINE){GRAPH_ENGINE.setStyle(name);return} if(GRAPH&&FG){graphRefreshNodeColors();graphRenderLegend();graphRender(false,false);} } /* ─── colorful graphs even when every node is one entity type: color by community or connections ─── */ @@ -766,6 +936,7 @@ function graphSetColorBy(mode){ if(['type','community','connections'].indexOf(mode)<0)mode='community'; GCOLORBY=mode;try{localStorage.setItem('engraphis-graph-colorby',mode)}catch(e){} var sel=document.getElementById('graph-colorby');if(sel&&sel.value!==mode)sel.value=mode; + if(GRAPH_ENGINE){GRAPH_ENGINE.setColorBy(mode);graphRenderLegend();return} if(GRAPH&&FG&&GACTIVE_DATA){graphComputeCommunities(GACTIVE_DATA.nodes);GMAXDEG=GACTIVE_DATA.nodes.reduce(function(m,n){return Math.max(m,n.degree||0);},1);graphRefreshNodeColors();graphRenderLegend();} } function graphApplyForces(){ @@ -790,6 +961,9 @@ function graphApplyForces(){ function graphSetHighlight(id){ GHILITE=id||null; GHOVERSET=id?new Set([id,...(GADJ[id]||[])]):null; + /* graphRedraw() is a no-op without the classic FG instance, so the opt-in engine needs + telling directly — otherwise hovering the entity list highlights nothing on canvas. */ + if(GRAPH_ENGINE){try{GRAPH_ENGINE.setHighlight(GHILITE)}catch(e){}return} graphRedraw(); } function graphRefreshNodeMetrics(){ @@ -800,6 +974,12 @@ function graphRedraw(){ if(!FG||GREDRAWFRAME)return; GREDRAWFRAME=requestAnimationFrame(()=>{GREDRAWFRAME=0;if(FG)FG.nodeCanvasObject(FG.nodeCanvasObject())}); } +/* ── on-demand graph assets ─────────────────────────────────────────────────────────────── + Neither script is in index.html. force-graph.min.js applies inline styles at runtime, and + under the production CSP (`style-src 'self'`) every one of those is blocked and reported; + loading it on a page that never opens the graph turns a plain dashboard view into a wall of + console errors. Both loaders are memoized, so a re-entrant graphRender() reuses the in-flight + fetch rather than appending a second + diff --git a/engraphis/stores/graph.py b/engraphis/stores/graph.py index d13c546..f5cc51c 100644 --- a/engraphis/stores/graph.py +++ b/engraphis/stores/graph.py @@ -2,9 +2,12 @@ from __future__ import annotations from typing import Any, Optional +import logging from engraphis.stores import get_conn, now_ts +logger = logging.getLogger("engraphis.stores.graph") + def upsert_entity(namespace: str, name: str, entity_type: Optional[str] = None) -> None: conn = get_conn() @@ -91,8 +94,8 @@ def graph_snapshot(namespace: Optional[str] = None, limit: int = 200, did = payload.get("document_id") if did and did not in doc_ids: doc_ids.append(did) - except Exception: - pass + except Exception as exc: + logger.debug("Entity document payload parse skipped (%s)", type(exc).__name__) ent["documents"] = doc_ids[:10] # Get a preview from the first document if ent["documents"]: diff --git a/pyproject.toml b/pyproject.toml index da56949..47dfb26 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ build-backend = "setuptools.build_meta" [project] name = "engraphis" -version = "1.0.1" +version = "1.1.0" description = "Local-first AI memory engine for agents — Ebbinghaus decay, interaction-aware recall, bi-temporal facts, hybrid retrieval, and an MCP server. You bring the LLM." readme = "README.md" license = "Apache-2.0" @@ -164,6 +164,11 @@ Repository = "https://github.com/Coding-Dev-Tools/engraphis" Issues = "https://github.com/Coding-Dev-Tools/engraphis/issues" [project.scripts] +# The front door. The account portal tells customers to run `engraphis connect --token ...`, +# so the bare verb form has to exist; scripts.entry dispatches to the same main() each +# engraphis- script below calls. +engraphis = "scripts.entry:main" +engraphis-connect = "scripts.connect:main" engraphis-server = "scripts.start_server:main" engraphis-cli = "scripts.cli:main" engraphis-mcp = "engraphis.mcp_cli:main" @@ -186,6 +191,7 @@ include = ["engraphis*", "scripts*"] # runtime artifacts such as static/__pycache__/*.pyc after a local compile check. # vendor/**/* covers nested vendor assets too (vendor/* alone doesn't cross "/"). "engraphis.static" = ["*.html", "*.css", "*.js", "*.png", "*.ico", "vendor/*", "vendor/**/*"] +"engraphis.dashboard_assets" = ["*.js"] "engraphis" = ["commercial_manifest.json"] [tool.setuptools.exclude-package-data] diff --git a/scripts/connect.py b/scripts/connect.py new file mode 100644 index 0000000..6652b0e --- /dev/null +++ b/scripts/connect.py @@ -0,0 +1,167 @@ +"""engraphis connect - redeem the connect token from your account portal. + +Your Engraphis account portal shows a one-time token and the exact command to run: + + engraphis connect --token engr_ct_... + +That exchanges the token with the control plane and writes the owner-only session file +``~/.engraphis/cloud_session.json`` that the dashboard, MCP server, and Cloud Sync all +read. Until this command existed the file had no writer at all, so a paying customer had +no supported way to connect a client. + + engraphis connect --token engr_ct_... # the normal case + engraphis connect --token - # read the token from stdin + engraphis connect --token ... --workspace ws_1 # bind the device to one workspace + engraphis connect --token ... --label "CI runner" + engraphis connect --token ... --json # redacted machine-readable summary + +The token is a credential. It is sent in the request body and nowhere else: it is never +printed, never logged, and never written to disk. Passing it as an argument does put it +in your shell history, so ``--token -`` is available for scripted and shared machines. + +Non-interactive by design (no prompts): safe in scripts, CI, and agent shells. +""" +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +from engraphis import cloud_session +from engraphis.device_connect import ( + DEFAULT_TIMEOUT_SECONDS, + DeviceConnectError, + connect, +) + + +def _prog(argv_zero: str) -> str: + """Name this command the way the caller actually invoked it.""" + + stem = Path(argv_zero or "").stem + return stem if stem.endswith("connect") and stem != "connect" else "engraphis connect" + + +def _read_token(value: str) -> str: + """Resolve ``--token -`` to one line of stdin, keeping it off the command line.""" + + if value != "-": + return value + if sys.stdin is None or sys.stdin.isatty(): + # A bare `--token -` on a terminal would silently hang waiting for input. + raise DeviceConnectError( + "--token - reads the token from stdin; pipe it in, for example " + "`printf %s \"$ENGRAPHIS_CONNECT_TOKEN\" | engraphis connect --token -`.", + status=400, + ) + return sys.stdin.readline() + + +def _print_summary(summary: dict) -> None: + rows = [ + ("organization", summary.get("organization_id")), + ("installation", summary.get("installation_id")), + ("device", summary.get("device_id")), + ("member", summary.get("member_id")), + ("workspace", summary.get("workspace_id")), + ("subject", summary.get("token_subject")), + ] + plan = str(summary.get("plan") or "").strip() + if plan: + active = summary.get("cloud_access_active") + rows.append(("plan", plan + ("" if active is None else + " (active)" if active else " (inactive)"))) + features = summary.get("cloud_features") + if isinstance(features, (list, tuple)) and features: + rows.append(("features", ", ".join(str(item) for item in features))) + rows.append(("control url", summary.get("control_url"))) + rows.append(("compute url", summary.get("compute_url") or "(not configured)")) + rows.append(("session file", summary.get("session_path"))) + + print("Connected this device to Engraphis Cloud.") + print() + for label, value in rows: + text = str(value or "").strip() + if text: + print(" %-14s %s" % (label, text)) + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser( + prog=_prog(sys.argv[0] if sys.argv else ""), + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + ap.add_argument("--token", required=True, metavar="TOKEN", + help="the connect token from your account portal, or - for stdin") + ap.add_argument("--control-url", default=None, metavar="URL", + help="control plane to connect to (default: the shipped endpoint, " + "or ENGRAPHIS_CLOUD_CONTROL_URL)") + ap.add_argument("--compute-url", default=None, metavar="URL", + help="managed compute endpoint (default: ENGRAPHIS_CLOUD_COMPUTE_URL, " + "or the shipped endpoint)") + ap.add_argument("--workspace", default=None, metavar="WORKSPACE_ID", + help="bind this device to a single workspace") + ap.add_argument("--label", default=None, metavar="TEXT", + help="label for this installation in your account portal") + ap.add_argument("--device-name", default=None, metavar="TEXT", + help="device name shown in your account portal (default: hostname)") + ap.add_argument("--timeout", type=float, default=DEFAULT_TIMEOUT_SECONDS, + metavar="SECONDS", help="network timeout (default: %(default)s)") + ap.add_argument("--json", action="store_true", + help="print the redacted summary as JSON instead of a report") + args = ap.parse_args(argv) + + try: + summary = connect( + _read_token(args.token), + control_url=args.control_url, + compute_url=args.compute_url, + workspace_id=args.workspace, + installation_label=args.label, + device_name=args.device_name, + timeout=args.timeout, + ) + except DeviceConnectError as exc: + # ``str(exc)`` is fixed public copy from device_connect; it never carries the + # token, the response body, or an internal hostname. + print("%s: %s" % (ap.prog, exc), file=sys.stderr) + return 1 + + # Prove the write actually produced a session the rest of the client will use, rather + # than reporting success on a file nothing can load. + # + # This runs *before* any success output is emitted, and that ordering is load-bearing. + # ``configured()`` reads the session back and can raise -- an invalid + # ``ENGRAPHIS_CLOUD_TOKEN_SUBJECT``, or the file changing under the read. Printing + # first meant a complete ``--json`` success object was already on stdout when the + # command then wrote an error and exited 1, so a consumer parsing stdout accepted a + # connect that had failed, and a human got two contradictory answers. + try: + usable = cloud_session.configured() + except cloud_session.CloudSessionError as exc: + print("%s: the session was written but is not usable: %s" % (ap.prog, exc), + file=sys.stderr) + return 1 + + if args.json: + print(json.dumps(summary, sort_keys=True, indent=2)) + else: + _print_summary(summary) + print() + if usable: + print("Next steps:") + print(" engraphis-dashboard # your plan's features are now unlocked") + print(" engraphis-init --check # verify the installation") + else: + print("Note: hosted compute is not configured yet, so managed compute " + "features stay off.") + print(" Set ENGRAPHIS_CLOUD_COMPUTE_URL (or rerun with --compute-url) " + "using the endpoint") + print(" shown in your account portal. Everything else is connected.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/entry.py b/scripts/entry.py new file mode 100644 index 0000000..c486055 --- /dev/null +++ b/scripts/entry.py @@ -0,0 +1,89 @@ +"""engraphis - the single front door, so the portal's copy-paste command actually runs. + +Every capability already ships as its own ``engraphis-`` console script; this adds +the spelling customers are *shown*. The account portal hands out + + engraphis connect --token engr_ct_... + +and until there was an ``engraphis`` executable that string was not a runnable command. + +This is a dispatcher, not a second CLI: it rewrites ``sys.argv`` and calls the same +``main()`` the matching ``engraphis-`` script calls, so a verb behaves identically +either way and there is exactly one implementation of each command. The import is lazy +so ``engraphis connect`` never drags in the memory/embedding stack that ``engraphis cli`` +needs. +""" +from __future__ import annotations + +import sys +from importlib import import_module + +#: verb -> "module:function", mirroring ``[project.scripts]`` with the prefix dropped. +COMMANDS = { + "connect": "scripts.connect:main", + "init": "scripts.init:main", + "cli": "scripts.cli:main", + "mcp": "engraphis.mcp_cli:main", + "server": "scripts.start_server:main", + "dashboard": "scripts.start_dashboard:main", + "inspector": "scripts.inspector:main", + "consolidate": "scripts.consolidate:main", + "graph": "scripts.graph_cli:main", + "graph-server": "scripts.graph_server:main", + "update": "scripts.update:main", +} + +_USAGE = """usage: engraphis [options] + +commands: + connect redeem the connect token from your account portal + init write a project .env and print agent setup snippets + cli store and recall memories from the terminal + mcp run the MCP server (Claude Code, Cursor, Cline, Zed) + server run the REST server + dashboard run the product dashboard + inspector inspect the local database + consolidate run consolidation over stored memories + graph query the knowledge graph + graph-server run the graph server + update check for and install a newer Engraphis release + +Run `engraphis --help` for a command's options. +Every command is also installed as `engraphis-`.""" + + +def main(argv=None) -> int: + args = list(sys.argv[1:] if argv is None else argv) + if not args or args[0] in {"-h", "--help", "help"}: + print(_USAGE) + return 0 if args else 2 + if args[0] in {"-V", "--version"}: + from engraphis import __version__ + + print(__version__) + return 0 + + verb = args[0] + target = COMMANDS.get(verb) + if target is None: + print("engraphis: unknown command %r\n" % verb, file=sys.stderr) + print(_USAGE, file=sys.stderr) + return 2 + + module_name, _, attribute = target.partition(":") + command = getattr(import_module(module_name), attribute) + + # Hand the verb its own argv. Some targets take ``main(argv)`` and some read + # ``sys.argv`` directly, so rewriting is the one approach that works for all of + # them -- and it also puts "engraphis connect" in that command's --help/usage. + saved = sys.argv + sys.argv = ["engraphis %s" % verb] + args[1:] + try: + result = command() + finally: + sys.argv = saved + return 0 if result is None else int(result) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/externalize_dashboard_assets.py b/scripts/externalize_dashboard_assets.py index 4c27b2d..927f4e2 100644 --- a/scripts/externalize_dashboard_assets.py +++ b/scripts/externalize_dashboard_assets.py @@ -19,6 +19,26 @@ CSS = STATIC / "dashboard.css" JS = STATIC / "dashboard.js" +#: First-party scripts the dashboard page loads besides ``dashboard.js``. They run under the +#: same strict CSP, so they are held to the same no-inline-style/no-inline-handler contract. +#: Vendored bundles under ``static/vendor`` are excluded: they are third-party artefacts we +#: do not rewrite, and pinning them is the job of the commercial-manifest check. +V2_ASSETS = ROOT / "engraphis" / "dashboard_assets" +EXTRA_SCRIPTS = (V2_ASSETS / "engraphis-graph.js",) + +#: Scripts that must never be referenced from a ``" + ) + + assert styles == [] + assert [asset.content for asset in scripts] == ["first()", "second()"] + + def test_migrate_uses_parsed_asset_boundaries(tmp_path, monkeypatch): index = tmp_path / "index.html" css = tmp_path / "dashboard.css" js = tmp_path / "dashboard.js" + # ``migrate()`` ends by running the full gate over what it just wrote, so the inline script + # has to carry the deferred-asset loaders the gate now requires. index.write_text( "" "" - "", + "", encoding="utf-8", ) monkeypatch.setattr(assets, "INDEX", index) @@ -52,3 +66,158 @@ def test_check_rejects_unclosed_inline_asset_at_eof(tmp_path, monkeypatch, tag): with pytest.raises(SystemExit, match=f"inline {tag} block"): assets.check() + + +# ── deferred graph assets ─────────────────────────────────────────────────────────────── +# force-graph.min.js applies inline styles at runtime, so under `style-src 'self'` loading it +# on every page reported a CSP violation per attempt. It and the opt-in engine are fetched on +# demand instead; these rules keep that true and keep the deferred references checked. + +_LOADERS = ( + "script.src='/static/vendor/force-graph.min.js';" + "script.src='/v2-assets/engraphis-graph.js';" +) + + +def _gate(tmp_path, monkeypatch, html: str, js: str) -> None: + index, css, script = ( + tmp_path / "index.html", + tmp_path / "dashboard.css", + tmp_path / "dashboard.js", + ) + index.write_text(html, encoding="utf-8") + css.write_text("", encoding="utf-8") + script.write_text(js, encoding="utf-8") + monkeypatch.setattr(assets, "INDEX", index) + monkeypatch.setattr(assets, "CSS", css) + monkeypatch.setattr(assets, "JS", script) + # Isolate these cases to the script-reference rules; the first-party CSP scan has its own. + monkeypatch.setattr(assets, "EXTRA_SCRIPTS", ()) + + +def test_check_rejects_an_eagerly_loaded_csp_hostile_script(tmp_path, monkeypatch): + """Putting the tag back in index.html is precisely the regression this rule catches.""" + _gate( + tmp_path, + monkeypatch, + '', + _LOADERS, + ) + + with pytest.raises(SystemExit, match="must not eagerly load: /static/vendor/force-graph"): + assets.check() + + +def test_check_rejects_an_eagerly_loaded_v2_graph_engine(tmp_path, monkeypatch): + _gate( + tmp_path, + monkeypatch, + '', + _LOADERS, + ) + + with pytest.raises(SystemExit, match="must not eagerly load: /v2-assets/engraphis-graph"): + assets.check() + + +def test_check_allows_a_deferred_script_named_only_in_a_comment(tmp_path, monkeypatch): + """The rule reads parsed ``' + "" + '' + '' + "" + "" + "" + ) + + assert found == [ + "/static/vendor/force-graph.min.js", + "/static/engraphis-graph.js", + "/static/dashboard.js", + ] + + +def test_check_rejects_an_eagerly_loaded_script_in_any_tag_case(tmp_path, monkeypatch): + """The uppercase spelling is loaded by browsers, so it must fail the same rule.""" + _gate( + tmp_path, + monkeypatch, + '', + _LOADERS, + ) + + with pytest.raises(SystemExit, match="must not eagerly load: /static/vendor/force-graph"): + assets.check() + + +def test_check_rejects_a_mixed_case_reference_to_a_missing_file(tmp_path, monkeypatch): + """The existence check must not have a case-shaped hole either.""" + _gate( + tmp_path, + monkeypatch, + "", + _LOADERS, + ) + + with pytest.raises(SystemExit, match=r"referenced script is missing: .*renamed-bundle\.js"): + assets.check() + + +def test_uppercase_script_src_is_not_mistaken_for_an_inline_block(tmp_path, monkeypatch): + """The inline-block rule reads the same parse, so an external tag stays external.""" + _gate( + tmp_path, + monkeypatch, + '', + _LOADERS, + ) + + assets.check() diff --git a/tests/test_graph_engine_asset.py b/tests/test_graph_engine_asset.py new file mode 100644 index 0000000..00ed262 --- /dev/null +++ b/tests/test_graph_engine_asset.py @@ -0,0 +1,1646 @@ +"""Contract checks for the opt-in browser graph engine (``?graph-engine=next``). + +These tests intentionally stay dependency-light: the dashboard's offline CI floor does +not need a browser or a JavaScript package manager just to validate a shipped static +asset. Where Node is available the asset is *executed* rather than pattern-matched, so +the checks assert behaviour (escaping, bridge detection, stack safety, load-order +independence) instead of the presence of source substrings. + +The properties guarded here are the ones whose failure is silent in a browser: + +* the asset must define its global without touching ``ForceGraph``/``document``, so a + blocked or missing vendor bundle degrades instead of white-screening the dashboard; +* every label crossing into force-graph must be escaped, because force-graph's tooltip + is an ``innerHTML`` sink and entity labels come from ingested memories; +* the client-side graph analysis must not recurse per node or run unbounded work; +* the per-style pane backgrounds must stay in CSS, since the production CSP sets + ``style-src-attr 'none'``. +""" + +from __future__ import annotations + +import json +import re +import shutil +import subprocess +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +STATIC = ROOT / "engraphis" / "static" +ASSET = ROOT / "engraphis" / "dashboard_assets" / "engraphis-graph.js" +LEGACY_ADAPTER = STATIC / "engraphis-graph.js" +INDEX = STATIC / "index.html" +CSS = STATIC / "dashboard.css" +DASHBOARD = STATIC / "dashboard.js" +VENDOR = STATIC / "vendor" / "force-graph.min.js" + +NODE = shutil.which("node") +requires_node = pytest.mark.skipif(NODE is None, reason="node is not installed") + +#: Evaluates the asset with nothing but a bare ``window`` object in scope. Any top-level +#: use of a browser or vendor global would raise here, which is the point. +PRELUDE = """ +const fs = require('fs'); +const source = fs.readFileSync(process.argv[1], 'utf8'); +const window = {}; +new Function('window', source)(window); +const G = window.EngraphisGraph; +const I = G._internals; +const emit = value => console.log(JSON.stringify(value)); +""" + + +#: Same, plus a recording stand-in for force-graph so ``create()`` can be *driven*. Every +#: accessor is a chainable setter that returns the stored value when called with no arguments — +#: force-graph's own kapsule semantics — so the paint configuration the engine installs can be +#: read back and invoked instead of pattern-matched. ``calls`` counts the invalidations the +#: engine requests, which is the only observable form a "redraw now" takes. ``invocations`` +#: counts the *argument-less* calls, which under kapsule semantics are the commands rather than +#: the setters — ``d3ReheatSimulation()`` is one, and it has no other observable effect here. +ENGINE_PRELUDE = """ +const fs = require('fs'); +const source = fs.readFileSync(process.argv[1], 'utf8'); +const window = {}; +globalThis.requestAnimationFrame = () => {}; +const store = {}, calls = {}, invocations = {}; +const fg = new Proxy({}, { + get: (_target, prop) => (...args) => { + if (!args.length) { invocations[prop] = (invocations[prop] || 0) + 1; return store[prop]; } + calls[prop] = (calls[prop] || 0) + 1; + store[prop] = args.length === 1 ? args[0] : args; + return fg; + }, +}); +globalThis.ForceGraph = () => () => fg; +const el = { + attrs: {}, innerHTML: '', clientWidth: 800, clientHeight: 600, + getAttribute(name) { return this.attrs[name] === undefined ? null : this.attrs[name]; }, + setAttribute(name, value) { this.attrs[name] = value; }, + removeAttribute(name) { delete this.attrs[name]; }, + classList: { toggle() {}, remove() {} }, +}; +const chain = count => { + const nodes = [], links = []; + for (let i = 0; i <= count; i++) nodes.push({ id: 'n' + i }); + for (let i = 0; i < count; i++) { + links.push({ source: 'n' + i, target: 'n' + (i + 1), layer: 'semantic' }); + } + return { nodes, links }; +}; +new Function('window', source)(window); +const G = window.EngraphisGraph; +const I = G._internals; +const emit = value => console.log(JSON.stringify(value)); +""" + + +def _run_node(script: str, prelude: str = PRELUDE) -> object: + result = subprocess.run( + [NODE, "-e", prelude + script, str(ASSET)], + cwd=ROOT, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr + return json.loads(result.stdout.strip().splitlines()[-1]) + + +def _run_engine(script: str) -> object: + return _run_node(script, prelude=ENGINE_PRELUDE) + + +# ── load order and failure isolation ──────────────────────────────────────────────── + + +def test_graph_assets_are_never_loaded_on_a_plain_page_view() -> None: + """Neither graph script may sit in index.html. + + force-graph applies inline styles at runtime, so under the production CSP + (``style-src 'self'``) every page load that fetched it reported a violation per attempt — + including the pages that never open the graph. + """ + html = INDEX.read_text(encoding="utf-8") + eager = re.findall(r']+src=["\'](/static/[^"\']+)["\']', html) + assert "/static/vendor/d3.min.js" in eager + assert "/static/dashboard.js" in eager + assert "/static/vendor/force-graph.min.js" not in eager + assert "/static/engraphis-graph.js" not in eager + + +def test_v1_graph_asset_is_only_a_compatibility_adapter() -> None: + """New renderer code stays on the v2 dashboard surface, not the legacy server.""" + adapter = LEGACY_ADAPTER.read_text(encoding="utf-8") + assert "canonicalAsset: '/v2-assets/engraphis-graph.js'" in adapter + assert "window.EngraphisGraph =" not in adapter + assert "window.EngraphisGraph =" in ASSET.read_text(encoding="utf-8") + + +def test_opt_in_graph_asset_is_lazily_loaded_after_its_dependencies() -> None: + """The load order the removed script tags used to guarantee now lives in graphRender(). + + ``graphRender`` returns early until ForceGraph is defined, so by the time the engine + branch runs its dependency is already in scope. + """ + source = DASHBOARD.read_text(encoding="utf-8") + assert "script.src='/static/vendor/force-graph.min.js'" in source + assert "script.src='/v2-assets/engraphis-graph.js'" in source + render = source[source.index("function graphRender("):] + render = render[: render.index("\nfunction ")] + force_graph_gate = render.index("typeof ForceGraph==='undefined'") + engine_gate = render.index("if(enginePending)") + classic = render.index("graphRenderEngine(data,fit,reheat)") + assert force_graph_gate < engine_gate < classic + + +def test_engine_node_labels_honor_the_configured_font_at_normal_zoom() -> None: + source = ASSET.read_text(encoding="utf-8") + assert "state.settings.font / scale / 3.4" not in source + assert "state.settings.font / scale" in source + + +#: Executes dashboard.js's real graph-render *routing* decision against a stub DOM. +#: ``graphEngineEnabled``, ``graphEngineFallback``, ``loadForceGraph``, ``loadGraphEngine`` and +#: the routing half of ``graphRender`` are verbatim source slices — nothing is re-implemented. +#: Only the classic renderer body below the routing decision is swapped for a ``CLASSIC()`` +#: marker, so the test can see which renderer a deep link actually reaches. +ROUTING_HARNESS = """ +const fs = require('fs'); +const src = fs.readFileSync(process.argv.slice(1).find(a => a.endsWith('dashboard.js')), 'utf8'); +const scenario = process.argv[process.argv.length - 1]; +const between = (from, to) => src.slice(src.indexOf(from), src.indexOf(to, src.indexOf(from))); +const flags = between('let GRAPH_ENGINE_FAILED=false;', 'function graphEngineEmptyMessage'); +const loaders = between('let FORCE_GRAPH_LOADING=null;', 'function graphRender('); +const DECISION = 'if(graphEngineEnabled()&&graphRenderEngine(data,fit,reheat))return;'; +const start = src.indexOf('function graphRender('); +const routing = src.slice(start, src.indexOf(DECISION, start) + DECISION.length) + + '\\n CLASSIC();\\n}'; + +const log = { appended: [], warned: [], engine: 0, classic: 0 }; +let pending = null; +const element = { clientWidth: 800, clientHeight: 600, classList: { toggle() {} }, + setAttribute() {}, set textContent(v) {} }; +globalThis.document = { + getElementById: () => element, + querySelectorAll: () => [], + createElement: () => (pending = {}), + head: { appendChild: s => log.appended.push(s.src) }, +}; +globalThis.window = { location: { search: '?graph-engine=next' }, GSET: { mode: 'compact' }, + console: globalThis.console }; +globalThis.console = { warn: (...a) => log.warned.push(String(a[0])) }; +globalThis.showAs = () => {}; +globalThis.graphSetLayoutStatus = () => {}; +globalThis.graphData = () => ({ nodes: [], links: [] }); +/* Mirrors graphRenderEngine's real first line — `if(!element||typeof EngraphisGraph=== + 'undefined')return false` — because that bail is exactly what a naive lazy-load would turn + into a silent Classic fallback. Asserted against the real source below. */ +globalThis.graphRenderEngine = () => { + if (typeof EngraphisGraph === 'undefined') return false; + log.engine += 1; + return true; +}; +globalThis.CLASSIC = () => { log.classic += 1; }; +globalThis.GRAPH_PRESETS = { compact: {} }; +globalThis.GRAPH_ENGINE = globalThis.GACTIVE_DATA = globalThis.GCOMPONENT_LAYOUT = null; +globalThis.GHILITE = globalThis.GHOVERSET = null; +/* The vendor bundle is already in scope: this exercises the engine gate, not the vendor gate. */ +globalThis.ForceGraph = function () {}; + +new Function(flags + loaders + routing + '\\nreturn {graphRender};')().graphRender(); +const settled = { engine: log.engine, classic: log.classic }; +if (scenario === 'loads') { globalThis.EngraphisGraph = { create() {} }; pending.onload(); } +else { pending.onerror(); } +setTimeout(() => process.stdout.write(JSON.stringify({ + beforeSettle: settled, engine: log.engine, classic: log.classic, + appended: log.appended, warned: log.warned, +})), 0); +""" + + +def _run_routing(scenario: str) -> dict: + result = subprocess.run( + [NODE, "-e", ROUTING_HARNESS, str(DASHBOARD), scenario], + cwd=ROOT, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr + return json.loads(result.stdout.strip().splitlines()[-1]) + + +@requires_node +def test_graph_engine_deep_link_reaches_the_next_engine_after_a_lazy_load() -> None: + """``?graph-engine=next`` must not degrade just because its asset is not loaded yet. + + ``graphRenderEngine`` bails when ``EngraphisGraph`` is undefined, and that bail cannot tell + "not fetched yet" from "unavailable". Deferring the script would turn every deep link into + that bail — the user asks for the new engine and silently gets Classic. So graphRender + fetches the asset and waits, then renders. + """ + # Keep the harness's stub honest: it only proves anything while the real function really + # does bail on an undefined global. + source = DASHBOARD.read_text(encoding="utf-8") + engine_path = source[source.index("function graphRenderEngine"):] + assert "typeof EngraphisGraph==='undefined')return false" in engine_path[:400] + + report = _run_routing("loads") + + assert report["appended"] == ["/v2-assets/engraphis-graph.js"] + # It waits rather than rendering something wrong in the meantime. + assert report["beforeSettle"] == {"engine": 0, "classic": 0} + # And it lands on the next engine, never touching the classic renderer. + assert report["engine"] == 1 + assert report["classic"] == 0 + assert report["warned"] == [] + + +@requires_node +def test_graph_engine_deep_link_degrades_loudly_when_the_asset_cannot_load() -> None: + """A genuine load failure is the only thing that reaches Classic, and it says so.""" + report = _run_routing("fails") + + assert report["engine"] == 0 + assert report["classic"] == 1 + assert report["warned"] == [ + "graph-engine=next failed; falling back to the classic renderer" + ] + + +def test_lazy_graph_engine_load_cannot_raise_an_unhandled_rejection() -> None: + """An unhandled rejection prints a console error — the exact thing this fix removes. + + ``graphRender`` can start the engine fetch on a pass that returns at the ForceGraph gate, + before it attaches its own handler, so the memoized promise carries its own. + """ + source = DASHBOARD.read_text(encoding="utf-8") + loader = source[source.index("function loadGraphEngine()"):] + loader = loader[: loader.index("\nfunction ")] + assert "GRAPH_ENGINE_LOADING.catch(()=>{})" in loader + # A 200 that never registers the global is a corrupt asset, not a success. + assert "reject(new Error('Graph engine asset loaded without registering EngraphisGraph'))" in loader + + +def test_force_graph_loader_rejects_a_success_without_the_vendor_global() -> None: + """A truncated 200 must not enter the render loop without ``ForceGraph``.""" + source = DASHBOARD.read_text(encoding="utf-8") + loader = source[source.index("function loadForceGraph()"):] + loader = loader[: loader.index("\nlet GRAPH_ENGINE_LOADING")] + assert "typeof ForceGraph==='undefined'" in loader + assert "reject(new Error('Force graph asset loaded without registering ForceGraph'))" in loader + + +@requires_node +def test_graph_asset_defines_its_global_without_touching_its_dependencies() -> None: + """Nothing may run at parse time except pure setup. + + ``PRELUDE`` supplies no ``ForceGraph``, no ``document`` and no ``requestAnimationFrame``. + If the asset reached for any of them at the top level this would throw, and in a browser + the same reach would abort the script and take ``window.EngraphisGraph`` with it. + """ + report = _run_node( + """ + emit({ + create: typeof G.create, + presets: Object.keys(G.PRESETS).sort(), + styles: Object.keys(G.STYLE_LAYERS).sort(), + }); + """ + ) + assert report["create"] == "function" + assert "communities" in report["presets"] + assert report["styles"] == ["classic", "cyber", "galaxy", "solar"] + + +@requires_node +def test_create_fails_loudly_when_force_graph_is_unavailable() -> None: + """A blocked vendor bundle must raise, not half-initialise a dead canvas.""" + report = _run_node( + """ + let message = null; + try { G.create({ getAttribute() { return null; } }, {}); } + catch (error) { message = error.message; } + emit({ message }); + """ + ) + assert report["message"] == "force-graph not loaded" + + +def test_dashboard_falls_back_to_the_classic_renderer_when_the_engine_throws() -> None: + source = DASHBOARD.read_text(encoding="utf-8") + # The opt-in flag must be latched off after a failure, and the render path must catch. + assert "GRAPH_ENGINE_FAILED" in source + assert "if(GRAPH_ENGINE_FAILED)return false" in source + assert "graphEngineFallback(error)" in source + engine_path = source[source.index("function graphRenderEngine"):] + engine_path = engine_path[: engine_path.index("\nfunction ")] + assert "try{" in engine_path and "}catch(error){" in engine_path + + +# ── XSS: untrusted entity labels reaching force-graph ─────────────────────────────── + + +def test_force_graph_tooltip_is_still_an_inner_html_sink() -> None: + """Guards the *reason* the engine sets its own label accessors. + + force-graph defaults ``nodeLabel``/``linkLabel`` to the accessor ``"name"`` and renders a + string label through ``innerHTML``. Node names here are entity labels extracted from + ingested memories, i.e. untrusted. If a vendor bump ever changes this, revisit whether + the explicit escaped accessors below are still the right shape. + """ + vendor = VENDOR.read_text(encoding="utf-8", errors="ignore") + assert 'nodeLabel:{default:"name"' in vendor + assert 'linkLabel:{default:"name"' in vendor + + +def test_engine_never_relies_on_the_default_label_accessor() -> None: + source = ASSET.read_text(encoding="utf-8") + assert ".nodeLabel(node => esc(nodeName(node)))" in source + assert ".linkLabel(" in source + assert "eval(" not in source + # The engine paints to canvas; the only markup sink it may use is clearing its own + # container on teardown. Anything else would be a route for an unescaped entity label. + writes = re.findall(r"\w+\.(?:inner|outer)HTML\s*=\s*[^;]+", source) + assert writes == ["el.innerHTML = ''"], writes + assert not re.search(r"insertAdjacentHTML|document\.write|createContextualFragment", source) + + +@requires_node +@pytest.mark.parametrize( + "payload", + [ + "", + "", + "\" onmouseover=\"alert(1)", + "", + ], +) +def test_entity_labels_are_escaped_before_they_can_reach_a_dom_sink(payload: str) -> None: + report = _run_node( + "emit({ escaped: I.esc(%s), named: I.nodeName({ label: %s }) });" + % (json.dumps(payload), json.dumps(payload)) + ) + escaped = report["escaped"] + assert "<" not in escaped and ">" not in escaped + assert '"' not in escaped and "'" not in escaped + assert "<" in escaped or """ in escaped + # nodeName is the raw value; escaping is the accessor's job, so this documents the split. + assert report["named"] == payload + + +# ── payload compatibility with the shipped /graph endpoint ────────────────────────── + + +@requires_node +def test_engine_accepts_both_the_api_and_renderer_link_shapes() -> None: + report = _run_node( + """ + const api = { from: 'a', to: 'b' }; + const renderer = { source: { id: 'c' }, target: 'd' }; + emit({ + apiSource: I.linkEndpoint(api, 'source'), + apiTarget: I.linkEndpoint(api, 'target'), + rendererSource: I.linkEndpoint(renderer, 'source'), + rendererTarget: I.linkEndpoint(renderer, 'target'), + label: I.nodeName({ label: 'Ada' }), + name: I.nodeName({ name: 'Grace' }), + fallback: I.nodeName({ id: 'ent_1' }), + }); + """ + ) + assert report["apiSource"] == "a" and report["apiTarget"] == "b" + assert report["rendererSource"] == "c" and report["rendererTarget"] == "d" + assert report["label"] == "Ada" + assert report["name"] == "Grace" + assert report["fallback"] == "ent_1" + + +@requires_node +def test_valid_time_accepts_seconds_milliseconds_and_iso_strings() -> None: + report = _run_node( + """ + emit({ + seconds: I.asOfValue(1700000000), + millis: I.asOfValue(1700000000000), + iso: I.asOfValue('2023-11-14T22:13:20Z'), + blank: I.asOfValue(''), + junk: I.asOfValue('not a date'), + }); + """ + ) + assert report["seconds"] == report["millis"] == 1700000000000 + assert report["iso"] == 1700000000000 + assert report["blank"] is None and report["junk"] is None + + +# ── client-side analysis: correctness and cost ────────────────────────────────────── + + +@requires_node +def test_bridge_detection_matches_a_known_graph() -> None: + """A triangle has no bridges; the tail hanging off it is all bridges.""" + report = _run_node( + """ + const nodes = ['a', 'b', 'c', 'd', 'e'].map(id => ({ id })); + const links = [['a','b'], ['b','c'], ['c','a'], ['c','d'], ['d','e']] + .map(([source, target]) => ({ source, target })); + const adj = I.communities(nodes, links); + I.findBridges(nodes, links, adj); + emit({ + bridges: links.filter(l => l.bridge).map(l => l.source + '-' + l.target), + communities: new Set(nodes.map(n => n.community)).size, + }); + """ + ) + assert report["bridges"] == ["c-d", "d-e"] + assert report["communities"] == 1 + + +@requires_node +def test_parallel_edges_are_not_reported_as_bridges() -> None: + report = _run_node( + """ + const nodes = [{ id: 'a' }, { id: 'b' }]; + const links = [{ source: 'a', target: 'b' }, { source: 'a', target: 'b' }]; + const adj = I.communities(nodes, links); + I.findBridges(nodes, links, adj); + emit({ bridges: links.filter(l => l.bridge).length }); + """ + ) + assert report["bridges"] == 0 + + +@requires_node +def test_disconnected_entities_are_labelled_as_separate_communities() -> None: + report = _run_node( + """ + const nodes = ['a', 'b', 'c', 'd'].map(id => ({ id })); + const links = [{ source: 'a', target: 'b' }, { source: 'c', target: 'd' }]; + const adj = I.communities(nodes, links); + emit({ groups: new Set(nodes.map(n => n.community)).size }); + """ + ) + assert report["groups"] == 2 + + +@requires_node +def test_graph_analysis_is_stack_safe_and_bounded_on_a_large_store() -> None: + """A long chain of entities is the worst case for both analyses. + + A recursive Tarjan overflows the call stack here, and exact Brandes betweenness is + O(V*E) — minutes of blocked main thread. Both are guarded, so this must finish well + inside the bound even on a slow machine. + """ + report = _run_node( + """ + const N = 40000; + const nodes = [], links = []; + for (let i = 0; i < N; i++) { + nodes.push({ id: 'n' + i }); + if (i) links.push({ source: 'n' + (i - 1), target: 'n' + i }); + } + const adj = I.communities(nodes, links); + const started = Date.now(); + I.findBridges(nodes, links, adj); + I.betweenness(nodes, adj); + const scores = nodes.map(n => n.betweenness); + emit({ + ms: Date.now() - started, + allBridges: links.every(l => l.bridge), + finite: scores.every(Number.isFinite), + peak: Math.max.apply(null, scores.slice(0, 1000).concat(scores.slice(-1000))), + }); + """ + ) + assert report["allBridges"] is True + assert report["finite"] is True + # Ends of a chain are never on a shortest path between others. + assert report["peak"] < 0.5 + assert report["ms"] < 30000, f"graph analysis took {report['ms']}ms on 40k entities" + + +@requires_node +def test_influence_relations_do_not_merge_two_topics_into_one_community() -> None: + """Community Islands must not fuse two topics over a single cross-topic relation. + + ``influences`` edges routinely span otherwise separate bodies of work. The classic + renderer keeps them drawn and traversable but builds its clustering adjacency without + them (``GCOMM_ADJ``); adding every link to one adjacency gives both topics the same + colour and the same force centre. + """ + report = _run_node( + """ + const nodes = ['a', 'b', 'c', 'd'].map(id => ({ id })); + const links = [ + { source: 'a', target: 'b', label: 'mentions' }, + { source: 'c', target: 'd', label: 'mentions' }, + { source: 'b', target: 'c', label: 'influences' }, + ]; + const adj = I.communities(nodes, links); + I.findBridges(nodes, links, adj); + emit({ + groups: new Set(nodes.map(n => n.community)).size, + merged: nodes[1].community === nodes[2].community, + neighbours: (adj.b || []).slice().sort(), + bridges: links.filter(l => l.bridge).length, + }); + """ + ) + assert report["groups"] == 2 + assert report["merged"] is False + # The relation itself stays in the traversal adjacency: hover neighbourhood, focus depth + # and bridge detection all still see it. Only the clustering ignores it. + assert report["neighbours"] == ["a", "c"] + assert report["bridges"] == 3 + + +@requires_node +def test_community_ids_are_ranked_by_size_so_the_legend_describes_the_right_nodes() -> None: + """Legend labels and canvas swatches must agree about which cluster is "Cluster 1". + + ``graphRenderLegend()`` sorts communities by size and calls the largest "Cluster 1", but + node colour indexes the palette by the community *id* (``commPal()[community % n]``). + Assigning ids in raw payload order therefore made the legend describe one component with + another's colour whenever a smaller component appeared first — which the payload order + alone decides. The classic ``graphComputeCommunities()`` sorts before assigning; so must + this. + """ + report = _run_node( + """ + // Payload order is deliberately worst-case: the singleton comes first, the largest + // component last, so raw iteration order and size order disagree completely. + const nodes = ['solo', 'm1', 'm2', 'a', 'b', 'c'].map(id => ({ id })); + const links = [ + { source: 'm1', target: 'm2' }, + { source: 'a', target: 'b' }, + { source: 'b', target: 'c' }, + ]; + I.communities(nodes, links); + const byId = {}; + nodes.forEach(n => { byId[n.id] = n.community; }); + emit({ byId, distinct: new Set(nodes.map(n => n.community)).size }); + """ + ) + assert report["distinct"] == 3 + # Largest component (3 nodes) owns palette slot 0, i.e. the legend's "Cluster 1". + assert report["byId"]["a"] == 0 + assert report["byId"]["b"] == 0 + assert report["byId"]["c"] == 0 + # Then the 2-node component, then the singleton — strictly by size, not by payload order. + assert report["byId"]["m1"] == 1 + assert report["byId"]["m2"] == 1 + assert report["byId"]["solo"] == 2 + + +@requires_node +def test_max_helper_survives_arrays_past_the_spread_limit() -> None: + """``Math.max(...array)`` throws RangeError long before a store is unrenderable.""" + report = _run_node("emit({ max: I.maxOf(new Array(400000).fill(7), 1) });") + assert report["max"] == 7 + + +@requires_node +def test_colour_helpers_handle_the_shorthand_hex_the_palettes_may_carry() -> None: + report = _run_node( + """ + emit({ + short: I.hexRgb('#abc'), + long: I.hexRgb('#8c83e8'), + empty: I.hexRgb(''), + light: I.contrastOn('#ffffff'), + dark: I.contrastOn('#000000'), + }); + """ + ) + assert report["short"] == [170, 187, 204] + assert report["long"] == [140, 131, 232] + assert report["empty"] == [140, 131, 232] + assert report["light"] == "#111827" + assert report["dark"] == "#f8fafc" + + +# ── render configuration: what the engine actually installs on force-graph ────────── + + +@requires_node +def test_flow_particles_are_capped_on_a_large_relation_set() -> None: + """Three animated particles per relation does not survive a real ``/graph`` response. + + force-graph advances every particle on every frame, so a few thousand relations is tens + of thousands of animated objects and an unusable canvas. The classic renderer refuses to + draw them past 800 links; the opt-in engine must use the same cutoff rather than trusting + that no store is big. + """ + report = _run_engine( + """ + const api = G.create(el, {}); + const particlesFor = link => store.linkDirectionalParticles(link || { layer: 'semantic' }); + api.setStyle('cyber'); + api.setSettings({ flow: true }); + api.setData(chain(40)); + const small = particlesFor(); + api.setData(chain(800)); + const atLimit = particlesFor(); + api.setData(chain(801)); + const overLimit = particlesFor(); + api.setData(chain(4000)); + emit({ small, atLimit, overLimit, realistic: particlesFor() * 4000 }); + """ + ) + assert report["small"] == 3 + assert report["atLimit"] == 3 + assert report["overLimit"] == 0 + # The number this guards: 4k relations x 3 particles was 12,000 animated objects a frame. + assert report["realistic"] == 0 + + +#: A canvas 2D stand-in that counts the fills the galaxy starfield performs. The engine wraps +#: ``onRenderFramePre`` in a try/catch, so a stub too thin to survive the real paint would read +#: as "no stars drawn"; the small-graph leg of the test below is what proves it is thick enough. +CANVAS_STUB = """ +let fills = 0; +const ctx = { + globalAlpha: 1, globalCompositeOperation: '', fillStyle: '', strokeStyle: '', lineWidth: 1, + save() {}, restore() {}, beginPath() {}, arc() {}, ellipse() {}, stroke() {}, + fill() { fills += 1; }, + createRadialGradient() { return { addColorStop() {} }; }, +}; +""" + + +@requires_node +def test_galaxy_stops_animating_once_the_graph_is_large() -> None: + """A settled graph must fall off the CPU, and galaxy was the one style that never did. + + The starfield lives in ``onRenderFramePre``, which force-graph's change detection cannot + see, so the engine holds ``autoPauseRedraw(false)`` for it — repainting every node and link + every frame, forever, even after particles and the simulation have stopped. The classic + path simply drops the starfield past ``GPERF.large`` (``if(GPERF.large)return``); with the + stars gone there is nothing left that needs a frame the vendor would not schedule itself. + """ + report = _run_engine( + CANVAS_STUB + + """ + const api = G.create(el, {}); + api.setStyle('galaxy'); + + api.setData(chain(40)); + const smallAutoPause = store.autoPauseRedraw; + fills = 0; store.onRenderFramePre(ctx, 1); + const smallStars = fills; + + // 3001 entities / 3000 relations — past the classic renderer's 600-node signal. + api.setData(chain(3000)); + const bigAutoPause = store.autoPauseRedraw; + fills = 0; store.onRenderFramePre(ctx, 1); + const bigStars = fills; + + // Style is what costs the frames, not size alone: cyber never asked for them. + api.setStyle('cyber'); + api.setData(chain(40)); + emit({ smallAutoPause, bigAutoPause, smallStars, bigStars, + cyberAutoPause: store.autoPauseRedraw }); + """ + ) + # Small galaxy graph: the animation is affordable, so the engine keeps driving frames. + assert report["smallAutoPause"] is False + assert report["smallStars"] > 0, "canvas stub never reached the starfield" + # Large galaxy graph: no starfield, and the redraw loop is handed back to force-graph. + assert report["bigStars"] == 0 + assert report["bigAutoPause"] is True, "a large galaxy graph repaints every frame forever" + assert report["cyberAutoPause"] is True + + +@requires_node +def test_large_graphs_reduce_collision_work_and_skip_cyber_shadows() -> None: + """The next engine keeps the classic path's two largest per-node costs bounded. + + Collision relaxation is needed twice per tick for a small layout but only once once the + classic ``GPERF.large`` threshold is crossed. Cyber's rich-node blur is similarly useful + at ordinary scale and prohibitively expensive when hundreds of nodes paint each frame. + Exercise the actual force and canvas accessors rather than matching their source text. + """ + report = _run_engine( + """ + const collide = () => { + let iterations = null; + return { + iterations(value) { + if (arguments.length) { iterations = value; return this; } + return iterations; + }, + }; + }; + globalThis.d3 = { + forceX: () => ({ strength() { return this; } }), + forceY: () => ({ strength() { return this; } }), + forceCollide: collide, + }; + const shadows = () => { + const writes = []; + const ctx = { + save() {}, restore() {}, beginPath() {}, arc() {}, fill() {}, stroke() {}, + set shadowColor(value) { writes.push(['color', value]); }, + set shadowBlur(value) { writes.push(['blur', value]); }, + }; + store.nodeCanvasObject( + { id: 'rich', x: 0, y: 0, radius: 5, color: '#22e0ff', degree: 3 }, ctx, 1 + ); + return writes; + }; + const api = G.create(el, {}); + api.setStyle('cyber'); + api.setData(chain(40)); + const smallIterations = store.d3Force[1].iterations(); + const smallShadows = shadows(); + api.setData(chain(601)); + const largeIterations = store.d3Force[1].iterations(); + const largeShadows = shadows(); + emit({ smallIterations, largeIterations, smallShadows, largeShadows }); + """ + ) + assert report["smallIterations"] == 2 + assert report["largeIterations"] == 1 + assert report["smallShadows"] == [["color", "#22e0ff"], ["blur", 13]] + assert report["largeShadows"] == [] + + +@requires_node +def test_type_colours_follow_the_active_theme_not_a_hard_coded_dark_palette() -> None: + """``applyTheme()`` recolours the canvas, but the engine had no theme to recolour to. + + The legend and controls read the ``--entity-*`` custom properties, so switching to Light, + Midnight, Solarized or Sepia moved them while the canvas kept the dark-theme constants — + an inconsistent palette and, on the light themes, poor contrast. The engine cannot read + CSS variables from a canvas, so the dashboard supplies the resolved values. + """ + report = _run_engine( + """ + const api = G.create(el, {}); + // setData first: the force-graph stand-in only starts answering graphData() once the + // engine has pushed data into it, where the real vendor seeds an empty graph. + // Linked, because the default scope hides degree-zero entities. + api.setData({ + nodes: [{ id: 'a', etype: 'person_or_concept' }, { id: 'b', etype: 'person_or_concept' }], + links: [{ source: 'a', target: 'b', layer: 'entity' }], + }); + api.setColorBy('type'); + api.setStyle('classic'); + // `store` holds the values handed to force-graph, so this is the node object the + // engine actually painted from — recoloured in place by refreshColors()/render(). + const colour = () => store.graphData.nodes[0].color; + + const fallback = colour(); + api.setThemeColors({ person_or_concept: '#112233' }); + const themed = colour(); + + // A style palette still outranks the theme, exactly as classic graphTypeColor() does. + api.setStyle('cyber'); + const styled = colour(); + + // ...and an explicit user override still outranks both. + api.setStyle('classic'); + api.setTypeColor('person_or_concept', '#abcdef'); + const overridden = colour(); + + // A theme with no entry for the type must not strand the previous theme's colour. + api.setThemeColors({}); + emit({ fallback, themed, styled, overridden, cleared: colour() }); + """ + ) + assert report["fallback"] == "#8c83e8" + assert report["themed"] == "#112233", "the engine ignores the active theme" + assert report["styled"] == "#ff3ea5" + assert report["overridden"] == "#abcdef" + # The override survives; only the theme tier was replaced. + assert report["cleared"] == "#abcdef" + + +@requires_node +def test_hovering_a_node_asks_for_a_redraw() -> None: + """A highlight nobody repaints is invisible. + + ``onNodeHover`` mutates closure state the paint callbacks read. With reduced motion on, + flow disabled, or a settled simulation, force-graph's ``autoPauseRedraw`` loop has nothing + left to animate and will not repaint just because the callback fired. + """ + report = _run_engine( + """ + const api = G.create(el, { reducedMotion: () => true }); + api.setData({ nodes: [{ id: 'a' }, { id: 'b' }], links: [{ source: 'a', target: 'b' }] }); + const settled = calls.nodeCanvasObject; + store.onNodeHover({ id: 'a' }); + const hovered = calls.nodeCanvasObject; + store.onNodeHover(null); + emit({ + settled, hovered, cleared: calls.nodeCanvasObject, + particles: store.linkDirectionalParticles({ layer: 'semantic' }), + }); + """ + ) + # Reduced motion: nothing is in flight, so an unrequested redraw would never arrive. + assert report["particles"] == 0 + assert report["hovered"] > report["settled"] + assert report["cleared"] > report["hovered"] + + +@requires_node +def test_unlinked_entities_are_shown_only_when_the_engine_is_told_to() -> None: + """The lever the dashboard has to pull for its "Show unlinked nodes" checkbox.""" + report = _run_engine( + """ + const seen = []; + const api = G.create(el, { onStats: stats => seen.push(stats.nodes) }); + api.setData({ + nodes: [{ id: 'a' }, { id: 'b' }, { id: 'lonely' }], + links: [{ source: 'a', target: 'b' }], + }); + const hidden = seen[seen.length - 1]; + api.setScope({ showUnlinked: true, minDegree: 0 }); + emit({ hidden, shown: seen[seen.length - 1] }); + """ + ) + assert report["hidden"] == 2 + assert report["shown"] == 3 + + +#: Executes the *real* ``graphRenderEngine`` source against stubs. Only its collaborators are +#: faked; the function itself is a verbatim slice, so what it forwards to the engine — and when +#: it parks a freshly created renderer — is observed rather than asserted about the source text. +RENDER_HARNESS = """ +const fs = require('fs'); +const src = fs.readFileSync(process.argv.slice(1).find(a => a.endsWith('dashboard.js')), 'utf8'); +const scenario = JSON.parse(process.argv[process.argv.length - 1]); +const start = src.indexOf('function graphRenderEngine('); +const slice = src.slice(start, src.indexOf('/* Nav away from the graph view', start)); + +/* The theme-colour lookup is sliced verbatim too, not stubbed: the property under test is + that the dashboard resolves the *active* CSS custom properties and hands them over, so + faking the resolver would assert nothing. Only `getComputedStyle` below is synthetic. */ +const between = (from, to) => src.slice(src.indexOf(from), src.indexOf(to, src.indexOf(from))); +const themeSrc = between('const ETYPE_TOKEN=', 'const GRAPH_PALETTES=') + + between('function cssvar(', 'function graphValidColor(') + + between('function graphThemeTypeColors(', 'function graphContrastColor('); + +/* A stand-in for a non-dark theme: every --entity-* token differs from the engine's + hard-coded THEME_ETYPE constants, so a renderer that ignored these would be visible. */ +const THEME_VARS = { + '--entity-concept': '#112233', '--entity-mention': '#223344', '--entity-hashtag': '#334455', + '--entity-email': '#445566', '--entity-organization': '#556677', '--entity-location': '#667788', + '--color-accent': '#778899', '--color-text-dim': '#123456', +}; +globalThis.getComputedStyle = () => ({ getPropertyValue: name => THEME_VARS[name] || '' }); + +const log = { created: 0, paused: 0, seeded: 0, scope: null, themeColors: null, error: null }; +const checkbox = { checked: scenario.showUnlinked }; +const element = { classList: { toggle() {} }, setAttribute() {}, set textContent(value) {} }; +globalThis.document = { + getElementById: id => (id === 'graph-show-iso' ? checkbox : element), + querySelectorAll: () => [], + body: {}, +}; +const engine = { + setSettings() {}, setStyle() {}, setColorBy() {}, setPalette() {}, setTypeColors() {}, + setLayers() {}, setScope(patch) { log.scope = patch; }, + setThemeColors(map) { log.themeColors = map; }, + setData(data) { log.seeded = data.nodes.length; }, +}; +const api = { + apply(fn) { fn(engine); }, communityMap: () => ({}), + freeze() {}, destroy() {}, resume() {}, pause() { log.paused += 1; }, +}; +globalThis.EngraphisGraph = { create() { log.created += 1; return api; } }; +globalThis.window = { GSET: { mode: 'compact', frozen: false } }; +globalThis.GRAPH = { nodes: [] }; +globalThis.GRAPH_ENGINE = null; +globalThis.GACTIVE_DATA = null; +globalThis.GCOLOR_OVERRIDES = {}; +/* The state the nav-away pause recorded while GRAPH_ENGINE was still null. */ +globalThis.GRAPH_ENGINE_PARKED = scenario.parked; +globalThis.showAs = () => {}; +globalThis.prefersReducedMotion = () => false; +for (const name of ['graphSetLayoutStatus', 'graphSyncReadouts', 'graphUpdateEditedBadge', + 'graphUpdateHud', 'graphRenderLegend', 'graphSetHighlight', + 'graphSetSimulationStatus', 'syncGraphExplorerSelection', 'graphNodeClick', + 'graphEngineEmptyMessage']) globalThis[name] = () => {}; +globalThis.graphEngineFallback = error => { + log.error = String((error && error.message) || error); +}; + +const graphRenderEngine = new Function(themeSrc + slice + '\\nreturn graphRenderEngine;')(); +const rendered = graphRenderEngine({ + nodes: [{ id: 'a' }, { id: 'b' }, { id: 'lonely' }], + links: [{ source: 'a', target: 'b' }], +}, true, true); +console.log(JSON.stringify(Object.assign({ rendered }, log))); +""" + + +def _run_render(*, show_unlinked: bool = False, parked: bool = False) -> dict: + source = DASHBOARD.read_text(encoding="utf-8") + # The harness slices real source; keep its landmarks honest. + assert "function graphRenderEngine(" in source + assert "/* Nav away from the graph view" in source + scenario = json.dumps({"showUnlinked": show_unlinked, "parked": parked}) + result = subprocess.run( + [NODE, "-e", RENDER_HARNESS, str(DASHBOARD), scenario], + cwd=ROOT, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr + report = json.loads(result.stdout.strip().splitlines()[-1]) + assert report["error"] is None, report["error"] + assert report["rendered"] is True + return report + + +@requires_node +@pytest.mark.parametrize("checked", [False, True]) +def test_dashboard_tells_the_engine_whether_to_show_unlinked_entities(checked: bool) -> None: + """"Show unlinked nodes" is filtered twice, and only one half was wired up. + + ``graphData()`` starts supplying degree-zero entities when the box is ticked, but the + engine re-filters on its own ``showUnlinked``/``minDegree`` state — which stays at the + defaults that drop exactly those entities — unless the dashboard says otherwise. + """ + report = _run_render(show_unlinked=checked) + + assert report["scope"] is not None, "the engine never learns the checkbox state" + assert report["scope"]["showUnlinked"] is checked + # minDegree matters just as much: showUnlinked alone still loses to `degree >= 1`. + assert report["scope"]["minDegree"] == (0 if checked else 1) + + +@requires_node +def test_dashboard_hands_the_engine_the_active_themes_entity_colours() -> None: + """The other half of the theme fix: the engine can only use what it is given.""" + report = _run_render() + + assert report["themeColors"] is not None, "the engine never learns the active theme" + # Resolved from the stubbed --entity-* custom properties, not from any JS constant. + assert report["themeColors"]["person_or_concept"] == "#112233" + assert report["themeColors"]["organization"] == "#556677" + assert report["themeColors"]["relation_label"] == "#123456" + assert report["themeColors"]["label"] == "#e7e9ee" + # Every type the legend can show must be covered, or the canvas falls back per type. + assert set(report["themeColors"]) == { + "person_or_concept", "mention", "hashtag", "email", "organization", "location", + "relation_label", "label", + } + + +def test_a_theme_switch_repaints_the_opt_in_canvas() -> None: + """``applyTheme()`` is the only place a theme change is observable. + + It already calls ``graphRecolor()``; that path has to reach the engine, or the canvas keeps + the previous theme until the next full graph render. + """ + source = DASHBOARD.read_text(encoding="utf-8") + assert "if(typeof graphRecolor==='function')graphRecolor()" in source + recolor = source[source.index("function graphRecolor()"):] + recolor = recolor[: recolor.index("\nfunction graphFit")] + assert "engine.setThemeColors(graphThemeTypeColors())" in recolor + + +@requires_node +def test_a_renderer_created_after_leaving_the_graph_view_is_born_paused() -> None: + """The rAF leak this PR already fixed once, reached by a different route. + + ``/graph`` and both lazy scripts resolve asynchronously. Leaving Graph before they do runs + the pause while ``GRAPH_ENGINE`` is still null, so the pending callback would create and + start a renderer against a hidden pane that nothing ever pauses again. + """ + parked = _run_render(parked=True) + assert parked["created"] == 1 + assert parked["paused"] == 1, "a renderer created off-view keeps repainting forever" + + # On the view, the same path must not park a renderer the user is looking at. + live = _run_render(parked=False) + assert live["created"] == 1 + assert live["paused"] == 0 + + +def test_leaving_the_graph_view_records_the_pause_as_well_as_applying_it() -> None: + source = DASHBOARD.read_text(encoding="utf-8") + assert "if(v==='graph')graphEngineResume();else graphEnginePause()" in source + pause = source[source.index("function graphEnginePause()"):] + pause = pause[: pause.index("\nfunction graphInvalidateData")] + assert "GRAPH_ENGINE_PARKED=true" in pause + assert "GRAPH_ENGINE_PARKED=false" in pause + + +#: Force-graph resolves each link's ``source``/``target`` from an id to the node object once it +#: owns the data, and the paint callbacks read ``.x``/``.y`` off those objects. The recording +#: stand-in stores the arrays untouched, so a test that wants to *drive* a link painter has to +#: do that resolution — and give the nodes coordinates — itself. +LAY_OUT = """ +const layOut = () => { + const data = store.graphData; + const byId = new Map(data.nodes.map(n => [n.id, n])); + data.nodes.forEach((n, i) => { n.x = i * 10; n.y = i; }); + data.links.forEach(l => { + const s = byId.get(l.source && l.source.id !== undefined ? l.source.id : l.source); + const t = byId.get(l.target && l.target.id !== undefined ? l.target.id : l.target); + if (s) l.source = s; + if (t) l.target = t; + }); + return data; +}; +let painted = []; +const linkCtx = { + font: '', fillStyle: '', textAlign: '', textBaseline: '', + fillText(text) { painted.push(String(text)); }, +}; +const paintLinks = (scale, links) => { + painted = []; + const mode = store.linkCanvasObjectMode ? store.linkCanvasObjectMode() : undefined; + const draw = store.linkCanvasObject; + if (mode === 'after' && draw) (links || store.graphData.links).forEach(l => draw(l, linkCtx, scale)); + return painted.slice(); +}; +""" + + +@requires_node +def test_relation_labels_are_painted_when_the_labels_box_is_ticked() -> None: + """**Labels** turns on two label layers on the classic path; the engine only had one. + + ``graphToggleLabels`` forwards the checkbox straight to ``setSettings({labels})``, and the + classic renderer answers it with *both* entity names and a ``linkCanvasObject`` that paints + each ``link.label``. The opt-in engine configured no link painter at all, so relation names + silently disappeared under ``?graph-engine=next`` and could only be read by hovering one + edge at a time. + """ + report = _run_engine( + LAY_OUT + + """ + const api = G.create(el, { reducedMotion: () => true }); + api.setData({ + nodes: [{ id: 'a' }, { id: 'b' }], + links: [{ source: 'a', target: 'b', layer: 'entity', label: 'mentions' }], + }); + layOut(); + const unticked = paintLinks(4); + api.setSettings({ labels: true }); + api.setThemeColors({ relation_label: '#123456' }); + const ticked = paintLinks(4); + const labelColor = linkCtx.fillStyle; + // Relation labels are the noisiest layer: they stay off until the user zooms in. + const zoomedOut = paintLinks(1); + emit({ unticked, ticked, zoomedOut, labelColor }); + """ + ) + assert report["unticked"] == [] + assert report["ticked"] == ["mentions"], "the Labels checkbox never paints relation names" + assert report["labelColor"] == "#123456", "relation labels ignore the active theme" + assert report["zoomedOut"] == [] + + +def test_entity_labels_preserve_the_configured_screen_font_size() -> None: + """The canvas transform already applies zoom; the label size compensates exactly once.""" + source = ASSET.read_text(encoding="utf-8") + assert "const size = state.settings.font / scale;" in source + assert "state.settings.font / scale / 3.4" not in source + + +@requires_node +def test_node_labels_are_capped_at_the_configured_density() -> None: + """Label density is a ranked cap, so dense graphs do bounded text work per frame.""" + report = _run_engine( + """ + let labels = []; + const ctx = { + globalAlpha: 1, fillStyle: '', strokeStyle: '', lineWidth: 1, font: '', + textBaseline: '', save() {}, restore() {}, beginPath() {}, arc() {}, stroke() {}, + fill() {}, createRadialGradient() { return { addColorStop() {} }; }, + fillText(text) { labels.push(String(text)); }, + }; + const api = G.create(el, { reducedMotion: () => true }); + api.setData(chain(20)); + api.setSettings({ labels: true, labelDensity: 3 }); + store.graphData.nodes.forEach((node, index) => { + node.x = index * 10; + node.y = 0; + store.nodeCanvasObject(node, ctx, 1); + }); + const names = labels.filter(value => value.startsWith('n')); + emit({ names, distinct: [...new Set(names)] }); + """ + ) + assert len(report["distinct"]) == 3 + assert len(report["names"]) == 6 # shadow and foreground per selected node + + +def test_collapsed_cluster_labels_use_the_active_theme_text_colour() -> None: + source = ASSET.read_text(encoding="utf-8") + cluster_start = source.index("if (node.cluster)") + cluster_paint = source[ + cluster_start : source.index("if (state.bridges", cluster_start) + ] + assert "state.themeColors.label || '#e7e9ee'" in cluster_paint + + +@requires_node +def test_normal_and_highlighted_node_labels_use_the_active_theme_text_colour() -> None: + """Entity labels remain visible on the light canvas, including the highlighted node.""" + report = _run_engine( + """ + const api = G.create(el, { reducedMotion: () => true }); + api.setData({ + nodes: [{ id: 'a', name: 'Alpha', degree: 20 }, { id: 'b', name: 'Beta' }], + links: [{ source: 'a', target: 'b' }], + }); + const node = store.graphData.nodes[0]; + node.x = 1; node.y = 2; + api.setSettings({ labels: true, labelDensity: 100 }); + api.setThemeColors({ label: '#123456' }); + const painted = []; + const ctx = new Proxy({ + fillStyle: '', + fillText(text) { painted.push({ text: String(text), color: this.fillStyle }); }, + }, { + get(target, name) { + if (name in target) return target[name]; + return () => {}; + }, + }); + store.nodeCanvasObject(node, ctx, 1); + const normal = painted[painted.length - 1]; + api.setHighlight('a'); + painted.length = 0; + store.nodeCanvasObject(node, ctx, 1); + emit({ normal, highlighted: painted[painted.length - 1] }); + """ + ) + assert report["normal"]["color"] == "#123456" + assert report["highlighted"]["color"] == "#123456" + + +@requires_node +def test_unfreezing_releases_drag_pins_even_with_reduced_motion() -> None: + report = _run_engine( + """ + const api = G.create(el, { reducedMotion: () => true }); + api.setData({ + nodes: [{ id: 'a' }, { id: 'b' }], + links: [{ source: 'a', target: 'b' }], + }); + const node = store.graphData.nodes[0]; + node.x = 10; node.y = 20; + store.onNodeDragEnd(node); + const pinned = node.fx === 10 && node.fy === 20; + api.freeze(true); + api.freeze(false); + emit({ + pinned, + released: node.fx === undefined && node.fy === undefined, + reheats: invocations.d3ReheatSimulation || 0, + }); + """ + ) + assert report == {"pinned": True, "released": True, "reheats": 0} + + +@requires_node +def test_focusing_an_entity_the_canvas_is_not_showing_does_not_report_success() -> None: + """``zoomToNode`` is the dashboard's visibility oracle, and it was answering from memory. + + ``graphFocus`` treats ``false`` as "offer the recovery path" — tick *Show unlinked*, retry, + and otherwise say *Entity not in view*. The engine answered from ``raw.nodes``, which keeps + the coordinates force-graph left on a node from an earlier render, so a node hidden by the + auto-collapsed view (only ``cluster-*`` bubbles are drawn below zoom 0.55) or by a scope + filter still reported success — the camera moved to nothing and the user got no explanation. + """ + report = _run_engine( + """ + const collapses = []; + const api = G.create(el, { + reducedMotion: () => true, onCollapseChange: value => collapses.push(value), + }); + api.setData({ + nodes: [{ id: 'a' }, { id: 'b' }, { id: 'c' }, { id: 'lonely' }], + links: [{ source: 'a', target: 'b' }, { source: 'b', target: 'c' }], + }); + const shownIds = () => (store.graphData.nodes || []).map(n => n.id); + // Everything visible once, so every entity carries real coordinates from here on. + api.setScope({ showUnlinked: true, minDegree: 0 }); + store.graphData.nodes.forEach((n, i) => { n.x = i * 10; n.y = i; }); + + // 1. Hidden by the scope filter, but still remembered with valid coordinates. + api.setScope({ showUnlinked: false, minDegree: 1 }); + const filtered = { found: api.zoomToNode('lonely'), shown: shownIds() }; + + // 2. Hidden by the collapsed view, which paints cluster bubbles instead of entities. + api.setCollapse(true); + const whileCollapsed = shownIds(); + const focused = api.zoomToNode('c'); + emit({ + filtered, whileCollapsed, focused, collapses, + afterFocus: shownIds(), collapsed: api.state().collapsed, + }); + """ + ) + # A filtered-out entity is not in view, so the dashboard must be told to recover. + assert report["filtered"]["found"] is False, "a filtered-out entity reported as visible" + assert "lonely" not in report["filtered"]["shown"] + # A collapsed view really is showing only bubbles... + assert report["whileCollapsed"] == ["cluster-0"] + # ...so focusing a named entity has to expand it rather than claim it is already on screen. + assert report["focused"] is True + assert report["collapsed"] is False + assert "c" in report["afterFocus"], "the entity is still not on the canvas" + assert report["collapses"][-1] is False, "the dashboard was never told the view expanded" + + +@requires_node +def test_appearance_only_changes_do_not_restart_the_layout() -> None: + """Style, Color by, Labels and Flow repaint the graph; they must not re-run it. + + ``visible()`` allocates fresh arrays on every call, and force-graph treats any ``graphData`` + call as a data update: it re-copies the nodes and d3 resets the simulation alpha to 1. So + every appearance-only setter threw the settled layout away and made the whole graph move. + The classic renderer guards the same seed with ``if(dataChanged)FG.graphData(data)``. + """ + report = _run_engine( + """ + const api = G.create(el, { reducedMotion: () => true }); + const nodes = [{ id: 'lonely', etype: 'organization' }], links = []; + for (let i = 0; i < 12; i++) nodes.push({ id: 'n' + i, etype: 'person_or_concept' }); + for (let i = 0; i < 11; i++) links.push({ source: 'n' + i, target: 'n' + (i + 1) }); + api.setData({ nodes, links }); + const seeded = calls.graphData; + const before = store.graphData.nodes[0].color; + const repaintsBefore = calls.nodeCanvasObject; + + api.setStyle('galaxy'); + api.setColorBy('type'); + api.setSettings({ labels: true }); + api.setSettings({ flow: false }); + const paintOnly = calls.graphData; + const recoloured = store.graphData.nodes[0].color; + const repaintsAfter = calls.nodeCanvasObject; + + // A genuine change to the visible set still has to reach force-graph. + api.setScope({ showUnlinked: true, minDegree: 0 }); + emit({ + seeded, paintOnly, afterScope: calls.graphData, before, recoloured, + repaintsBefore, repaintsAfter, shown: store.graphData.nodes.length, + }); + """ + ) + assert report["paintOnly"] == report["seeded"], "an appearance change restarted the layout" + assert report["afterScope"] > report["seeded"], "a real view change never reached the canvas" + assert report["shown"] == 13 + # Skipping the reseed must not mean skipping the paint. + assert report["recoloured"] != report["before"] + assert report["repaintsAfter"] > report["repaintsBefore"] + + +@requires_node +def test_simulation_time_is_bounded_on_a_large_graph() -> None: + """force-graph's default cooldown is 15 seconds; nothing here was overriding it. + + The classic path caps a large graph at 1.1s / 80 ticks precisely because running the layout + — and therefore repainting every node and link — for the full default window is what makes a + big store feel broken on load and after every reheat. + """ + report = _run_engine( + """ + const api = G.create(el, {}); + api.setData(chain(40)); + const small = { + time: store.cooldownTime, ticks: store.cooldownTicks, warmup: store.warmupTicks, + alpha: store.d3AlphaDecay, velocity: store.d3VelocityDecay, + }; + // 3001 entities / 3000 relations — past the classic renderer's 600-node signal. + api.setData(chain(3000)); + const big = { + time: store.cooldownTime, ticks: store.cooldownTicks, warmup: store.warmupTicks, + alpha: store.d3AlphaDecay, velocity: store.d3VelocityDecay, + }; + const still = G.create(el, { reducedMotion: () => true }); + still.setData(chain(40)); + emit({ + small, big, + reduced: { time: store.cooldownTime, ticks: store.cooldownTicks }, + }); + """ + ) + assert report["small"]["time"] == 2200 + assert report["small"]["ticks"] == 160 + # The number this guards: the vendor default left a 3k-relation store simulating for 15s. + assert report["big"]["time"] == 1100 + assert report["big"]["ticks"] == 80 + assert report["big"]["warmup"] == 18 + # A large graph also settles harder, exactly as GPERF.large does on the classic path. + assert report["big"]["alpha"] > report["small"]["alpha"] + assert report["big"]["velocity"] > report["small"]["velocity"] + # Reduced motion asks for a static layout, not a shorter animation. + assert report["reduced"]["time"] == 0 + assert report["reduced"]["ticks"] == 1 + + +@requires_node +def test_physics_sliders_reheat_the_simulation_the_way_the_classic_renderer_does() -> None: + """Installing a new force on a settled graph moves nothing without a reheat. + + ``graphSet`` (dashboard.js) routes Repel/Link/Gravity/Size/Font/Link-width/Label-density + through ``setSettings`` under ``?graph-engine=next``. The classic branch of that same + function treats ``repel|link|gravity|size`` as *layout* changes: it re-applies the forces + and then reheats unless the user asked for reduced motion. The engine's ``applyForces()`` + only swaps the charge/link/forceX-forceY/collide values into the running simulation — and a + settled graph sits at alpha~0 — so without the reheat those four sliders are inert until + the user finds the Reheat button. The paint-only settings must *not* reheat: restarting + the layout because a label got bigger throws away the arrangement the user is reading. + """ + report = _run_engine( + """ + const reheats = () => invocations.d3ReheatSimulation || 0; + const bump = (api, patch) => { const before = reheats(); api.setSettings(patch); return reheats() - before; }; + + const api = G.create(el, {}); + api.setData(chain(40)); + const layout = { + repel: bump(api, { repel: 260 }), + link: bump(api, { link: 90 }), + gravity: bump(api, { gravity: 12 }), + size: bump(api, { size: 5 }), + mode: bump(api, { mode: 'radial' }), + }; + const paint = { + font: bump(api, { font: 11 }), + linkw: bump(api, { linkw: 2.4 }), + labelDensity: bump(api, { labelDensity: 40 }), + labels: bump(api, { labels: true }), + flow: bump(api, { flow: false }), + }; + + // The classic path's `if(layout&&!prefersReducedMotion())` exemption. + const still = G.create(el, { reducedMotion: () => true }); + still.setData(chain(40)); + const reducedMotion = bump(still, { repel: 260 }); + emit({ layout, paint, reducedMotion }); + """ + ) + # The four sliders the classic renderer calls a layout change, plus the preset itself. + assert report["layout"] == { + "repel": 1, "link": 1, "gravity": 1, "size": 1, "mode": 1 + }, "a physics slider installed new forces on a settled graph and nothing moved" + # Appearance-only settings keep the arrangement the user is looking at. + assert report["paint"] == { + "font": 0, "linkw": 0, "labelDensity": 0, "labels": 0, "flow": 0 + }, "an appearance change restarted the layout" + assert report["reducedMotion"] == 0, "reduced motion still got an animated relayout" + + +@requires_node +def test_curves_arrows_and_relation_labels_are_dropped_on_a_dense_graph() -> None: + """Three per-edge costs the classic path turns off past ``GPERF.dense`` (links > 1500). + + A curved link is a quadratic bezier instead of a straight line, an arrowhead is a filled + triangle, and a relation label is a text layout — each per relation, each every frame. At + this density they are unreadable anyway, so the classic renderer pays for none of them. + """ + report = _run_engine( + LAY_OUT + + """ + const api = G.create(el, { reducedMotion: () => true }); + api.setSettings({ labels: true }); + + api.setData(chain(1500)); + const atLimit = { + curve: store.linkCurvature, arrow: store.linkDirectionalArrowLength, + }; + + api.setData(chain(1501)); + const overLimit = { + curve: store.linkCurvature, arrow: store.linkDirectionalArrowLength, + }; + // One laid-out relation is enough to drive the label painter at this size. + const data = layOut(); + data.links[0].label = 'mentions'; + const denseUnhighlighted = paintLinks(4, [data.links[0]]); + store.onNodeHover(data.nodes[0]); + const denseHighlighted = paintLinks(4, [data.links[0]]); + emit({ atLimit, overLimit, denseUnhighlighted, denseHighlighted }); + """ + ) + # 1500 links is the classic threshold itself, so nothing is dropped yet. + assert report["atLimit"]["curve"] == 0.12 + assert report["atLimit"]["arrow"] == 2.5 + assert report["overLimit"]["curve"] == 0 + assert report["overLimit"]["arrow"] == 0 + # Relation labels come back for the one neighbourhood the user is actually pointing at. + assert report["denseUnhighlighted"] == [] + assert report["denseHighlighted"] == ["mentions"] + + +#: A ``d3`` stand-in for the force constructors ``applyForces()`` reaches for. The asset reads +#: ``d3`` as a free variable, so assigning it on ``globalThis`` is what the browser's global +#: script tag does; without it ``applyForces()`` returns before it ever configures collision. +D3_STUB = """ +let collide = null; +globalThis.d3 = { + forceX: () => ({ strength: () => ({}) }), + forceY: () => ({ strength: () => ({}) }), + forceRadial: () => ({ strength: () => ({}) }), + forceCollide: radius => ({ radius, iterations(n) { collide = { radius, iterations: n }; return this; } }), +}; +""" + + +@requires_node +def test_collision_runs_one_pass_on_a_large_graph_like_the_classic_renderer() -> None: + """``forceCollide().iterations(2)`` is a second full quadtree traversal per node per tick. + + ``graphApplyForces()`` on the classic path spends it only when it is affordable + (``.iterations(GPERF.large?1:2)``). The opt-in engine computes the same ``large`` signal for + its cooldown and alpha-decay constants but was pinning two iterations regardless, so the one + case where the extra pass hurts most — the initial layout and every reheat of a big store — + was the case that paid for it twice over. + """ + report = _run_engine( + D3_STUB + + """ + const api = G.create(el, { reducedMotion: () => true }); + + api.setData(chain(40)); + const small = collide.iterations; + + // 601 entities / 600 relations — one past the classic renderer's 600-node cutoff. + api.setData(chain(600)); + const big = collide.iterations; + + // A slider move re-runs applyForces() on the running simulation; it must not undo this. + api.setSettings({ repel: 90 }); + const afterSlider = collide.iterations; + emit({ small, big, afterSlider, radiusIsAFunction: typeof collide.radius === 'function' }); + """ + ) + assert report["small"] == 2 + assert report["big"] == 1, "a large graph still runs two collision passes per tick" + assert report["afterSlider"] == 1, "a slider move restored the expensive collision pass" + # Guards the whole call rather than the argument in isolation: a per-node radius, not a + # constant, is what makes collision agree with the sizes the renderer actually painted. + assert report["radiusIsAFunction"] is True + + +#: Counts the two per-node glow primitives independently. ``createRadialGradient`` is the halo +#: and the solar sphere shading; ``shadowBlur`` is the cyber bloom. Both are per node, per frame. +GLOW_CANVAS_STUB = """ +let gradients = 0, blurs = 0, fills = 0; +const ctx = { + globalAlpha: 1, globalCompositeOperation: '', strokeStyle: '', lineWidth: 1, font: '', + textBaseline: '', shadowColor: '', + set shadowBlur(v) { if (v) blurs += 1; }, + get shadowBlur() { return 0; }, + set fillStyle(v) {}, get fillStyle() { return ''; }, + save() {}, restore() {}, beginPath() {}, arc() {}, ellipse() {}, stroke() {}, + setLineDash() {}, fillText() {}, + fill() { fills += 1; }, + createRadialGradient() { gradients += 1; return { addColorStop() {} }; }, +}; +const paintNodes = () => { + gradients = 0; blurs = 0; fills = 0; + const draw = store.nodeCanvasObject; + store.graphData.nodes.forEach((n, i) => { n.x = i * 10; n.y = i; draw(n, ctx, 4); }); + return { gradients, blurs, fills }; +}; +""" + + +@requires_node +@pytest.mark.parametrize("style", ["cyber", "galaxy", "solar"]) +def test_per_node_glow_is_dropped_on_a_large_graph(style: str) -> None: + """Every ``rich`` node was getting a bloom or a gradient on every frame, at any size. + + The classic renderer gates all three of them on ``!GPERF.large`` — the galaxy halo, the solar + corona *and* its sphere shading, and the cyber ``shadowBlur``. A shadow blur is a full + convolution of the drawn shape and a radial gradient is a fresh object per node; at the + >600-node cutoff that is hundreds of them rebuilt per tick, on top of the layout, which is + what made a dense workspace crawl even after the other large-graph optimisations kicked in. + + ``fills`` is the control: the nodes are still being drawn, so a zero glow count means the + effect was skipped, not that the paint never ran. + """ + report = _run_engine( + GLOW_CANVAS_STUB + + f""" + const api = G.create(el, {{ reducedMotion: () => true }}); + api.setStyle("{style}"); + + api.setData(chain(40)); + const small = paintNodes(); + + api.setData(chain(600)); + const big = paintNodes(); + emit({{ small, big }}); + """ + ) + small, big = report["small"], report["big"] + assert small["fills"] > 0 and big["fills"] > 0, "canvas stub never reached the node painter" + assert small["gradients"] + small["blurs"] > 0, "the small graph lost its glow entirely" + assert big["gradients"] == 0, f"{style} still builds a radial gradient per node when large" + assert big["blurs"] == 0, f"{style} still shadow-blurs every node when large" + + +def _community_palettes(source: str) -> dict: + """Parse a ``COMMUNITY_PALS`` literal out of either renderer.""" + # Anchor on the declaration: both files also name the table in prose comments. + match = re.search(r"COMMUNITY_PALS\s*=\s*\{", source) + assert match is not None, "COMMUNITY_PALS is not declared here" + block = source[match.end():source.index("};", match.end())] + return { + name: re.findall(r"#[0-9a-fA-F]{3,8}", body) + for name, body in re.findall(r"(\w+)\s*:\s*\[([^\]]*)\]", block) + } + + +def test_community_colours_match_the_dashboard_and_the_legend_swatches() -> None: + """The cluster legend is painted from CSS, so palette *order* is a contract, not a taste. + + ``graphRenderLegend`` sorts communities by size and gives the largest a + ``.graph-cluster-0`` swatch, while the canvas colours that same community with palette slot + 0. The swatch colours live in ``dashboard.css`` and encode the Cyber palette — the default + style — so a renderer whose slot 0 is a different colour makes the legend describe cluster 1 + with cluster 2's colour, on the default style, for every workspace. + """ + engine = _community_palettes(ASSET.read_text(encoding="utf-8")) + classic = _community_palettes(DASHBOARD.read_text(encoding="utf-8")) + assert engine, "COMMUNITY_PALS could not be parsed out of the engine" + assert engine == classic, "the opt-in renderer paints communities a different colour" + + swatches = dict( + re.findall(r"\.graph-cluster-(\d+)\{background:(#[0-9a-fA-F]{3,8})\}", + CSS.read_text(encoding="utf-8")) + ) + assert swatches, "the cluster legend swatches are missing from the stylesheet" + for index, colour in sorted(swatches.items()): + assert engine["cyber"][int(index)].lower() == colour.lower(), ( + f"legend swatch {index} does not match the canvas colour for that cluster" + ) + + +# ── CSP, styling and lifecycle ────────────────────────────────────────────────────── + + +def test_pane_backgrounds_are_owned_by_css_not_by_the_asset() -> None: + """``style-src-attr 'none'`` forbids writing these onto the element.""" + css = CSS.read_text(encoding="utf-8") + source = ASSET.read_text(encoding="utf-8") + for style in ("galaxy", "solar", "cyber"): + assert f'#graph-net[data-graph-style="{style}"]' in css + assert "data-graph-style" in source + # The gradients must exist in exactly one place, or the two copies drift. + assert "radial-gradient" not in source + assert "linear-gradient" not in source + + +def test_hover_cursor_class_the_asset_toggles_exists_in_css() -> None: + css = CSS.read_text(encoding="utf-8") + source = ASSET.read_text(encoding="utf-8") + assert "engraphis-graph-node-hover" in source + assert ".engraphis-graph-node-hover" in css + + +def test_csp_gate_covers_the_graph_asset() -> None: + from scripts.externalize_dashboard_assets import EXTRA_SCRIPTS, check + + assert ASSET in EXTRA_SCRIPTS, "the graph engine must be inside the CSP drift gate" + check() + + +def test_engine_exposes_a_teardown_and_the_dashboard_drives_it() -> None: + source = ASSET.read_text(encoding="utf-8") + dashboard = DASHBOARD.read_text(encoding="utf-8") + for member in ("api.destroy", "api.pause", "api.resume", "api.resize"): + assert member in source + # force-graph keeps a rAF alive while resumed; leaving the view must park it. + assert "if(v==='graph')graphEngineResume();else graphEnginePause()" in dashboard + assert "GRAPH_ENGINE.destroy()" in dashboard + + +def test_reduced_motion_is_honoured_by_the_opt_in_renderer() -> None: + source = ASSET.read_text(encoding="utf-8") + dashboard = DASHBOARD.read_text(encoding="utf-8") + assert "prefers-reduced-motion: reduce" in source + assert "opts.reducedMotion" in source + assert "reducedMotion:prefersReducedMotion" in dashboard + + +def test_graph_engine_is_syntactically_valid_when_node_is_installed() -> None: + if NODE is None: + pytest.skip("node is not installed") + result = subprocess.run( + [NODE, "--check", str(ASSET)], + cwd=ROOT, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr diff --git a/tests/test_hosted_plan_resolution.py b/tests/test_hosted_plan_resolution.py index a867f36..45900e8 100644 --- a/tests/test_hosted_plan_resolution.py +++ b/tests/test_hosted_plan_resolution.py @@ -22,6 +22,7 @@ from __future__ import annotations import ast +import calendar import io import json import os @@ -40,7 +41,7 @@ # install. Skip rather than error at collection, matching the rest of the suite. pytest.importorskip("fastapi", reason="full-stack extra not installed") -from engraphis import cloud_session # noqa: E402 +from engraphis import cloud_session, hosted_client # noqa: E402 from engraphis.routes import v2_api # noqa: E402 REPO_ROOT = Path(__file__).resolve().parents[1] @@ -56,21 +57,68 @@ "pro": ["analytics", "automation", "sync"], "team": ["analytics", "automation", "sync", "team"], } +# The trial disclosure this client now depends on, mirrored from (read-only) +# engraphis-cloud ``EntitlementDTO`` and ``DeviceRegistrationResponse``. Both surfaces +# carry the same names on purpose. Renaming any of them server-side reverts this client to +# "not a trial, none consumed" in silence -- which is precisely the state this suite exists +# to keep it out of -- so the drift has to fail here instead. +SERVER_TRIAL_FIELDS = ("status", "is_trial", "trial_consumed", "trial_ends_at") + + +def test_license_route_emits_plan_neutral_account_url(monkeypatch) -> None: + """Billing/account actions must not silently inherit the Pro checkout.""" + + monkeypatch.setenv("ENGRAPHIS_CLOUD_PLAN", "pro") + monkeypatch.setenv("ENGRAPHIS_UPGRADE_URL", "https://cloud.test/account") + monkeypatch.setenv("ENGRAPHIS_PRO_UPGRADE_URL", "https://cloud.test/checkout/pro") + monkeypatch.setenv("ENGRAPHIS_TEAM_UPGRADE_URL", "https://cloud.test/checkout/team") + + payload = v2_api.get_license() + + assert payload["account_url"] == "https://cloud.test/account" + assert payload["pro_upgrade_url"] == "https://cloud.test/checkout/pro" + assert payload["team_upgrade_url"] == "https://cloud.test/checkout/team" + assert payload["upgrade_url"] == "https://cloud.test/checkout/pro" + + monkeypatch.delenv("ENGRAPHIS_UPGRADE_URL") + assert v2_api.get_license()["account_url"] == hosted_client.DEFAULT_CLOUD_URL + + +#: A three-day trial that is still running, and one that is not, as the control plane +#: serializes them (``engraphis-cloud`` ``EntitlementDTO``; Pydantic emits a trailing ``Z`` +#: for UTC, which this client's Python 3.9 floor cannot hand straight to +#: ``datetime.fromisoformat``). +# Far enough ahead that tests for a live server disclosure do not depend on wall-clock +# time. Expiry itself is covered with an explicitly frozen boundary below. +TRIAL_ENDS_AT = "2099-07-28T12:00:00Z" + + +def _epoch(iso: str) -> float: + """Independent reference conversion, so the assertions do not use the code under test.""" + + return calendar.timegm(time.strptime(iso.rstrip("Zz"), "%Y-%m-%dT%H:%M:%S")) def _entitlement_dto(plan: str, *, active: bool = True, - organization_id: str = ORGANIZATION) -> dict: + organization_id: str = ORGANIZATION, + is_trial: bool = False, trial_consumed: bool = False, + status: str = "") -> dict: return { "organization_id": organization_id, "entitlement_id": "ent_1", "plan": plan, - "status": "active" if active else "past_due", + "status": status or ("trialing" if is_trial and active + else "active" if active else "past_due"), "cloud_access_active": active, "cloud_features": SERVER_PLAN_FEATURES[plan] if active else [], "seat_limit": 5, "seat_assignment_basis": "named_members", "starts_at": "2026-01-01T00:00:00Z", "expires_at": None, + "is_trial": is_trial, + "trial_consumed": trial_consumed or is_trial, + "trial_duration_seconds": 259_200 if is_trial else None, + "trial_ends_at": TRIAL_ENDS_AT if is_trial else None, "version": 3, } @@ -986,6 +1034,32 @@ def _lapsed(*_args, **_kwargs): assert cached["plan"] == "team", "the lapsed plan is still named for the UI" +def test_a_compatibility_entitlement_402_settles_paid_access(monkeypatch) -> None: + """An old control plane can deny only the fallback entitlement request. + + It refreshes tokens but puts no entitlement fields on those responses, so the GET + compatibility path owns the cached answer. Its 402 is just as authoritative as a + token-refresh 402; treating it as a transport failure leaves stale paid features live. + """ + + _connect(monkeypatch, pinned_token=False) + cloud = _FakeControlPlane(_entitlement_dto("team"), registration={}) + _serve(monkeypatch, cloud) + assert _settled_license(monkeypatch)["cloud_access_active"] is True + assert cloud_session.saved_entitlement() == {} + + cloud.error = urllib.error.HTTPError( + CONTROL_URL, 402, "payment required", {}, io.BytesIO(b"{}") + ) + assert v2_api._fetch_authoritative_entitlement() is None + + cached = v2_api._read_entitlement_cache() + assert cached["cloud_access_active"] is False + assert cached["features"] == [] + assert cached["plan"] == "team" + assert v2_api.get_license()["cloud_access_active"] is False + + def _age_session_clock(seconds: float = 3600.0) -> None: """Push the saved session's entitlement stamp past any refresh interval.""" @@ -1516,7 +1590,367 @@ def test_a_replaced_hosted_plan_still_overrides_the_resolved_entitlement( assert "team" in upgraded["features"] -# ── (7) the shipped modules must still build on the supported floor ─────────── +# ── (7) the trial the control plane owns, rendered by the client that never read it ── +# +# ``/api/license`` hardcoded ``"is_trial": False`` and ``"trial": {"used": False, ...}``. +# Those were the only assignments in the file, which made two customer-visible states +# unreachable and one permanently wrong: +# +# * the dashboard's ``'TRIAL'`` badge branch could never be taken, so a trialist saw the +# same confident PRO/TEAM badge a subscriber sees; +# * ``trial.used`` never became true, so "Start hosted Pro/Team trial" was offered to +# everyone forever - including paying subscribers and customers whose trial was already +# spent, both of whom the control plane answers ``TrialAlreadyConsumedError`` for; and +# * ``cloud_access_active`` was computed, returned, and rendered nowhere, so a lapsed or +# expired customer got a PRO badge with every feature row locked and no explanation. + + +def test_a_live_trial_is_reported_as_a_trial(monkeypatch) -> None: + """The badge branch the dashboard has always had, finally reachable.""" + + _connect(monkeypatch) + _serve(monkeypatch, _FakeControlPlane(_entitlement_dto("pro", is_trial=True))) + + payload = _settled_license(monkeypatch) + + assert payload["is_trial"] is True + assert payload["access_state"] == "trial" + assert payload["entitlement_status"] == "trialing" + assert payload["cloud_access_active"] is True + assert payload["trial"]["active"] is True + assert payload["trial"]["used"] is True + assert payload["trial"]["ends_at"] == pytest.approx( + _epoch(TRIAL_ENDS_AT) + ) + # A live trial is not an offer to start another one. + assert payload["trial"]["available"] is False + # A trialist still has the paid feature set while the trial runs. + assert set(payload["features"]) >= {"analytics", "automation"} + + +def test_an_offline_cached_trial_expires_at_its_known_boundary(monkeypatch) -> None: + """A stale positive access flag cannot extend a trial while the client is offline.""" + + boundary = 1_800_000_000.0 + entitlement = { + "plan": "pro", + "features": ["analytics", "automation"], + "source": "session", + "cloud_access_active": True, + "checked_at": boundary - 60, + "status": "trialing", + "is_trial": True, + "trial_consumed": True, + "trial_ends_at": boundary, + } + monkeypatch.setattr(v2_api, "_plan_entitlement", lambda: dict(entitlement)) + monkeypatch.setattr(v2_api.time, "time", lambda: boundary + 1) + + payload = v2_api.get_license() + + assert payload["access_state"] == "trial_expired" + assert payload["cloud_access_active"] is False + assert payload["features"] == [] + assert payload["trial"]["active"] is False + assert payload["trial"]["ends_at"] == boundary + + +def test_a_spent_trial_is_told_apart_from_a_lapsed_subscription(monkeypatch) -> None: + """Both read ``cloud_access_active=false``; only ``is_trial`` separates the copy.""" + + _connect(monkeypatch) + _serve( + monkeypatch, + _FakeControlPlane(_entitlement_dto("pro", active=False, is_trial=True)), + ) + + payload = _settled_license(monkeypatch) + + assert payload["access_state"] == "trial_expired" + assert payload["is_trial"] is True + assert payload["trial"]["used"] is True + assert payload["trial"]["active"] is False + assert payload["trial"]["available"] is False + assert payload["features"] == [] + + +def test_a_lapsed_subscription_is_never_offered_another_trial(monkeypatch) -> None: + """A customer whose subscription stopped needs billing, not a 409 from the server.""" + + _connect(monkeypatch) + _serve( + monkeypatch, + _FakeControlPlane( + _entitlement_dto("team", active=False, trial_consumed=True) + ), + ) + + payload = _settled_license(monkeypatch) + + assert payload["access_state"] == "lapsed" + assert payload["is_trial"] is False + assert payload["trial"]["available"] is False + assert payload["plan"] == "team" + assert payload["features"] == [] + + +def test_a_paying_customer_is_never_offered_a_trial(monkeypatch) -> None: + """``start_trial`` refuses any organization that already holds an entitlement. + + That includes a customer who bought outright and never trialled, so the offer has to + be suppressed by the *connection*, not only by ``trial_consumed``. + """ + + _connect(monkeypatch) + _serve(monkeypatch, _FakeControlPlane(_entitlement_dto("team"))) + + payload = _settled_license(monkeypatch) + + assert payload["access_state"] == "active" + assert payload["is_trial"] is False + assert payload["trial"]["used"] is False + assert payload["trial"]["available"] is False + # No trial boundary is replayed at a paying customer. + assert payload["trial"]["ends_at"] == 0.0 + + +def test_an_unconnected_installation_is_the_one_place_a_trial_is_offered() -> None: + """The acquisition CTA must survive: an installation with no organization can trial.""" + + payload = v2_api.get_license() + + assert payload["plan"] == "local" + assert payload["access_state"] == "inactive" + assert payload["trial"]["available"] is True + assert payload["trial"]["used"] is False + + +def test_a_connected_but_unanswered_installation_offers_no_trial(monkeypatch) -> None: + """First boot after onboarding: the plan is inferred, the trial must not be. + + A connected installation always belongs to an organization, and that organization is + trialling, has spent its trial, or is paying. Guessing "no trial consumed" here is how + the button that can only return 409 gets drawn during the one window nothing has + answered yet. + """ + + monkeypatch.delenv("ENGRAPHIS_CLOUD_CONTROL_URL", raising=False) + monkeypatch.setattr(cloud_session, "configured", lambda **kw: True) + monkeypatch.setattr(v2_api, "_session_entitlement", dict) + monkeypatch.setattr(v2_api, "_read_entitlement_cache", dict) + + payload = v2_api.get_license() + + assert payload["plan"] == "pro" + assert payload["plan_source"] == "connected" + assert payload["trial"]["available"] is False + assert payload["trial"]["used"] is True + + +def test_the_trial_arrives_on_the_handshake_the_client_already_makes(monkeypatch) -> None: + """The primary path: no extra request, correct on the first boot after onboarding.""" + + _connect(monkeypatch, pinned_token=False) + cloud = _FakeControlPlane(registration={ + "plan": "team", + "cloud_access_active": True, + "cloud_features": SERVER_PLAN_FEATURES["team"], + "status": "trialing", + "is_trial": True, + "trial_consumed": True, + "trial_ends_at": TRIAL_ENDS_AT, + }) + _serve(monkeypatch, cloud) + + payload = _settled_license(monkeypatch) + + assert payload["plan_source"] == "session" + assert payload["access_state"] == "trial" + assert payload["trial"]["ends_at"] == pytest.approx(_epoch(TRIAL_ENDS_AT)) + # Nothing dialled the entitlements route: the handshake already answered. + assert _entitlement_reads(cloud) == [] + + +def test_a_billing_denial_turns_a_running_trial_into_an_expired_one(monkeypatch) -> None: + """A 402 is authoritative. The trial facts survive it; the stale status does not.""" + + _connect(monkeypatch, pinned_token=False) + cloud = _FakeControlPlane(registration={ + "plan": "pro", + "cloud_access_active": True, + "cloud_features": SERVER_PLAN_FEATURES["pro"], + "status": "trialing", + "is_trial": True, + "trial_consumed": True, + "trial_ends_at": TRIAL_ENDS_AT, + }) + _serve(monkeypatch, cloud) + assert _settled_license(monkeypatch)["access_state"] == "trial" + + assert cloud_session.record_billing_denial() is True + payload = v2_api.get_license() + + assert payload["access_state"] == "trial_expired" + assert payload["is_trial"] is True + assert payload["trial"]["used"] is True + assert payload["trial"]["available"] is False + assert payload["features"] == [] + # The last status the cloud named now contradicts the denial and is not rendered. + assert payload["entitlement_status"] == "" + + +def test_a_control_plane_without_the_trial_fields_claims_no_trial(monkeypatch) -> None: + """An older control plane omits them; the client must not invent a trial.""" + + _connect(monkeypatch) + legacy = _entitlement_dto("pro") + for field in ("is_trial", "trial_consumed", "trial_duration_seconds", + "trial_ends_at", "status"): + legacy.pop(field) + _serve(monkeypatch, _FakeControlPlane(legacy)) + + payload = _settled_license(monkeypatch) + + assert payload["plan"] == "pro" + assert payload["is_trial"] is False + assert payload["access_state"] == "active" + assert payload["entitlement_status"] == "" + assert payload["trial"]["ends_at"] == 0.0 + + +def test_the_trial_disclosure_survives_a_restart(monkeypatch, tmp_path) -> None: + """The compatibility cache round trips the trial through its own parser.""" + + monkeypatch.setenv("ENGRAPHIS_STATE_DIR", str(tmp_path)) + _connect(monkeypatch) + _serve( + monkeypatch, + _FakeControlPlane(_entitlement_dto("team", is_trial=True)), + ) + assert _settled_license(monkeypatch)["access_state"] == "trial" + + reread = v2_api._read_entitlement_cache() + + assert reread["is_trial"] is True + assert reread["trial_consumed"] is True + assert reread["status"] == "trialing" + assert reread["trial_ends_at"] == pytest.approx(_epoch(TRIAL_ENDS_AT)) + + +@pytest.mark.parametrize("value", [ + "2099-07-28T12:00:00Z", + "2099-07-28T12:00:00z", + "2099-07-28T12:00:00+00:00", +]) +def test_the_servers_utc_timestamp_is_read_on_the_python_39_floor(value) -> None: + """``datetime.fromisoformat`` on 3.9 rejects the ``Z`` Pydantic emits for UTC.""" + + assert v2_api._epoch_seconds(value) == pytest.approx(_epoch(TRIAL_ENDS_AT)) + + +@pytest.mark.parametrize("value", [ + None, "", " ", "not-a-date", 0, -1, True, [], {}, + float("inf"), float("-inf"), float("nan"), 10**400, + "2026-13-45T99:99:99Z", +]) +def test_an_unreadable_trial_boundary_is_unknown_rather_than_raising(value) -> None: + """This runs on the ``/api/bootstrap`` boot path; nothing here may raise.""" + + assert v2_api._epoch_seconds(value) == 0.0 + + +def test_a_forged_entitlement_status_is_not_rendered() -> None: + """Only the control plane's own status vocabulary reaches the panel's copy.""" + + assert v2_api._normalized_status("PAST_DUE") == "past_due" + for value in ("", None, 7, "your account is fine", "active"): + assert v2_api._normalized_status(value) == "" + + +def test_every_access_state_the_dashboard_renders_is_one_the_client_can_produce() -> None: + """Close the loop on the shipped JS: an unhandled state would render as a blank badge.""" + + script = DASHBOARD_JS.read_text(encoding="utf-8") + block = script[script.index("function licAccessState()"):] + handled = set(re.findall(r"s==='([a-z_]+)'", block[:block.index("\n")])) + + assert handled | {"inactive"} == set(v2_api._ACCESS_STATES) + + produced = set() + for plan in ("local", "pro", "team"): + for active in (True, False): + for is_trial in (True, False): + produced.add(v2_api._access_state({ + "plan": plan, "cloud_access_active": active, "is_trial": is_trial, + })) + assert produced == set(v2_api._ACCESS_STATES) + + +def test_the_dashboard_never_offers_a_trial_it_was_not_told_is_available() -> None: + """The one gate on every "Start trial" affordance in the shipped asset. + + ``trial.used`` was hardcoded false, so every one of these drew a button whose only + outcome, for a connected customer, was a ``TrialAlreadyConsumedError``. + """ + + script = DASHBOARD_JS.read_text(encoding="utf-8") + + assert "function licTrialAvailable(){return !!(LIC&&LIC.trial&&LIC.trial.available)}" \ + in script + # No affordance may still be gated on the old always-false flag. + assert "LIC.trial.used" not in script + for marker in ("Start hosted Pro trial", "Start hosted Team trial", + "Start exactly '+TRIAL_DAYS+' days free"): + index = script.index(marker) + window = script[max(0, index - 400):index] + assert "licTrialAvailable()" in window, marker + + +def test_the_dashboard_explains_a_locked_state_instead_of_badging_it() -> None: + """A PRO badge over rows of locks with no reason given is the defect itself.""" + + script = DASHBOARD_JS.read_text(encoding="utf-8") + banner = script[script.index("function licStateBanner("):] + banner = banner[:banner.index("\nfunction licActionsHtml")] + + for state in ("trial_expired", "lapsed", "inactive"): + assert "state==='%s'" % state in banner, state + # Each locked state says what happened and what is still safe. + assert "free trial has ended" in banner + assert "no longer active" in banner + assert "local database" in banner + # And the badge itself follows the access state, not the plan name alone. + assert "function updateLicBadge()" in script + badge = script[script.index("function updateLicBadge()"):] + badge = badge[:badge.index("\n")] + assert "licAccessState()" in badge + assert "TRIAL ENDED" in badge and "INACTIVE" in badge + + +def test_both_persisted_answers_read_every_trial_field_the_server_sends() -> None: + """One vocabulary across the handshake, the entitlements read, and the cache.""" + + resolver = (REPO_ROOT / "engraphis" / "routes" / "v2_api.py").read_text( + encoding="utf-8" + ) + session = (REPO_ROOT / "engraphis" / "cloud_session.py").read_text(encoding="utf-8") + + for field in SERVER_TRIAL_FIELDS: + assert '"%s"' % field in resolver, "the resolver ignores %s" % field + assert '"%s"' % field in session, "the session record drops %s" % field + + +def test_the_plan_source_diagnostic_is_finally_rendered() -> None: + """``plan_source``/``plan_checked_at`` were emitted for support and shown nowhere.""" + + script = DASHBOARD_JS.read_text(encoding="utf-8") + + assert "d.plan_source" in script + assert "d.plan_checked_at" in script + assert "LIC_SOURCE_LABEL" in script + + +# ── (8) the shipped modules must still build on the supported floor ─────────── @pytest.mark.parametrize("relative", [ "engraphis/routes/v2_api.py", "engraphis/routes/memory.py", diff --git a/tests/test_licensing_launch.py b/tests/test_licensing_launch.py index 9f07b57..7ba94d0 100644 --- a/tests/test_licensing_launch.py +++ b/tests/test_licensing_launch.py @@ -158,7 +158,6 @@ def open(self, request, timeout=None): [ urllib.error.URLError(ConnectionRefusedError("connection refused")), urllib.error.URLError(socket.gaierror("name resolution failed")), - TimeoutError("timed out"), ], ) def test_transport_failures_report_a_retryable_outage(monkeypatch, error) -> None: @@ -170,6 +169,31 @@ def test_transport_failures_report_a_retryable_outage(monkeypatch, error) -> Non cloud_session._post_refresh("https://control.example.test", "r", "ws", "member") +@pytest.mark.parametrize( + "error", + [ + TimeoutError("timed out"), + ConnectionResetError("reset waiting for status"), + OSError("TLS connection failed while reading status"), + ], +) +def test_ambiguous_refresh_transport_requires_a_non_retryable_reconnect( + monkeypatch, error +) -> None: + """An unwrapped post-write transport failure may leave the credential spent.""" + + monkeypatch.setattr( + cloud_session, + "build_pinned_https_opener", + _opener_raising(error), + ) + + with pytest.raises(CloudSessionError, match="rotated credential could not be saved") as caught: + cloud_session._post_refresh("https://control.example.test", "r", "ws", "member") + + assert caught.value.status == 409 + + # ── (a)/(6) billing and authorization copy must be actionable ───────────────── def test_lapsed_subscription_is_not_reported_as_an_outage(monkeypatch, _upgrade_url): """402 is the control plane's "no active paid entitlement". diff --git a/tests/test_memory_routes_fixes.py b/tests/test_memory_routes_fixes.py index a307155..aefee77 100644 --- a/tests/test_memory_routes_fixes.py +++ b/tests/test_memory_routes_fixes.py @@ -8,6 +8,7 @@ """ import threading +import numpy as np import pytest from engraphis.config import settings @@ -137,3 +138,356 @@ def chat_with_context(self, **kwargs): result = response.json()["data"] assert result["llm_error"] == "LLM service unavailable" assert secret not in repr(result) + + +@pytest.mark.parametrize( + ("path", "extra_form"), + [ + ("/memory/vaults/upload-folder", {}), + ( + "/memory/vaults/upload-folder-smart", + {"auto_categorize_flag": "false"}, + ), + ], +) +def test_upload_batch_limit_remains_an_http_413( + monkeypatch, + tmp_path, + path, + extra_form, +): + from engraphis.routes import vault as vault_routes + + monkeypatch.setattr(vault_routes, "MAX_IMPORT_RESOURCE_BYTES", 4) + monkeypatch.setattr(vault_routes, "MAX_IMPORT_TOTAL_BYTES", 1) + with _client(monkeypatch, tmp_path) as client: + response = client.post( + path, + data={"namespace": "ns", "memory_type": "semantic", **extra_form}, + files={"files": ("memory.md", b"xy", "text/markdown")}, + ) + + assert response.status_code == 413 + assert response.json()["detail"] == {"error": "upload batch exceeds 1 bytes"} + + +@pytest.mark.parametrize( + "path", + [ + "/api/workspaces/import-files", + "/api/workspaces/import-files/", + "/memory/vaults/upload-folder", + "/memory/vaults/upload-folder/", + "/memory/vaults/upload-folder-smart", + "/memory/vaults/upload-folder-smart/", + ], +) +def test_vault_upload_rejects_declared_oversize_before_multipart_parse( + monkeypatch, + tmp_path, + path, +): + import engraphis.app as app_module + + monkeypatch.setattr(app_module, "VAULT_UPLOAD_REQUEST_BYTES", 16) + with _client(monkeypatch, tmp_path) as client: + response = client.post( + path, + content=b"not parsed", + headers={ + "content-type": "multipart/form-data; boundary=unused", + "content-length": "17", + }, + ) + + assert response.status_code == 413 + assert response.json() == { + "error": "request body too large", + "max_bytes": 16, + } + + +def test_vault_upload_counts_chunked_body_without_content_length(): + import asyncio + import json + + from engraphis.app import _VaultUploadLimitMiddleware + + messages = iter([ + {"type": "http.request", "body": b"12345", "more_body": True}, + {"type": "http.request", "body": b"67890", "more_body": False}, + ]) + sent = [] + + async def receive(): + return next(messages) + + async def send(message): + sent.append(message) + + async def drain_body(_scope, receive_body, send_response): + while True: + message = await receive_body() + if not message.get("more_body"): + break + await send_response({ + "type": "http.response.start", + "status": 204, + "headers": [], + }) + await send_response({"type": "http.response.body", "body": b""}) + + middleware = _VaultUploadLimitMiddleware(drain_body, max_bytes=8) + asyncio.run(middleware( + { + "type": "http", + "method": "POST", + "path": "/memory/vaults/upload-folder", + "headers": [(b"transfer-encoding", b"chunked")], + }, + receive, + send, + )) + + assert sent[0]["status"] == 413 + assert json.loads(sent[1]["body"])["max_bytes"] == 8 + + +def test_vault_upload_limits_valid_streamed_multipart_in_whole_app( + monkeypatch, + tmp_path, +): + import asyncio + import json + + import engraphis.app as app_module + + _setup_store(monkeypatch, tmp_path) + monkeypatch.setattr(settings, "loop_interval", 0) + monkeypatch.setattr(settings, "embed_model", "") + monkeypatch.setattr(app_module, "VAULT_UPLOAD_REQUEST_BYTES", 96) + body = ( + b"--vault\r\n" + b'Content-Disposition: form-data; name="namespace"\r\n\r\n' + b"ns\r\n" + b"--vault\r\n" + b'Content-Disposition: form-data; name="files"; filename="memory.md"\r\n' + b"Content-Type: text/markdown\r\n\r\n" + + b"x" * 64 + + b"\r\n--vault--\r\n" + ) + chunks = [ + { + "type": "http.request", + "body": body[offset:offset + 32], + "more_body": offset + 32 < len(body), + } + for offset in range(0, len(body), 32) + ] + messages = iter(chunks) + sent = [] + + async def receive(): + return next(messages, {"type": "http.disconnect"}) + + async def send(message): + sent.append(message) + + app = app_module.create_app() + asyncio.run(app( + { + "type": "http", + "asgi": {"version": "3.0"}, + "http_version": "1.1", + "method": "POST", + "scheme": "http", + "path": "/memory/vaults/upload-folder", + "raw_path": b"/memory/vaults/upload-folder", + "query_string": b"", + "root_path": "", + "headers": [ + (b"content-type", b"multipart/form-data; boundary=vault"), + (b"transfer-encoding", b"chunked"), + ], + "client": ("127.0.0.1", 12345), + "server": ("testserver", 80), + }, + receive, + send, + )) + + response_start = next( + message for message in sent if message["type"] == "http.response.start" + ) + response_body = b"".join( + message.get("body", b"") + for message in sent + if message["type"] == "http.response.body" + ) + assert response_start["status"] == 413 + assert json.loads(response_body)["max_bytes"] == 96 + + +@pytest.mark.parametrize( + ("path", "extra_form"), + [ + ("/memory/vaults/upload-folder", {}), + ( + "/memory/vaults/upload-folder-smart", + {"auto_categorize_flag": "false"}, + ), + ], +) +def test_vault_upload_stops_multipart_parse_at_file_count_limit( + monkeypatch, + tmp_path, + path, + extra_form, +): + from engraphis.routes import vault as vault_routes + + monkeypatch.setattr(vault_routes, "MAX_IMPORT_FILES", 2) + files = [ + ("files", (f"memory-{index}.md", b"# Memory", "text/markdown")) + for index in range(3) + ] + with _client(monkeypatch, tmp_path) as client: + response = client.post( + path, + data={"namespace": "ns", "memory_type": "semantic", **extra_form}, + files=files, + ) + + assert response.status_code == 413 + assert response.json()["detail"] == {"error": "too many files (max 2)"} + assert vault_routes.vault_store.get_vault("ns") is None + + +def test_vault_upload_instantiates_multipart_parser_with_bounded_file_cap( + monkeypatch, + tmp_path, +): + """The route handler, not a post-parse endpoint check, owns the parser cap.""" + import starlette.requests + + from engraphis.routes import vault as vault_routes + + monkeypatch.setattr(vault_routes, "MAX_IMPORT_FILES", 2) + parser_calls = [] + parser_type = starlette.requests.MultiPartParser + + class RecordingMultiPartParser(parser_type): + def __init__(self, headers, stream, *, max_files=1000, max_fields=1000, + max_part_size=1024 * 1024): + parser_calls.append((max_files, max_fields)) + super().__init__( + headers, + stream, + max_files=max_files, + max_fields=max_fields, + max_part_size=max_part_size, + ) + + monkeypatch.setattr(starlette.requests, "MultiPartParser", RecordingMultiPartParser) + with _client(monkeypatch, tmp_path) as client: + response = client.post( + "/memory/vaults/upload-folder", + data={"namespace": "ns", "memory_type": "semantic"}, + files=[ + ("files", (f"memory-{index}.md", b"# Memory", "text/markdown")) + for index in range(3) + ], + ) + + assert response.status_code == 413 + assert parser_calls == [(2, vault_routes._UPLOAD_FORM_FIELDS)] + + +def test_duplicate_candidate_query_uses_indexed_ordering_without_temp_sort(monkeypatch, tmp_path): + from engraphis.routes import vault as vault_routes + + _setup_store(monkeypatch, tmp_path) + vector = np.ones(8, dtype=np.float32) + mem_store.upsert_memory( + namespace="first", document_id="older", title="t", content="c", vector=vector, + updated_at=10, + ) + mem_store.upsert_memory( + namespace="first", document_id="newer", title="t", content="c", vector=vector, + updated_at=20, + ) + mem_store.upsert_memory( + namespace="second", document_id="latest-id", title="t", content="c", vector=vector, + updated_at=5, + ) + + conn = get_conn() + scoped_sql, scoped_params = vault_routes._duplicate_candidate_query("first") + global_sql, global_params = vault_routes._duplicate_candidate_query(None) + scoped_plan = " ".join( + row["detail"] for row in conn.execute( + "EXPLAIN QUERY PLAN " + scoped_sql + " LIMIT ?", (*scoped_params, 10) + ) + ) + global_plan = " ".join( + row["detail"] for row in conn.execute( + "EXPLAIN QUERY PLAN " + global_sql + " LIMIT ?", (*global_params, 10) + ) + ) + + assert "idx_mem_updated" in scoped_plan + assert "TEMP B-TREE" not in scoped_plan.upper() + assert "TEMP B-TREE" not in global_plan.upper() + assert [row["document_id"] for row in conn.execute(scoped_sql, scoped_params)] == [ + "newer", "older" + ] + assert conn.execute(global_sql, global_params).fetchone()["document_id"] == "latest-id" + + +def test_duplicate_health_bounds_candidates_results_and_uses_worker( + monkeypatch, + tmp_path, +): + import numpy as np + + from engraphis.routes import vault as vault_routes + + _setup_store(monkeypatch, tmp_path) + vector = np.ones(8, dtype=np.float32) / np.sqrt(8) + for index in range(4): + mem_store.upsert_memory( + namespace="ns", + document_id=f"d{index}", + title=f"Memory {index}", + content=f"duplicate content {index}", + vector=vector, + ) + + monkeypatch.setattr(vault_routes, "_DUPLICATE_CANDIDATE_LIMIT", 3) + monkeypatch.setattr(vault_routes, "_DUPLICATE_RESULT_LIMIT", 2) + monkeypatch.setattr( + vault_routes.mem_store, + "all_vectors", + lambda **_kwargs: pytest.fail("route must not load every vector"), + ) + worker_calls = [] + real_worker = vault_routes.asyncio.to_thread + + async def tracked_worker(function, *args): + worker_calls.append(function) + return await real_worker(function, *args) + + monkeypatch.setattr(vault_routes.asyncio, "to_thread", tracked_worker) + with _client(monkeypatch, tmp_path) as client: + response = client.get("/memory/health/duplicates?namespace=ns") + + assert response.status_code == 200 + data = response.json()["data"] + assert data["candidate_count"] == 3 + assert data["candidate_limit"] == 3 + assert data["matches_considered"] == 3 + assert data["count"] == 2 + assert len(data["duplicates"]) == 2 + assert data["result_limit"] == 2 + assert data["truncated"] is True + assert worker_calls == [vault_routes._duplicate_pairs] diff --git a/tests/test_postgres_schema.py b/tests/test_postgres_schema.py index caa103e..6344019 100644 --- a/tests/test_postgres_schema.py +++ b/tests/test_postgres_schema.py @@ -2,6 +2,8 @@ import sys import types +import pytest + from engraphis.backends import postgres_schema from engraphis.core.interfaces import SchemaSnapshot, SearchFilter from engraphis.service import MemoryService @@ -85,7 +87,7 @@ def connect(dsn, **kwargs): monkeypatch.setenv("ENGRAPHIS_POSTGRES_STATEMENT_TIMEOUT_MS", "45000") snapshot = postgres_schema.PostgresSchemaIntrospector().inspect( - "postgresql://local/appdb" + "postgresql://localhost/appdb" ) assert snapshot.metadata["database"] == "appdb" @@ -95,6 +97,143 @@ def connect(dsn, **kwargs): assert timeout_call[1] == ("45000",) +def test_postgres_connect_pins_validated_address_without_losing_tls_host(monkeypatch): + captured = {} + resolutions = 0 + + def resolve(*args, **kwargs): + nonlocal resolutions + resolutions += 1 + if resolutions > 1: + return [ + ( + postgres_schema.socket.AF_INET, + postgres_schema.socket.SOCK_STREAM, + postgres_schema.socket.IPPROTO_TCP, + "", + ("10.0.0.8", 5432), + ) + ] + return [ + ( + postgres_schema.socket.AF_INET, + postgres_schema.socket.SOCK_STREAM, + postgres_schema.socket.IPPROTO_TCP, + "", + ("93.184.216.34", 5432), + ) + ] + + def connect(dsn, **kwargs): + captured["dsn"] = dsn + captured.update(kwargs) + return _Connection() + + monkeypatch.setattr(postgres_schema.socket, "getaddrinfo", resolve) + monkeypatch.setitem(sys.modules, "psycopg", types.SimpleNamespace(connect=connect)) + + dsn = ( + "postgresql://db.example/appdb" + "?host=10.0.0.8&hostaddr=10.0.0.8" + ) + postgres_schema._connect(dsn) + + assert resolutions == 1 + assert captured["dsn"] == dsn + assert captured["host"] == "db.example" + assert captured["hostaddr"] == "93.184.216.34" + + +@pytest.mark.parametrize( + "address", + [ + "10.0.0.8", + "100.64.0.1", + "127.0.0.2", + "169.254.1.1", + "192.0.2.1", + "224.0.0.1", + "::", + "fe80::1", + "ff02::1", + ], +) +def test_postgres_connect_rejects_non_global_addresses(monkeypatch, address): + family = ( + postgres_schema.socket.AF_INET6 + if ":" in address + else postgres_schema.socket.AF_INET + ) + monkeypatch.setattr( + postgres_schema.socket, + "getaddrinfo", + lambda *args, **kwargs: [ + ( + family, + postgres_schema.socket.SOCK_STREAM, + postgres_schema.socket.IPPROTO_TCP, + "", + (address, 5432), + ) + ], + ) + + with pytest.raises( + postgres_schema.PostgresIntrospectionError, + match="global unicast", + ): + postgres_schema._connect("postgresql://db.example/appdb") + + +def test_postgres_connect_rejects_mixed_global_and_private_dns(monkeypatch): + monkeypatch.setattr( + postgres_schema.socket, + "getaddrinfo", + lambda *args, **kwargs: [ + ( + postgres_schema.socket.AF_INET, + postgres_schema.socket.SOCK_STREAM, + postgres_schema.socket.IPPROTO_TCP, + "", + ("93.184.216.34", 5432), + ), + ( + postgres_schema.socket.AF_INET, + postgres_schema.socket.SOCK_STREAM, + postgres_schema.socket.IPPROTO_TCP, + "", + ("10.0.0.8", 5432), + ), + ], + ) + + with pytest.raises( + postgres_schema.PostgresIntrospectionError, + match="global unicast", + ): + postgres_schema._connect("postgresql://db.example/appdb") + + +def test_postgres_connect_keeps_explicit_dsn_and_loopback_exceptions(monkeypatch): + calls = [] + + def connect(dsn, **kwargs): + calls.append((dsn, kwargs)) + return _Connection() + + explicit = "postgresql://db.internal/appdb" + monkeypatch.setenv("ENGRAPHIS_POSTGRES_DSN", explicit) + monkeypatch.setitem(sys.modules, "psycopg", types.SimpleNamespace(connect=connect)) + + postgres_schema._connect(explicit) + monkeypatch.delenv("ENGRAPHIS_POSTGRES_DSN") + postgres_schema._connect("postgresql://localhost/appdb") + + assert calls[0][1] == {"connect_timeout": 10} + assert calls[1][1]["host"] == "localhost" + assert calls[1][1]["hostaddr"] == "127.0.0.1" + + def test_postgres_introspection_is_filtered_bounded_and_cross_schema_safe(monkeypatch): connection = _Connection() monkeypatch.setattr(postgres_schema, "_connect", lambda dsn: connection) diff --git a/tests/test_release_infrastructure.py b/tests/test_release_infrastructure.py index 52d3b17..d2b6ea3 100644 --- a/tests/test_release_infrastructure.py +++ b/tests/test_release_infrastructure.py @@ -202,6 +202,8 @@ def test_public_capability_and_support_docs_match_the_shipped_tree(): changelog = _text("CHANGELOG.md") assert "ForceGraph + D3 renderer" in changelog + assert "## [1.1.0] - 2026-07-26" in changelog + assert "Public 1.1.0 hosted-connect and graph-experience release." in changelog assert "## [1.0.1] - 2026-07-24" in changelog assert "Public 1.0.1 client reliability release." in changelog assert "## [1.0.0] - 2026-07-23" in changelog