From 60d3738fcb9080eef7ba3c7daa50198c93663c6f Mon Sep 17 00:00:00 2001 From: Nitjsefnie Date: Wed, 22 Jul 2026 10:15:07 +0000 Subject: [PATCH] lsp: stdlib completion + hover generated from lib/ signature comments (#590) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit handle_completion/handle_hover served builtin_docs[] and document-local symbols, but the 75-module pure-EigenScript lib/ layer (815 public functions) was invisible: typing `stats.` offered nothing, and hovering a lib/ call showed nothing. tools/gen_lsp_stdlib_index.sh (shell+awk, zero-dependency, in the style of tools/stdlib_index_check.sh) scrapes lib/*.eigs for public `define name(params)` definitions (leading-underscore names stay private) plus their rule-2 `# name of args -> type` signature comment (docs/STDLIB.md "Writing Library Functions"), emitting src/lsp_stdlib_index.h: a static {module, name, params, detail} table. Functions without a signature comment fall back to the `define name(params) as:` shape — back-filling those comments is tracked separately in STDLIB.md, not papered over here. The header is a build artifact, not committed (the build/ amalgamation precedent, #397): .gitignore'd, and regenerated by the Makefile lsp target and build.sh lsp before eigenlsp compiles, so install.sh and both CI lsp legs pick it up automatically. eigenlsp serves the table through the existing paths, scoped by the document's own import-symbol tracking: - completion: an imported module's public functions are offered as kind-3 Function items with the signature as detail; unimported modules stay invisible. - hover: a dotted `mod.name` pins the module; a bare name matches any imported module's function. Either way the detail is the signature comment or the define-shape fallback. tests/test_lsp.py gains 8 checks (imported offered w/ signature detail, fallback detail, unimported absent, no-import empty, hover signature, hover fallback, hover unimported -> null); the [88] header count moves 23 -> 80 (it had already drifted from the real 72). Gates: release suite 3138/3138, asan+ubsan suite (detect_leaks=1) 3142/3142, test_lsp_asan.sh 80/80 with no sanitizer reports, tools/stdlib_index_check.sh + selftest green. Co-Authored-By: Kimi K3 --- .gitignore | 2 + Makefile | 9 ++- build.sh | 4 +- src/eigenlsp.c | 61 +++++++++++++++++++ tests/run_all_tests.sh | 2 +- tests/test_lsp.py | 60 +++++++++++++++++++ tools/gen_lsp_stdlib_index.sh | 106 ++++++++++++++++++++++++++++++++++ 7 files changed, 241 insertions(+), 3 deletions(-) create mode 100755 tools/gen_lsp_stdlib_index.sh diff --git a/.gitignore b/.gitignore index 76f993ba..e5d2992e 100644 --- a/.gitignore +++ b/.gitignore @@ -46,3 +46,5 @@ tests/tmp_mod_373*.eigs # Generated embeddable artifacts (make lib / make amalgamation, #397) build/ +# Generated LSP stdlib index (make lsp / build.sh lsp, #590) +src/lsp_stdlib_index.h diff --git a/Makefile b/Makefile index 15ae861e..ed5ff74e 100644 --- a/Makefile +++ b/Makefile @@ -114,7 +114,14 @@ install: build lsp @echo "Installed: $(PREFIX)/bin/eigenlsp (v$(VERSION))" @echo "Stdlib: $(PREFIX)/lib/eigenscript/" -lsp: +# Generated stdlib index for the LSP (#590): public define/signature-comment +# table scraped from lib/*.eigs. A build artifact, not committed (the build/ +# amalgamation precedent, #397) — regenerated whenever lib/ or the script +# changes, before eigenlsp compiles. build.sh's lsp branch does the same. +$(SRC_DIR)/lsp_stdlib_index.h: $(wildcard lib/*.eigs) tools/gen_lsp_stdlib_index.sh + bash tools/gen_lsp_stdlib_index.sh + +lsp: $(SRC_DIR)/lsp_stdlib_index.h $(CC) $(CFLAGS) -o $(LSP_BINARY) $(LSP_SOURCES) \ -DEIGENSCRIPT_EXT_HTTP=0 \ -DEIGENSCRIPT_EXT_MODEL=0 \ diff --git a/build.sh b/build.sh index 45945d04..9888273d 100755 --- a/build.sh +++ b/build.sh @@ -36,7 +36,9 @@ if [ "$1" = "lsp" ]; then # toolchain. Links eigenlsp.c against the runtime (SOURCES minus the # CLI-only units, read from the Makefile's CLI_ONLY so the drop list # can't drift either), minimal extensions, gcc-only like the rest of - # build.sh. + # build.sh. The stdlib index header it includes (#590) is generated + # from lib/ first — same rule as the Makefile lsp target. + bash ../tools/gen_lsp_stdlib_index.sh LSP_SOURCES=" $SOURCES " for u in $CLI_ONLY; do LSP_SOURCES="${LSP_SOURCES/ $u / }"; done LSP_SOURCES="$LSP_SOURCES eigenlsp.c" diff --git a/src/eigenlsp.c b/src/eigenlsp.c index 6aeb2fbd..364692b4 100644 --- a/src/eigenlsp.c +++ b/src/eigenlsp.c @@ -126,6 +126,15 @@ static const char *keyword_docs[][2] = { {NULL, NULL} }; +/* ================================================================ + * STDLIB (lib/) DOCUMENTATION TABLE — GENERATED + * ================================================================ + * stdlib_docs[][4]: module, name, params, detail (rule-2 signature + * comment, else the `define name(params) as:` fallback). Scraped from + * the lib/ .eigs modules by tools/gen_lsp_stdlib_index.sh at build + * time (#590). */ +#include "lsp_stdlib_index.h" + /* ================================================================ * SYMBOL TABLE * ================================================================ */ @@ -699,6 +708,16 @@ static Token* find_token_at(Document *doc, int line, int col) { * LSP HANDLERS * ================================================================ */ +/* Does the document import this module? Scopes the stdlib table (#590): + * a module's functions surface only after the file `import`s it. */ +static int doc_imports_module(Document *doc, const char *module) { + for (int i = 0; i < doc->symbol_count; i++) { + Symbol *s = &doc->symbols[i]; + if (s->kind == SYM_IMPORT && strcmp(s->name, module) == 0) return 1; + } + return 0; +} + static void handle_initialize(int id) { lsp_response(id, "{" @@ -929,6 +948,19 @@ static void handle_completion(int id, const char *params) { } strbuf_append_char(&sb, '}'); } + /* Stdlib functions of the modules this document imports + * (#590): a `stats.` prefix offers stats.* with the + * signature as detail; unimported modules stay invisible. */ + for (int i = 0; stdlib_docs[i][0]; i++) { + if (!doc_imports_module(doc, stdlib_docs[i][0])) continue; + if (!first) strbuf_append_char(&sb, ','); + first = 0; + strbuf_append(&sb, "{\"label\":\""); + strbuf_append(&sb, stdlib_docs[i][1]); + strbuf_append(&sb, "\",\"kind\":3,\"detail\":"); /* 3 = Function */ + json_escape_to(&sb, stdlib_docs[i][3]); + strbuf_append_char(&sb, '}'); + } } free(uri); } @@ -1063,6 +1095,35 @@ static void handle_hover(int id, const char *params) { } } + /* Check stdlib (lib/) functions (#590): import-scoped, like completion. + * A dotted `mod.name` pins the module (a dict member access whose + * qualifier is not an imported module is left alone); a bare name + * matches any imported module's function in table order. */ + if (!hover_text) { + ptrdiff_t ti = tok - doc->tokens.tokens; + int dotted = (ti >= 1 && doc->tokens.tokens[ti - 1].type == TOK_DOT); + const char *dot_module = NULL; + if (dotted && ti >= 2) { + Token *t2 = &doc->tokens.tokens[ti - 2]; + if (t2->type == TOK_IDENT && t2->str_val && + doc_imports_module(doc, t2->str_val)) + dot_module = t2->str_val; + } + if (!dotted || dot_module) { + for (int i = 0; stdlib_docs[i][0]; i++) { + if (dot_module) { + if (strcmp(stdlib_docs[i][0], dot_module) != 0) continue; + } else if (!doc_imports_module(doc, stdlib_docs[i][0])) { + continue; + } + if (strcmp(tok->str_val, stdlib_docs[i][1]) == 0) { + hover_text = stdlib_docs[i][3]; + break; + } + } + } + } + if (!hover_text) { lsp_response_null(id); return; } strbuf sb; diff --git a/tests/run_all_tests.sh b/tests/run_all_tests.sh index 528446de..886685ac 100755 --- a/tests/run_all_tests.sh +++ b/tests/run_all_tests.sh @@ -3053,7 +3053,7 @@ echo "" # assert initialize/diagnostics/completion/hover/definition/references/ # shutdown. The LSP was previously only compile-checked. Skips cleanly # without python3 or the eigenlsp build. -echo "[88] LSP Behavioral (23 checks)" +echo "[88] LSP Behavioral (80 checks)" LSP_OUTPUT=$(bash "$TESTS_DIR/test_lsp.sh" 2>&1) if echo "$LSP_OUTPUT" | grep -q "SKIP:"; then echo "$LSP_OUTPUT" | grep "SKIP:" diff --git a/tests/test_lsp.py b/tests/test_lsp.py index 1e65ff14..218bbad6 100755 --- a/tests/test_lsp.py +++ b/tests/test_lsp.py @@ -203,6 +203,66 @@ def main(): check("completion does not advertise a phantom 'input' builtin", isinstance(items, list) and not any(it.get("label") == "input" for it in items)) + # --- #590: stdlib (lib/) completion + hover from the generated index --- + # Completion is import-aware: the document's own `import`s scope which + # modules' functions are offered; the detail is the rule-2 signature + # comment, or the `define name(params) as:` shape when a module never + # wrote one (stats has none for median; list.zip has one). + std_doc = "import stats\nimport list\nresult is stats.mean of [1, 2, 3]\n" + comp_std = {"jsonrpc": "2.0", "id": 26, "method": "textDocument/completion", + "params": {"textDocument": {"uri": URI}, "position": {"line": 2, "character": 12}}} + r = converse([INIT, did_open(std_doc), comp_std, SHUTDOWN, EXIT]) + res = (by_id(r, 26) or {}).get("result", {}) + items = res.get("items") if isinstance(res, dict) and isinstance(res.get("items"), list) else [] + zip_item = next((it for it in items if it.get("label") == "zip"), None) + check("stdlib completion offers an imported module's function (list.zip)", + isinstance(zip_item, dict) and zip_item.get("kind") == 3) + check("stdlib completion detail is the signature comment", + isinstance(zip_item, dict) + and zip_item.get("detail") == "zip of [list_a, list_b] -> list of [a, b] pairs") + median_item = next((it for it in items if it.get("label") == "median"), None) + check("stdlib completion falls back to the define shape (stats.median)", + isinstance(median_item, dict) + and median_item.get("detail") == "define median(vals) as:") + check("stdlib completion omits unimported modules (map.map_get absent)", + not any(it.get("label") == "map_get" for it in items)) + + # No imports in the document → no stdlib items at all. + comp_plain = {"jsonrpc": "2.0", "id": 27, "method": "textDocument/completion", + "params": {"textDocument": {"uri": URI}, "position": {"line": 0, "character": 0}}} + r = converse([INIT, did_open("plain is 1\n"), comp_plain, SHUTDOWN, EXIT]) + res = (by_id(r, 27) or {}).get("result", {}) + items = res.get("items") if isinstance(res, dict) and isinstance(res.get("items"), list) else [] + check("stdlib completion is empty without any import", + not any(it.get("label") in ("zip", "median", "map_get") for it in items)) + + # Hover over a dotted module call shows the signature comment... + hover_zip = {"jsonrpc": "2.0", "id": 28, "method": "textDocument/hover", + "params": {"textDocument": {"uri": URI}, "position": {"line": 1, "character": 11}}} + r = converse([INIT, did_open("import list\nz is list.zip of [[1, 2], [3, 4]]\n"), + hover_zip, SHUTDOWN, EXIT]) + res = (by_id(r, 28) or {}).get("result") + value = ((res or {}).get("contents") or {}).get("value", "") if isinstance(res, dict) else "" + check("hover over an imported module's call shows the signature", + "zip of [list_a, list_b] -> list of [a, b] pairs" in value) + + # ...and the define-shape fallback when the module wrote no comment. + hover_mean = {"jsonrpc": "2.0", "id": 29, "method": "textDocument/hover", + "params": {"textDocument": {"uri": URI}, "position": {"line": 2, "character": 17}}} + r = converse([INIT, did_open(std_doc), hover_mean, SHUTDOWN, EXIT]) + res = (by_id(r, 29) or {}).get("result") + value = ((res or {}).get("contents") or {}).get("value", "") if isinstance(res, dict) else "" + check("hover over a stdlib call without a signature shows the define shape", + "define mean(vals) as:" in value) + + # A function from a module the document never imported gets no hover. + hover_noimp = {"jsonrpc": "2.0", "id": 30, "method": "textDocument/hover", + "params": {"textDocument": {"uri": URI}, "position": {"line": 1, "character": 7}}} + r = converse([INIT, did_open("import stats\nz is map_get of [m, \"k\"]\n"), + hover_noimp, SHUTDOWN, EXIT]) + check("hover ignores a function from an unimported module", + (by_id(r, 30) or {}).get("result") is None) + # --- hover over a defined symbol returns contents --- hover = {"jsonrpc": "2.0", "id": 3, "method": "textDocument/hover", "params": {"textDocument": {"uri": URI}, "position": {"line": 0, "character": 0}}} diff --git a/tools/gen_lsp_stdlib_index.sh b/tools/gen_lsp_stdlib_index.sh new file mode 100755 index 00000000..de8876f3 --- /dev/null +++ b/tools/gen_lsp_stdlib_index.sh @@ -0,0 +1,106 @@ +#!/bin/bash +# Generate src/lsp_stdlib_index.h — the stdlib (lib/*.eigs) index the LSP +# serves as import-aware completion + hover (#590). +# +# Scrapes every lib/*.eigs for PUBLIC `define name(params)` definitions +# (leading-underscore names are private, per docs/STDLIB.md) plus their +# rule-2 signature comment (`# name of args -> return_type`, docs/STDLIB.md +# "Writing Library Functions"). Functions without a signature comment are +# tolerated: the detail falls back to the `define name(params) as:` shape +# (back-filling the ~200 missing comments is tracked separately — do not +# paper over it here). +# +# Parsing conventions follow tools/stdlib_index_check.sh (cheap grep/awk +# over lib/, zero dependencies — no python). The generated header is a +# build artifact, NOT committed (the build/ amalgamation precedent, #397): +# the Makefile `lsp` target and `build.sh lsp` regenerate it before +# compiling eigenlsp. +# +# Usage: tools/gen_lsp_stdlib_index.sh [output-header] +# default output: src/lsp_stdlib_index.h +# Exit 0 on success. + +set -u +cd "$(dirname "$0")/.." + +OUT="${1:-src/lsp_stdlib_index.h}" + +# LC_ALL=C: deterministic glob order + byte-wise awk on any box. +LC_ALL=C +export LC_ALL + +awk -v OUT="$OUT" ' +function c_escape(s) { + gsub(/\\/, "\\\\", s) + gsub(/"/, "\\\"", s) + return s +} +# Signature comment attached to this define: the LAST line in the comment +# block directly above that reads `# of ...` (banner lines like +# `# ---- name: desc ----` never match; note lines below the signature do +# not hide it). Returns "" when the module never wrote one (fallback). +function sig_from_block(name, i, text, found) { + found = "" + for (i = 1; i <= ncb; i++) { + text = cb[i] + sub(/^#[ \t]*/, "", text) + if (index(text, name " of ") == 1) found = text + } + return found +} +BEGIN { + print "/* Generated by tools/gen_lsp_stdlib_index.sh from the lib/ .eigs modules" > OUT + print " * -- DO NOT EDIT. Public define name(params) definitions + their rule-2" > OUT + print " * signature comments (docs/STDLIB.md \"Writing Library Functions\"), served" > OUT + print " * by src/eigenlsp.c as import-aware completion + hover (#590). Fallback" > OUT + print " * detail is the define shape. Regenerated by the Makefile lsp target /" > OUT + print " * build.sh lsp; not committed. Columns: module, name, params, detail. */" > OUT + print "#include " > OUT + print "static const char *stdlib_docs[][4] = {" > OUT +} +FNR == 1 { + module = FILENAME + sub(/.*\//, "", module) + sub(/\.eigs$/, "", module) + ncb = 0 + modules++ +} +/^[ \t]*#/ { + cb[++ncb] = $0 + next +} +/^define[ \t]+[A-Za-z_][A-Za-z0-9_]*/ { + line = $0 + sub(/^define[ \t]+/, "", line) + match(line, /^[A-Za-z_][A-Za-z0-9_]*/) + name = substr(line, 1, RLENGTH) + rest = substr(line, RLENGTH + 1) + params = "" + if (substr(rest, 1, 1) == "(") { + # No nesting or `is`-defaults in lib parameter lists (verified); + # the first ")" closes the list. + p = index(rest, ")") + params = substr(rest, 2, p - 2) + } + if (substr(name, 1, 1) != "_") { + detail = sig_from_block(name) + if (detail == "") + detail = "define " name "(" params ") as:" + printf " {\"%s\", \"%s\", \"%s\", \"%s\"},\n", \ + c_escape(module), c_escape(name), c_escape(params), c_escape(detail) > OUT + funcs++ + } + ncb = 0 + next +} +{ + # Any non-comment, non-define line (including blank) detaches the block. + ncb = 0 +} +END { + print " {NULL, NULL, NULL, NULL}" > OUT + print "};" > OUT + printf "lsp_stdlib_index: %d public functions across %d lib modules -> %s\n", \ + funcs, modules, OUT > "/dev/stderr" +} +' lib/*.eigs