LSP: stdlib completion + hover generated from lib/ signature comments (fixes #590)

.gitignore

@@ -46,3 +46,5 @@ tests/tmp_mod_373*.eigs
 
 # Generated embeddable artifacts (make lib / make amalgamation, #397)
 build/
+# Generated LSP stdlib index (make lsp / build.sh lsp, #590)
+src/lsp_stdlib_index.h

Makefile

@@ -114,7 +114,14 @@ install: build lsp
 	@echo "Installed: $(PREFIX)/bin/eigenlsp (v$(VERSION))"
 	@echo "Stdlib:    $(PREFIX)/lib/eigenscript/"
 
-lsp:
+# Generated stdlib index for the LSP (#590): public define/signature-comment
+# table scraped from lib/*.eigs. A build artifact, not committed (the build/
+# amalgamation precedent, #397) — regenerated whenever lib/ or the script
+# changes, before eigenlsp compiles. build.sh's lsp branch does the same.
+$(SRC_DIR)/lsp_stdlib_index.h: $(wildcard lib/*.eigs) tools/gen_lsp_stdlib_index.sh
+	bash tools/gen_lsp_stdlib_index.sh
+
+lsp: $(SRC_DIR)/lsp_stdlib_index.h
 	$(CC) $(CFLAGS) -o $(LSP_BINARY) $(LSP_SOURCES) \
 		-DEIGENSCRIPT_EXT_HTTP=0 \
 		-DEIGENSCRIPT_EXT_MODEL=0 \

build.sh

@@ -36,7 +36,9 @@ if [ "$1" = "lsp" ]; then
     # toolchain. Links eigenlsp.c against the runtime (SOURCES minus the
     # CLI-only units, read from the Makefile's CLI_ONLY so the drop list
     # can't drift either), minimal extensions, gcc-only like the rest of
-    # build.sh.
+    # build.sh. The stdlib index header it includes (#590) is generated
+    # from lib/ first — same rule as the Makefile lsp target.
+    bash ../tools/gen_lsp_stdlib_index.sh
     LSP_SOURCES=" $SOURCES "
     for u in $CLI_ONLY; do LSP_SOURCES="${LSP_SOURCES/ $u / }"; done
     LSP_SOURCES="$LSP_SOURCES eigenlsp.c"

src/eigenlsp.c

@@ -126,6 +126,15 @@ static const char *keyword_docs[][2] = {
     {NULL, NULL}
 };
 
+/* ================================================================
+ * STDLIB (lib/) DOCUMENTATION TABLE — GENERATED
+ * ================================================================
+ * stdlib_docs[][4]: module, name, params, detail (rule-2 signature
+ * comment, else the `define name(params) as:` fallback). Scraped from
+ * the lib/ .eigs modules by tools/gen_lsp_stdlib_index.sh at build
+ * time (#590). */
+#include "lsp_stdlib_index.h"
+
 /* ================================================================
  * SYMBOL TABLE
  * ================================================================ */
@@ -699,6 +708,16 @@ static Token* find_token_at(Document *doc, int line, int col) {
  * LSP HANDLERS
  * ================================================================ */
 
+/* Does the document import this module? Scopes the stdlib table (#590):
+ * a module's functions surface only after the file `import`s it. */
+static int doc_imports_module(Document *doc, const char *module) {
+    for (int i = 0; i < doc->symbol_count; i++) {
+        Symbol *s = &doc->symbols[i];
+        if (s->kind == SYM_IMPORT && strcmp(s->name, module) == 0) return 1;
+    }
+    return 0;
+}
+
 static void handle_initialize(int id) {
     lsp_response(id,
         "{"
@@ -929,6 +948,19 @@ static void handle_completion(int id, const char *params) {
                     }
                     strbuf_append_char(&sb, '}');
                 }
+                /* Stdlib functions of the modules this document imports
+                 * (#590): a `stats.` prefix offers stats.* with the
+                 * signature as detail; unimported modules stay invisible. */
+                for (int i = 0; stdlib_docs[i][0]; i++) {
+                    if (!doc_imports_module(doc, stdlib_docs[i][0])) continue;
+                    if (!first) strbuf_append_char(&sb, ',');
+                    first = 0;
+                    strbuf_append(&sb, "{\"label\":\"");
+                    strbuf_append(&sb, stdlib_docs[i][1]);
+                    strbuf_append(&sb, "\",\"kind\":3,\"detail\":");  /* 3 = Function */
+                    json_escape_to(&sb, stdlib_docs[i][3]);
+                    strbuf_append_char(&sb, '}');
+                }
             }
             free(uri);
         }
@@ -1063,6 +1095,35 @@ static void handle_hover(int id, const char *params) {
         }
     }
 
+    /* Check stdlib (lib/) functions (#590): import-scoped, like completion.
+     * A dotted `mod.name` pins the module (a dict member access whose
+     * qualifier is not an imported module is left alone); a bare name
+     * matches any imported module's function in table order. */
+    if (!hover_text) {
+        ptrdiff_t ti = tok - doc->tokens.tokens;
+        int dotted = (ti >= 1 && doc->tokens.tokens[ti - 1].type == TOK_DOT);
+        const char *dot_module = NULL;
+        if (dotted && ti >= 2) {
+            Token *t2 = &doc->tokens.tokens[ti - 2];
+            if (t2->type == TOK_IDENT && t2->str_val &&
+                doc_imports_module(doc, t2->str_val))
+                dot_module = t2->str_val;
+        }
+        if (!dotted || dot_module) {
+            for (int i = 0; stdlib_docs[i][0]; i++) {
+                if (dot_module) {
+                    if (strcmp(stdlib_docs[i][0], dot_module) != 0) continue;
+                } else if (!doc_imports_module(doc, stdlib_docs[i][0])) {
+                    continue;
+                }
+                if (strcmp(tok->str_val, stdlib_docs[i][1]) == 0) {
+                    hover_text = stdlib_docs[i][3];
+                    break;
+                }
+            }
+        }
+    }
+
     if (!hover_text) { lsp_response_null(id); return; }
 
     strbuf sb;

tests/run_all_tests.sh

@@ -3053,7 +3053,7 @@ echo ""
 # assert initialize/diagnostics/completion/hover/definition/references/
 # shutdown. The LSP was previously only compile-checked. Skips cleanly
 # without python3 or the eigenlsp build.
-echo "[88] LSP Behavioral (23 checks)"
+echo "[88] LSP Behavioral (80 checks)"
 LSP_OUTPUT=$(bash "$TESTS_DIR/test_lsp.sh" 2>&1)
 if echo "$LSP_OUTPUT" | grep -q "SKIP:"; then
     echo "$LSP_OUTPUT" | grep "SKIP:"

tests/test_lsp.py

@@ -203,6 +203,66 @@ def main():
     check("completion does not advertise a phantom 'input' builtin",
           isinstance(items, list) and not any(it.get("label") == "input" for it in items))
 
+    # --- #590: stdlib (lib/) completion + hover from the generated index ---
+    # Completion is import-aware: the document's own `import`s scope which
+    # modules' functions are offered; the detail is the rule-2 signature
+    # comment, or the `define name(params) as:` shape when a module never
+    # wrote one (stats has none for median; list.zip has one).
+    std_doc = "import stats\nimport list\nresult is stats.mean of [1, 2, 3]\n"
+    comp_std = {"jsonrpc": "2.0", "id": 26, "method": "textDocument/completion",
+                "params": {"textDocument": {"uri": URI}, "position": {"line": 2, "character": 12}}}
+    r = converse([INIT, did_open(std_doc), comp_std, SHUTDOWN, EXIT])
+    res = (by_id(r, 26) or {}).get("result", {})
+    items = res.get("items") if isinstance(res, dict) and isinstance(res.get("items"), list) else []
+    zip_item = next((it for it in items if it.get("label") == "zip"), None)
+    check("stdlib completion offers an imported module's function (list.zip)",
+          isinstance(zip_item, dict) and zip_item.get("kind") == 3)
+    check("stdlib completion detail is the signature comment",
+          isinstance(zip_item, dict)
+          and zip_item.get("detail") == "zip of [list_a, list_b] -> list of [a, b] pairs")
+    median_item = next((it for it in items if it.get("label") == "median"), None)
+    check("stdlib completion falls back to the define shape (stats.median)",
+          isinstance(median_item, dict)
+          and median_item.get("detail") == "define median(vals) as:")
+    check("stdlib completion omits unimported modules (map.map_get absent)",
+          not any(it.get("label") == "map_get" for it in items))
+
+    # No imports in the document → no stdlib items at all.
+    comp_plain = {"jsonrpc": "2.0", "id": 27, "method": "textDocument/completion",
+                  "params": {"textDocument": {"uri": URI}, "position": {"line": 0, "character": 0}}}
+    r = converse([INIT, did_open("plain is 1\n"), comp_plain, SHUTDOWN, EXIT])
+    res = (by_id(r, 27) or {}).get("result", {})
+    items = res.get("items") if isinstance(res, dict) and isinstance(res.get("items"), list) else []
+    check("stdlib completion is empty without any import",
+          not any(it.get("label") in ("zip", "median", "map_get") for it in items))
+
+    # Hover over a dotted module call shows the signature comment...
+    hover_zip = {"jsonrpc": "2.0", "id": 28, "method": "textDocument/hover",
+                 "params": {"textDocument": {"uri": URI}, "position": {"line": 1, "character": 11}}}
+    r = converse([INIT, did_open("import list\nz is list.zip of [[1, 2], [3, 4]]\n"),
+                  hover_zip, SHUTDOWN, EXIT])
+    res = (by_id(r, 28) or {}).get("result")
+    value = ((res or {}).get("contents") or {}).get("value", "") if isinstance(res, dict) else ""
+    check("hover over an imported module's call shows the signature",
+          "zip of [list_a, list_b] -> list of [a, b] pairs" in value)
+
+    # ...and the define-shape fallback when the module wrote no comment.
+    hover_mean = {"jsonrpc": "2.0", "id": 29, "method": "textDocument/hover",
+                  "params": {"textDocument": {"uri": URI}, "position": {"line": 2, "character": 17}}}
+    r = converse([INIT, did_open(std_doc), hover_mean, SHUTDOWN, EXIT])
+    res = (by_id(r, 29) or {}).get("result")
+    value = ((res or {}).get("contents") or {}).get("value", "") if isinstance(res, dict) else ""
+    check("hover over a stdlib call without a signature shows the define shape",
+          "define mean(vals) as:" in value)
+
+    # A function from a module the document never imported gets no hover.
+    hover_noimp = {"jsonrpc": "2.0", "id": 30, "method": "textDocument/hover",
+                   "params": {"textDocument": {"uri": URI}, "position": {"line": 1, "character": 7}}}
+    r = converse([INIT, did_open("import stats\nz is map_get of [m, \"k\"]\n"),
+                  hover_noimp, SHUTDOWN, EXIT])
+    check("hover ignores a function from an unimported module",
+          (by_id(r, 30) or {}).get("result") is None)
+
     # --- hover over a defined symbol returns contents ---
     hover = {"jsonrpc": "2.0", "id": 3, "method": "textDocument/hover",
              "params": {"textDocument": {"uri": URI}, "position": {"line": 0, "character": 0}}}

tools/gen_lsp_stdlib_index.sh

@@ -0,0 +1,106 @@
+#!/bin/bash
+# Generate src/lsp_stdlib_index.h — the stdlib (lib/*.eigs) index the LSP
+# serves as import-aware completion + hover (#590).
+#
+# Scrapes every lib/*.eigs for PUBLIC `define name(params)` definitions
+# (leading-underscore names are private, per docs/STDLIB.md) plus their
+# rule-2 signature comment (`# name of args -> return_type`, docs/STDLIB.md
+# "Writing Library Functions"). Functions without a signature comment are
+# tolerated: the detail falls back to the `define name(params) as:` shape
+# (back-filling the ~200 missing comments is tracked separately — do not
+# paper over it here).
+#
+# Parsing conventions follow tools/stdlib_index_check.sh (cheap grep/awk
+# over lib/, zero dependencies — no python). The generated header is a
+# build artifact, NOT committed (the build/ amalgamation precedent, #397):
+# the Makefile `lsp` target and `build.sh lsp` regenerate it before
+# compiling eigenlsp.
+#
+# Usage: tools/gen_lsp_stdlib_index.sh [output-header]
+#   default output: src/lsp_stdlib_index.h
+# Exit 0 on success.
+
+set -u
+cd "$(dirname "$0")/.."
+
+OUT="${1:-src/lsp_stdlib_index.h}"
+
+# LC_ALL=C: deterministic glob order + byte-wise awk on any box.
+LC_ALL=C
+export LC_ALL
+
+awk -v OUT="$OUT" '
+function c_escape(s) {
+    gsub(/\\/, "\\\\", s)
+    gsub(/"/, "\\\"", s)
+    return s
+}
+# Signature comment attached to this define: the LAST line in the comment
+# block directly above that reads `# <name> of ...` (banner lines like
+# `# ---- name: desc ----` never match; note lines below the signature do
+# not hide it). Returns "" when the module never wrote one (fallback).
+function sig_from_block(name,    i, text, found) {
+    found = ""
+    for (i = 1; i <= ncb; i++) {
+        text = cb[i]
+        sub(/^#[ \t]*/, "", text)
+        if (index(text, name " of ") == 1) found = text
+    }
+    return found
+}
+BEGIN {
+    print "/* Generated by tools/gen_lsp_stdlib_index.sh from the lib/ .eigs modules" > OUT
+    print " * -- DO NOT EDIT. Public define name(params) definitions + their rule-2" > OUT
+    print " * signature comments (docs/STDLIB.md \"Writing Library Functions\"), served" > OUT
+    print " * by src/eigenlsp.c as import-aware completion + hover (#590). Fallback" > OUT
+    print " * detail is the define shape. Regenerated by the Makefile lsp target /" > OUT
+    print " * build.sh lsp; not committed. Columns: module, name, params, detail. */" > OUT
+    print "#include <stddef.h>" > OUT
+    print "static const char *stdlib_docs[][4] = {" > OUT
+}
+FNR == 1 {
+    module = FILENAME
+    sub(/.*\//, "", module)
+    sub(/\.eigs$/, "", module)
+    ncb = 0
+    modules++
+}
+/^[ \t]*#/ {
+    cb[++ncb] = $0
+    next
+}
+/^define[ \t]+[A-Za-z_][A-Za-z0-9_]*/ {
+    line = $0
+    sub(/^define[ \t]+/, "", line)
+    match(line, /^[A-Za-z_][A-Za-z0-9_]*/)
+    name = substr(line, 1, RLENGTH)
+    rest = substr(line, RLENGTH + 1)
+    params = ""
+    if (substr(rest, 1, 1) == "(") {
+        # No nesting or `is`-defaults in lib parameter lists (verified);
+        # the first ")" closes the list.
+        p = index(rest, ")")
+        params = substr(rest, 2, p - 2)
+    }
+    if (substr(name, 1, 1) != "_") {
+        detail = sig_from_block(name)
+        if (detail == "")
+            detail = "define " name "(" params ") as:"
+        printf "    {\"%s\", \"%s\", \"%s\", \"%s\"},\n", \
+            c_escape(module), c_escape(name), c_escape(params), c_escape(detail) > OUT
+        funcs++
+    }
+    ncb = 0
+    next
+}
+{
+    # Any non-comment, non-define line (including blank) detaches the block.
+    ncb = 0
+}
+END {
+    print "    {NULL, NULL, NULL, NULL}" > OUT
+    print "};" > OUT
+    printf "lsp_stdlib_index: %d public functions across %d lib modules -> %s\n", \
+        funcs, modules, OUT > "/dev/stderr"
+}
+' lib/*.eigs