From a56199dbde37bacaaa7a738da2fb668e94937254 Mon Sep 17 00:00:00 2001 From: Esteban Beltran Date: Mon, 30 Mar 2026 10:26:26 +0200 Subject: [PATCH] Optimize LLM review prompts for concise output --- pkg/analysis/passes/llmreview/llmreview.go | 34 +++++++++++----------- pkg/llmclient/agentic_tools.go | 4 +-- pkg/llmvalidate/llmvalidate.go | 21 ++++++++++++- 3 files changed, 39 insertions(+), 20 deletions(-) diff --git a/pkg/analysis/passes/llmreview/llmreview.go b/pkg/analysis/passes/llmreview/llmreview.go index eaf8089e..dacd1345 100644 --- a/pkg/analysis/passes/llmreview/llmreview.go +++ b/pkg/analysis/passes/llmreview/llmreview.go @@ -80,71 +80,71 @@ var Analyzer = &analysis.Analyzer{ var Questions = []llmvalidate.LLMQuestion{ { - Question: "Only for go/golang code: Does this code directly read from or write to the file system? (Look for uses of os.Open, os.Create, ioutil.ReadFile, ioutil.WriteFile, etc.). Provide the specific code snippet if found.", + Question: "Only for go/golang code: Does this code directly read from or write to the file system? (Look for os.Open, os.Create, ioutil.ReadFile, ioutil.WriteFile, etc.)", ExpectedAnswer: false, }, { - Question: "Does this code execute user input as code in a browser environment? (Look for eval(), new Function(), document.write() with unescaped content, innerHTML with script tags, etc.). Provide the specific code snippet if found.", + Question: "Does this code execute user input as code in a browser environment? (Look for eval(), new Function(), document.write() with unescaped content, innerHTML with script tags, etc.)", ExpectedAnswer: false, }, { - Question: "Only for go/golang code: Does this code execute user input as commands or code in the backend? (Look for exec.Command, syscall.Exec, template.Execute with user data, etc.). Provide the specific code snippet if found.", + Question: "Only for go/golang code: Does this code execute user input as commands or code in the backend? (Look for exec.Command, syscall.Exec, template.Execute with user data, etc.)", ExpectedAnswer: false, }, { - Question: "Does this code introduce third-party analytics or tracking features? (Grafana's reportInteraction from @grafana/runtime is allowed, but external services like Google Analytics, Mixpanel, etc. are not). Provide the specific code snippet if found.", + Question: "Does this code introduce third-party analytics or tracking features? (Grafana's reportInteraction from @grafana/runtime is allowed, but external services like Google Analytics, Mixpanel, etc. are not.)", ExpectedAnswer: false, }, { - Question: "Does this code modify or create properties on the global window object? (Look for direct assignments like window.customVariable = x, window.functionName = function(){}, or adding undeclared variables in global scope). Exclude standard browser API usage. Provide the specific code snippet if found.", + Question: "Does this code modify or create properties on the global window object? (Look for direct assignments like window.customVariable = x, window.functionName = function(){}. Exclude standard browser API usage.)", ExpectedAnswer: false, }, { - Question: "Does this code introduce global CSS not scoped to components? (Emotion CSS and CSS modules are allowed, but look for direct style tags, global class definitions, or modification of document.styleSheets). Provide the specific code snippet if found.", + Question: "Does this code introduce global CSS not scoped to components? (Emotion CSS and CSS modules are allowed. Look for direct style tags, global class definitions, or document.styleSheets modification.)", ExpectedAnswer: false, }, { - Question: "Does this code dynamically inject external third-party scripts? (Look for createElement('script'), setting src attributes to external domains, document.write with script tags, or dynamic import() from external sources). Provide the specific code snippet with the external URL if found.", + Question: "Does this code dynamically inject external third-party scripts? (Look for createElement('script') with external src, document.write with script tags, or dynamic import() from external sources.)", ExpectedAnswer: false, }, { - Question: "Only for go/golang code: Are there any opened resources that are NOT properly closed? (Check for files, network connections, etc. that lack proper closure with defer, in finally blocks, or using 'with' statements). Identify any improperly closed resources with a code snippet. If there is no backend code reply negatively", + Question: "Only for go/golang code: Are there any opened resources (files, connections) NOT properly closed with defer? If there is no backend code, answer No.", ExpectedAnswer: false, }, { - Question: "Does this code use global DOM selectors outside of component lifecycle methods? (Look for direct usage of document.querySelector(), document.getElementById(), document.getElementsByClassName(), etc. that aren't scoped to specific components or that bypass React refs). Component-scoped element access like useRef() or this.elementRef is acceptable. Provide the specific code snippet showing the global access if found.", + Question: "Does this code use global DOM selectors outside of component lifecycle methods? (Look for document.querySelector(), document.getElementById(), etc. not scoped to components. useRef() and this.elementRef are acceptable.)", ExpectedAnswer: false, }, { - Question: "Only for go/golang code: Does this code create HTTP clients without using github.com/grafana/grafana-plugin-sdk-go/backend/httpclient? (Look for direct creation of http.Client{}, http.NewRequest, calls to third-party NewClient/NewHTTPClient functions that don't accept or use the SDK's httpclient, or any other HTTP client initialization that doesn't use github.com/grafana/grafana-plugin-sdk-go/backend/httpclient. The httpclient from github.com/grafana/grafana-plugin-sdk-go/backend/httpclient should be used directly or passed to the HTTP client being created. This includes cases where third-party libraries create HTTP clients internally - those libraries should accept the SDK's httpclient as a parameter). Provide the specific code snippet if found.", + Question: "Only for go/golang code: Does this code create HTTP clients without using github.com/grafana/grafana-plugin-sdk-go/backend/httpclient? (Look for direct http.Client{}, http.NewRequest, or third-party client creation that doesn't use or accept the SDK httpclient.)", ExpectedAnswer: false, }, { - Question: "Does this code log sensitive information such as credentials, tokens, passwords, API keys, request bodies, or full request/response objects at INFO level or higher? (These should use DEBUG level only). Provide the specific code snippet if found.", + Question: "Does this code log sensitive information (credentials, tokens, passwords, API keys, request/response bodies) at INFO level or higher? (These should use DEBUG level only.)", ExpectedAnswer: false, }, { - Question: "Only for go/golang code: Does this code use incorrect log formatting? (Look for patterns like `log.Info(\"message\", err)` instead of the correct `log.Info(\"message\", \"error\", err)` with key-value pairs, or logging that produces 'EXTRA_VALUE_AT_END' in output). Provide the specific code snippet if found.", + Question: "Only for go/golang code: Does this code use incorrect log formatting? (Look for `log.Info(\"message\", err)` instead of `log.Info(\"message\", \"error\", err)` with key-value pairs.)", ExpectedAnswer: false, }, { - Question: "Does this code render user-supplied or dynamic content as HTML without sanitization? (Look for dangerouslySetInnerHTML without DOMPurify, d3.html() with user data, innerHTML assignments, or markdown-it with html:true without sanitization). Provide the specific code snippet if found.", + Question: "Does this code render user-supplied or dynamic content as HTML without sanitization? (Look for dangerouslySetInnerHTML without DOMPurify, innerHTML assignments, or markdown-it with html:true without sanitization.)", ExpectedAnswer: false, }, { - Question: "Only for go/golang code: Does this code use panic() for error handling instead of returning errors properly? (panic should only be used for truly unrecoverable situations, not for regular error handling). Provide the specific code snippet if found.", + Question: "Only for go/golang code: Does this code use panic() for error handling instead of returning errors? (panic should only be used for truly unrecoverable situations.)", ExpectedAnswer: false, }, { - Question: "Does this code use localStorage or sessionStorage with generic key names (not namespaced with the plugin ID) that could conflict with Grafana core or other plugins? Provide the specific code snippet if found.", + Question: "Does this code use localStorage or sessionStorage with generic key names not namespaced with the plugin ID?", ExpectedAnswer: false, }, { - Question: "For plugins that have multiple plugin.json files: Are the grafanaDependency values inconsistent across them? (The grafanaDependency property must be consistent across all plugins). Provide the specific plugin.json files and their grafanaDependency values if found to be inconsistent.", + Question: "For plugins with multiple plugin.json files: Are the grafanaDependency values inconsistent across them?", ExpectedAnswer: false, }, { - Question: "Only for go/golang code: Does this code access attributes or methods of a returned value before checking if it is nil? (Code that accesses returned values must be moved after error/nil checks to prevent nil pointer dereference crashes. For example, if a function returns `(req *Request, err error)`, code accessing `req` should be after checking `if err != nil` or `if req == nil`). Provide the specific code snippet showing the unsafe access if found.", + Question: "Only for go/golang code: Does this code access attributes or methods of a returned value before checking if it is nil? (e.g., accessing `req` before checking `if err != nil` or `if req == nil`.)", ExpectedAnswer: false, }, } diff --git a/pkg/llmclient/agentic_tools.go b/pkg/llmclient/agentic_tools.go index 71610ba1..d84b6bf9 100644 --- a/pkg/llmclient/agentic_tools.go +++ b/pkg/llmclient/agentic_tools.go @@ -135,7 +135,7 @@ func submitAnswerTool() llmprovider.Tool { "properties": map[string]interface{}{ "answer": map[string]interface{}{ "type": "string", - "description": "Your detailed answer explaining your findings", + "description": "A brief answer: start with Yes/No, then ONE sentence explaining the finding. Max 2-3 sentences total. Do not include code here (use code_snippet instead). Do not suggest fixes, mitigations, or recommendations — report findings only.", }, "short_answer": map[string]interface{}{ "type": "boolean", @@ -148,7 +148,7 @@ func submitAnswerTool() llmprovider.Tool { }, "code_snippet": map[string]interface{}{ "type": "string", - "description": "A relevant code snippet. Pass an empty string if not relevant.", + "description": "The single most relevant code snippet (max 5 lines). Pick the clearest example if the issue appears in multiple places. Pass empty string if answer is No.", }, }, "required": []string{"answer", "short_answer", "files", "code_snippet"}, diff --git a/pkg/llmvalidate/llmvalidate.go b/pkg/llmvalidate/llmvalidate.go index 229a3a37..dff61792 100644 --- a/pkg/llmvalidate/llmvalidate.go +++ b/pkg/llmvalidate/llmvalidate.go @@ -87,7 +87,26 @@ REVIEWER NOTE: Ignore code that exists only for testing or development: - Dockerfiles, makefiles, bash scripts - Files clearly not part of the plugin -Focus your review on production code that will run as part of a Grafana Plugin.` +Focus your review on production code that will run as part of a Grafana Plugin. + +RESPONSE FORMAT: Be extremely concise. This is a purely investigative task — report findings only. +1. Start with "Yes" or "No". +2. If Yes, add ONE sentence explaining the issue. +3. For code_snippet, include ONLY the single most relevant snippet (max 5 lines). Do NOT repeat similar patterns — if the same issue appears in multiple places, just list the files. +4. Never include full function bodies. Show only the specific problematic line(s). + +Do NOT: +- Suggest fixes or improvements +- Explain how to resolve the issue +- Discuss mitigations, workarounds, or risk levels +- Offer recommendations or next steps +- Provide context about why the pattern is problematic +- Qualify findings with "however" or "the risk is mitigated by" + +Your sole job is to answer Yes/No and point to the evidence. Nothing more. + +Example good answer: "Yes, user input flows into template.Execute via BuildUserPrompt." +Example bad answer: A multi-paragraph explanation with full function bodies, mitigations, suggestions, and repeated code blocks.` type Client struct { provider string