diff --git a/docs/admin/code-hosts/aws-codecommit.mdx b/docs/admin/code-hosts/aws-codecommit.mdx index 92975a892..1baf5cbcb 100644 --- a/docs/admin/code-hosts/aws-codecommit.mdx +++ b/docs/admin/code-hosts/aws-codecommit.mdx @@ -37,7 +37,7 @@ AWS CodeCommit connections support the following configuration options, which ar {/* SCHEMA_SYNC_START: admin/code_hosts/aws_codecommit.schema.json */} {/* WARNING: This section is auto-generated during releases. Do not edit manually. */} -{/* Last updated: 2026-03-30T11:57:12Z */} +{/* Last updated: 2026-04-14T00:48:03Z */} ```json { // REQUIRED: diff --git a/docs/admin/code-hosts/azuredevops.mdx b/docs/admin/code-hosts/azuredevops.mdx index 60ea02fb2..306e0fcbc 100644 --- a/docs/admin/code-hosts/azuredevops.mdx +++ b/docs/admin/code-hosts/azuredevops.mdx @@ -69,7 +69,7 @@ Azure DevOps connections support the following configuration options, which are {/* SCHEMA_SYNC_START: admin/code_hosts/azuredevops.schema.json */} {/* WARNING: This section is auto-generated during releases. Do not edit manually. */} -{/* Last updated: 2026-03-30T11:57:12Z */} +{/* Last updated: 2026-04-14T00:48:03Z */} ```json // Authentication alternatives: token OR windowsPassword @@ -131,19 +131,23 @@ Azure DevOps connections support the following configuration options, which are // ] "projects": null, - // An array of "org/project/repo" strings specifying which Azure DevOps repositories within a project should be mirrored in Sourcegraph. - "repos": [ - "myorg/my-project/myrepo", // Format: // - "myorg/myproject/myproject", // Format: repository where project and repo share the same name - "myorg/my project/my project" // Format: spaces in project or repository names - ] - // Rate limit applied when making background API requests. "rateLimit": { "enabled": false, "requestsPerHour": 0 }, + // An array of repository "org/project/repo" strings specifying which Azure DevOps repositories to mirror on Sourcegraph. If both 'repos' and 'projects' or 'orgs' are specified, the union of both sets will be synced. + // Other example values: + // - [ + // "myorg/myproject/myrepo" + // ] + // - [ + // "myorg/my project/myrepo", + // "myorg/anotherproject/anotherrepo" + // ] + "repos": null, + // The pattern used to generate the corresponding Sourcegraph repository name for a Azure DevOps repository. // - "{host}" is replaced with the Azure DevOps URL's host (such as dev.azure.com) // - "{orgName}" is replaced with the repository's parent projects owning organization (or collection on DevOps server) diff --git a/docs/admin/code-hosts/bitbucket-cloud.mdx b/docs/admin/code-hosts/bitbucket-cloud.mdx index 6e28423cf..4b988dc3a 100644 --- a/docs/admin/code-hosts/bitbucket-cloud.mdx +++ b/docs/admin/code-hosts/bitbucket-cloud.mdx @@ -116,7 +116,7 @@ Bitbucket Cloud connections support the following configuration options, which a {/* SCHEMA_SYNC_START: admin/code_hosts/bitbucket_cloud.schema.json */} {/* WARNING: This section is auto-generated during releases. Do not edit manually. */} -{/* Last updated: 2026-03-30T11:57:12Z */} +{/* Last updated: 2026-04-14T00:48:03Z */} ```json { // The workspace access token to use when authenticating with Bitbucket Cloud. diff --git a/docs/admin/code-hosts/bitbucket-server.mdx b/docs/admin/code-hosts/bitbucket-server.mdx index 4b32be04a..78d6f9907 100644 --- a/docs/admin/code-hosts/bitbucket-server.mdx +++ b/docs/admin/code-hosts/bitbucket-server.mdx @@ -202,7 +202,7 @@ Bitbucket Server / Bitbucket Data Center connections support the following confi {/* SCHEMA_SYNC_START: admin/code_hosts/bitbucket_server.schema.json */} {/* WARNING: This section is auto-generated during releases. Do not edit manually. */} -{/* Last updated: 2026-03-30T11:57:12Z */} +{/* Last updated: 2026-04-14T00:48:03Z */} ```json // Authentication alternatives: token OR password diff --git a/docs/admin/code-hosts/gerrit.mdx b/docs/admin/code-hosts/gerrit.mdx index 9fd9f2825..dce57c1fa 100644 --- a/docs/admin/code-hosts/gerrit.mdx +++ b/docs/admin/code-hosts/gerrit.mdx @@ -113,7 +113,7 @@ Gerrit connections support the following configuration options, which are specif {/* SCHEMA_SYNC_START: admin/code_hosts/gerrit.schema.json */} {/* WARNING: This section is auto-generated during releases. Do not edit manually. */} -{/* Last updated: 2026-03-30T11:57:12Z */} +{/* Last updated: 2026-04-14T00:48:03Z */} ```json { // If non-null, enforces Gerrit repository permissions. This requires that there is an item in the [site configuration json](https://sourcegraph.com/docs/admin/config/site_config#auth-providers) `auth.providers` field, of type "gerrit" with the same `url` field as specified in this `GerritConnection`. diff --git a/docs/admin/code-hosts/github.mdx b/docs/admin/code-hosts/github.mdx index 93826d807..11cfa94e6 100644 --- a/docs/admin/code-hosts/github.mdx +++ b/docs/admin/code-hosts/github.mdx @@ -454,7 +454,7 @@ GitHub connections support the following configuration options, which are specif {/* SCHEMA_SYNC_START: admin/code_hosts/github.schema.json */} {/* WARNING: This section is auto-generated during releases. Do not edit manually. */} -{/* Last updated: 2026-03-30T11:57:12Z */} +{/* Last updated: 2026-04-14T00:48:03Z */} ```json // Authentication alternatives: token OR gitHubAppDetails OR externalAccount OR useRandomExternalAccount diff --git a/docs/admin/code-hosts/gitlab.mdx b/docs/admin/code-hosts/gitlab.mdx index 0f844657f..a58cac4ee 100644 --- a/docs/admin/code-hosts/gitlab.mdx +++ b/docs/admin/code-hosts/gitlab.mdx @@ -189,7 +189,7 @@ See [Internal rate limits](/admin/code-hosts/rate-limits#internal-rate-limits). {/* SCHEMA_SYNC_START: admin/code_hosts/gitlab.schema.json */} {/* WARNING: This section is auto-generated during releases. Do not edit manually. */} -{/* Last updated: 2026-03-30T11:57:12Z */} +{/* Last updated: 2026-04-14T00:48:03Z */} ```json { // If non-null, enforces GitLab repository permissions. This requires that there be an item in the `auth.providers` field of type "gitlab" with the same `url` field as specified in this `GitLabConnection`. diff --git a/docs/admin/code-hosts/gitolite.mdx b/docs/admin/code-hosts/gitolite.mdx index c3e945707..773b93c5f 100644 --- a/docs/admin/code-hosts/gitolite.mdx +++ b/docs/admin/code-hosts/gitolite.mdx @@ -25,7 +25,7 @@ To connect Gitolite to Sourcegraph: {/* SCHEMA_SYNC_START: admin/code_hosts/gitolite.schema.json */} {/* WARNING: This section is auto-generated during releases. Do not edit manually. */} -{/* Last updated: 2026-03-30T11:57:12Z */} +{/* Last updated: 2026-04-14T00:48:03Z */} ```json { // A list of repositories to never mirror from this Gitolite instance. Supports excluding by exact name ({"name": "foo"}). diff --git a/docs/admin/code-hosts/other.mdx b/docs/admin/code-hosts/other.mdx index 38903208f..d3b73b3d2 100644 --- a/docs/admin/code-hosts/other.mdx +++ b/docs/admin/code-hosts/other.mdx @@ -68,7 +68,7 @@ Repositories must be listed individually: {/* SCHEMA_SYNC_START: admin/code_hosts/other_external_service.schema.json */} {/* WARNING: This section is auto-generated during releases. Do not edit manually. */} -{/* Last updated: 2026-03-30T11:57:12Z */} +{/* Last updated: 2026-04-14T00:48:03Z */} ```json { // A list of repositories to never mirror by name after applying repositoryPathPattern. Supports excluding by exact name ({"name": "myrepo"}) or regular expression ({"pattern": ".*secret.*"}). diff --git a/docs/admin/code-hosts/phabricator.mdx b/docs/admin/code-hosts/phabricator.mdx index 49461a8f0..4fd1b944c 100644 --- a/docs/admin/code-hosts/phabricator.mdx +++ b/docs/admin/code-hosts/phabricator.mdx @@ -76,7 +76,7 @@ The Sourcegraph instance's site admin must [update the `corsOrigin` site config {/* SCHEMA_SYNC_START: admin/code_hosts/phabricator.schema.json */} {/* WARNING: This section is auto-generated during releases. Do not edit manually. */} -{/* Last updated: 2026-03-30T11:57:12Z */} +{/* Last updated: 2026-04-14T00:48:03Z */} ```json { // SSH cipher to use when cloning via SSH. Must be a valid choice from `ssh -Q cipher`. diff --git a/docs/admin/config/settings.mdx b/docs/admin/config/settings.mdx index 3073d85c5..6bf175da4 100644 --- a/docs/admin/config/settings.mdx +++ b/docs/admin/config/settings.mdx @@ -27,7 +27,7 @@ Settings options and their default values are shown below. {/* SCHEMA_SYNC_START: admin/config/settings.schema.json */} {/* WARNING: This section is auto-generated during releases. Do not edit manually. */} -{/* Last updated: 2026-03-30T11:57:12Z */} +{/* Last updated: 2026-04-14T00:48:03Z */} ```json { diff --git a/docs/admin/config/site-config.mdx b/docs/admin/config/site-config.mdx index b2c318347..6d42c02ec 100644 --- a/docs/admin/config/site-config.mdx +++ b/docs/admin/config/site-config.mdx @@ -21,7 +21,7 @@ All site configuration options and their default values are shown below. {/* SCHEMA_SYNC_START: admin/config/site.schema.json */} {/* WARNING: This section is auto-generated during releases. Do not edit manually. */} -{/* Last updated: 2026-03-30T11:57:12Z */} +{/* Last updated: 2026-04-14T00:48:03Z */} ```json { @@ -180,9 +180,6 @@ All site configuration options and their default values are shown below. // Enable/disable sharing of Deep Search conversations via read tokens "deepSearch.sharing.enabled": false, - // Use database-backed worker for Deep Search question processing instead of in-memory worker pool. Enables persistence across frontend restarts. - "deepSearch.worker.enabled": true, - // The rate limit (in requests per hour) for the default rate limiter in the rate limiters registry. By default this is disabled and the default rate limit is infinity. "defaultRateLimit": -1, diff --git a/docs/admin/repo/perforce.mdx b/docs/admin/repo/perforce.mdx index 8cb33dbab..8159ea637 100644 --- a/docs/admin/repo/perforce.mdx +++ b/docs/admin/repo/perforce.mdx @@ -228,7 +228,7 @@ With this setting, Sourcegraph will ignore any rules with a host other than `*`, {/* SCHEMA_SYNC_START: admin/code_hosts/perforce.schema.json */} {/* WARNING: This section is auto-generated during releases. Do not edit manually. */} -{/* Last updated: 2026-03-30T11:57:12Z */} +{/* Last updated: 2026-04-14T00:48:03Z */} ```json { // If non-null, enforces Perforce depot permissions. diff --git a/docs/admin/telemetry/private-metadata-allowlist.mdx b/docs/admin/telemetry/private-metadata-allowlist.mdx index 609fe90f9..78bdcc02a 100644 --- a/docs/admin/telemetry/private-metadata-allowlist.mdx +++ b/docs/admin/telemetry/private-metadata-allowlist.mdx @@ -34,4 +34,5 @@ To learn more, refer to the [telemetry documentation](https://sourcegraph.com/do | `cody.smart-apply.context` | `applied` | `model` — High-cardinality model identifier; helpful for determining the model that was selected. | | `deepsearch` | `search.toolcall` | `toolName` — High-cardinality tool name; helpful for determining which tools are being used during deep search.
`toolId` — High-cardinality tool identifier; helpful for determining which tools are being used during deep search.
`model` — High-cardinality model identifier; helpful for determining which models are being used during deep search. | | `admin.users` | `delete` | `userIDs` — Numeric identifiers of users being deleted; needed for audit and analytics of admin user management actions. | -| `externalApi` | `request` | `procedure` — ConnectRPC procedure path (e.g. '/sourcegraph.users.v1.UsersService/GetUser'); not sensitive and needed to distinguish which external API RPCs are being used. | \ No newline at end of file +| `externalApi` | `request` | `procedure` — ConnectRPC procedure path (e.g. '/sourcegraph.users.v1.UsersService/GetUser'); not sensitive and needed to distinguish which external API RPCs are being used. | +| `metering` | `consume` | `meteringEventId` — The UUID of the billing event in the metering export queue, used to correlate billing events with telemetry events. | \ No newline at end of file diff --git a/docs/admin/telemetry/protocol.mdx b/docs/admin/telemetry/protocol.mdx index 258aa669a..8b25a7a42 100644 --- a/docs/admin/telemetry/protocol.mdx +++ b/docs/admin/telemetry/protocol.mdx @@ -439,6 +439,7 @@ Sourcegraph Analytics features. | username | [string](#string) | |

Username of the user.

| | primary_email | [string](#string) | |

Primary email of the user.

| | assigned_rbac_roles | [string](#string) | repeated |

Assigned RBAC roles for the user.

| +| service_account | [bool](#bool) | |

Whether the user is a service account.

| diff --git a/docs/cli/references/auth.mdx b/docs/cli/references/auth.mdx new file mode 100644 index 000000000..eb5d3834e --- /dev/null +++ b/docs/cli/references/auth.mdx @@ -0,0 +1,22 @@ +# `src auth` + + + +## Usage + +``` +'src auth' provides authentication-related helper commands. + +Usage: + + src auth command [command options] + +The commands are: + + token prints the current authentication token + +Use "src auth [command] -h" for more information about a command. + + +``` + \ No newline at end of file diff --git a/docs/cli/references/index.mdx b/docs/cli/references/index.mdx index 5f606371d..9c0eb3737 100644 --- a/docs/cli/references/index.mdx +++ b/docs/cli/references/index.mdx @@ -5,6 +5,7 @@ * [`admin`](references/admin) * [`api`](references/api) +* [`auth`](references/auth) * [`batch`](references/batch) * [`code-intel`](references/code-intel) * [`codeowners`](references/codeowners) diff --git a/docs/cli/references/login.mdx b/docs/cli/references/login.mdx index a53c76fd7..19a74edaf 100644 --- a/docs/cli/references/login.mdx +++ b/docs/cli/references/login.mdx @@ -19,7 +19,7 @@ Usage: - src login SOURCEGRAPH_URL + src login [flags] SOURCEGRAPH_URL Examples: @@ -31,6 +31,10 @@ Examples: $ src login https://sourcegraph.com + If no access token is configured, 'src login' uses OAuth device flow automatically: + + $ src login https://sourcegraph.com + -dump-requests Log GraphQL requests and responses to stdout -get-curl diff --git a/docs/self-hosted/observability/alerts.mdx b/docs/self-hosted/observability/alerts.mdx index 1909109c1..9034041ff 100644 --- a/docs/self-hosted/observability/alerts.mdx +++ b/docs/self-hosted/observability/alerts.mdx @@ -6546,6 +6546,175 @@ Generated query for warning alert: `max((sum(increase(src_telemetrygatewayexport
+## metering: metering_exporter_queue_growth + +

rate of growth of metering export queue over 30m

+ +**Descriptions** + +- warning metering: 1+ rate of growth of metering export queue over 30m for 1h0m0s +- critical metering: 1+ rate of growth of metering export queue over 30m for 36h0m0s + +**Next steps** + +- Check the "number of metering events exported per batch over 30m" dashboard panel to see if export throughput is at saturation. +- Check worker logs for metering exporter failures and Enterprise Portal connectivity issues. +- More help interpreting this metric is available in the [dashboards reference](dashboards#metering-metering_exporter_queue_growth). +- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert: + +```json +"observability.silenceAlerts": [ + "warning_metering_metering_exporter_queue_growth", + "critical_metering_metering_exporter_queue_growth" +] +``` + +*Managed by the Sourcegraph Services team.* + +
+Technical details + +Generated query for warning alert: `max(((deriv((sum(src_meteringexporter_queue_size and on (instance, job) ((increase(src_meteringexporter_queue_metrics_reporter_total{job=~"^worker.*"\}[15m]) > 0) or on (instance, job) (increase(src_meteringexporter_queue_metrics_reporter_errors_total\{job=~"^worker.*"}[15m]) > 0))))[30m:])) or on () vector(0)) > 1)` + +Generated query for critical alert: `max(((deriv((sum(src_meteringexporter_queue_size and on (instance, job) ((increase(src_meteringexporter_queue_metrics_reporter_total{job=~"^worker.*"\}[15m]) > 0) or on (instance, job) (increase(src_meteringexporter_queue_metrics_reporter_errors_total\{job=~"^worker.*"}[15m]) > 0))))[30m:])) or on () vector(0)) > 1)` + +
+ +
+ +## metering: metering_exporter_oldest_unexported_event_age + +

age of the oldest unexported metering event

+ +**Descriptions** + +- warning metering: 1800s+ age of the oldest unexported metering event for 30m0s +- critical metering: 7200s+ age of the oldest unexported metering event for 2h0m0s + +**Next steps** + +- Check whether the metering export job is failing or making no forward progress. +- Compare this panel with "rate of growth of metering export queue over 30m" and the exporter error panels below. +- Check worker logs and verify the Enterprise Portal is reachable. +- More help interpreting this metric is available in the [dashboards reference](dashboards#metering-metering_exporter_oldest_unexported_event_age). +- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert: + +```json +"observability.silenceAlerts": [ + "warning_metering_metering_exporter_oldest_unexported_event_age", + "critical_metering_metering_exporter_oldest_unexported_event_age" +] +``` + +*Managed by the Sourcegraph Services team.* + +
+Technical details + +Generated query for warning alert: `max(((max(src_meteringexporter_oldest_unexported_event and on (instance, job) ((increase(src_meteringexporter_queue_metrics_reporter_total{job=~"^worker.*"\}[15m]) > 0) or on (instance, job) (increase(src_meteringexporter_queue_metrics_reporter_errors_total\{job=~"^worker.*"}[15m]) > 0)))) or on () vector(0)) > 1800)` + +Generated query for critical alert: `max(((max(src_meteringexporter_oldest_unexported_event and on (instance, job) ((increase(src_meteringexporter_queue_metrics_reporter_total{job=~"^worker.*"\}[15m]) > 0) or on (instance, job) (increase(src_meteringexporter_queue_metrics_reporter_errors_total\{job=~"^worker.*"}[15m]) > 0)))) or on () vector(0)) > 7200)` + +
+ +
+ +## metering: meteringexporter_exporter_errors_total + +

events exporter operation errors every 30m

+ +**Descriptions** + +- warning metering: 0+ events exporter operation errors every 30m + +**Next steps** + +- Failures indicate that exporting credit metering events from Sourcegraph are failing. +- Check worker logs for metering exporter failures and Enterprise Portal connectivity issues. +- Learn more about the related dashboard panel in the [dashboards reference](dashboards#metering-meteringexporter_exporter_errors_total). +- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert: + +```json +"observability.silenceAlerts": [ + "warning_metering_meteringexporter_exporter_errors_total" +] +``` + +*Managed by the Sourcegraph Services team.* + +
+Technical details + +Generated query for warning alert: `max((sum(increase(src_meteringexporter_exporter_errors_total{job=~"^worker.*"}[30m]))) > 0)` + +
+ +
+ +## metering: meteringexporter_queue_cleanup_errors_total + +

events export queue cleanup operation errors every 30m

+ +**Descriptions** + +- warning metering: 0+ events export queue cleanup operation errors every 30m + +**Next steps** + +- Failures indicate that pruning already-exported metering events from the queue is failing. +- Check worker logs for metering exporter cleanup failures. +- Learn more about the related dashboard panel in the [dashboards reference](dashboards#metering-meteringexporter_queue_cleanup_errors_total). +- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert: + +```json +"observability.silenceAlerts": [ + "warning_metering_meteringexporter_queue_cleanup_errors_total" +] +``` + +*Managed by the Sourcegraph Services team.* + +
+Technical details + +Generated query for warning alert: `max((sum(increase(src_meteringexporter_queue_cleanup_errors_total{job=~"^worker.*"}[30m]))) > 0)` + +
+ +
+ +## metering: meteringexporter_queue_metrics_reporter_errors_total + +

events export backlog metrics reporting operation errors every 30m

+ +**Descriptions** + +- warning metering: 0+ events export backlog metrics reporting operation errors every 30m + +**Next steps** + +- Failures indicate that metering export queue metrics are not being refreshed. +- Check worker logs for metering exporter metrics reporter failures. +- Learn more about the related dashboard panel in the [dashboards reference](dashboards#metering-meteringexporter_queue_metrics_reporter_errors_total). +- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert: + +```json +"observability.silenceAlerts": [ + "warning_metering_meteringexporter_queue_metrics_reporter_errors_total" +] +``` + +*Managed by the Sourcegraph Services team.* + +
+Technical details + +Generated query for warning alert: `max((sum(increase(src_meteringexporter_queue_metrics_reporter_errors_total{job=~"^worker.*"}[30m]))) > 0)` + +
+ +
+ ## otel-collector: otel_span_refused

spans refused per receiver

@@ -6815,7 +6984,6 @@ Generated query for critical alert: `max((max(deriv(src_deepsearch_questions_in_ **Next steps** - Check frontend logs for `Worker failed to process question` errors. -- Review LLM stream errors in the `LLM streaming` panel below. - Check for upstream LLM provider issues. - More help interpreting this metric is available in the [dashboards reference](dashboards#deepsearch-deepsearch_question_processing_error_rate). - **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert: @@ -6840,23 +7008,27 @@ Generated query for critical alert: `max((sum(rate(src_deepsearch_question_proce
-## deepsearch: deepsearch_llm_stream_fatal_errors +## deepsearch: deepsearch_slow_questions -

fatal LLM stream errors over 5m

+

questions taking longer than 5 minutes over 15m

**Descriptions** -- warning deepsearch: 20+ fatal LLM stream errors over 5m +- warning deepsearch: 3+ questions taking longer than 5 minutes over 15m +- critical deepsearch: 10+ questions taking longer than 5 minutes over 15m **Next steps** -- Check frontend logs for `fatal error in LLM stream`. -- More help interpreting this metric is available in the [dashboards reference](dashboards#deepsearch-deepsearch_llm_stream_fatal_errors). +- Search frontend logs for questionIDs with long processing times. +- Check `deepsearch_turn_p95_duration` to see if individual turns are slow. +- Check `deepsearch_question_outcome_by_reason` for elevated truncation or error rates. +- More help interpreting this metric is available in the [dashboards reference](dashboards#deepsearch-deepsearch_slow_questions). - **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert: ```json "observability.silenceAlerts": [ - "warning_deepsearch_deepsearch_llm_stream_fatal_errors" + "warning_deepsearch_deepsearch_slow_questions", + "critical_deepsearch_deepsearch_slow_questions" ] ``` @@ -6865,7 +7037,46 @@ Generated query for critical alert: `max((sum(rate(src_deepsearch_question_proce
Technical details -Generated query for warning alert: `max((sum(increase(src_deepsearch_question_processing_errors_total{operation="llm_stream_fatal"}[5m]))) > 20)` +Generated query for warning alert: `max((sum(increase(src_deepsearch_question_processing_duration_seconds_count{operation="question"\}[15m])) - sum(increase(src_deepsearch_question_processing_duration_seconds_bucket\{le="300",operation="question"}[15m]))) > 3)` + +Generated query for critical alert: `max((sum(increase(src_deepsearch_question_processing_duration_seconds_count{operation="question"\}[15m])) - sum(increase(src_deepsearch_question_processing_duration_seconds_bucket\{le="300",operation="question"}[15m]))) > 10)` + +
+ +
+ +## deepsearch: deepsearch_response_truncated + +

response truncated questions over 15m

+ +**Descriptions** + +- warning deepsearch: 3+ response truncated questions over 15m +- critical deepsearch: 10+ response truncated questions over 15m + +**Next steps** + +- Search frontend logs for: "answer truncated due to output token limit" to find affected questionIDs. +- This usually indicates model overthinking and spending all output tokens on thinking with no useful answer, or the user requested an extremely large output. +- Check if a recent model change may have caused increased thinking token usage. +- More help interpreting this metric is available in the [dashboards reference](dashboards#deepsearch-deepsearch_response_truncated). +- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert: + +```json +"observability.silenceAlerts": [ + "warning_deepsearch_deepsearch_response_truncated", + "critical_deepsearch_deepsearch_response_truncated" +] +``` + +*Managed by the Sourcegraph Code Understanding team.* + +
+Technical details + +Generated query for warning alert: `max((sum(increase(src_deepsearch_question_outcome_total{outcome="response_truncated"}[15m]))) > 3)` + +Generated query for critical alert: `max((sum(increase(src_deepsearch_question_outcome_total{outcome="response_truncated"}[15m]))) > 10)`
diff --git a/docs/self-hosted/observability/dashboards.mdx b/docs/self-hosted/observability/dashboards.mdx index 0e5a20306..38290ef9e 100644 --- a/docs/self-hosted/observability/dashboards.mdx +++ b/docs/self-hosted/observability/dashboards.mdx @@ -6205,9 +6205,7 @@ sum(src_gitserver_clone_queue)

Number of concurrent requests running against gitserver client

-This metric is only for informational purposes. It indicates the current number of concurrently running requests by process against gitserver gRPC. - -It does not indicate any problems with the instance, but can give a good indication of load spikes or request throttling. +This metric is only for informational purposes. It indicates the current number of concurrently running requests by process against gitserver gRPC. It does not indicate any problems with the instance, but can give a good indication of load spikes or request throttling. This panel has no related alerts. @@ -6227,6 +6225,30 @@ sum by (job, instance) (src_gitserver_client_concurrent_requests)
+#### gitserver: repos_not_on_primary + +

Cloned repos not on primary shard

+ +The number of cloned repositories currently observed off-primary. For shard relocation operations this should trend towards zero. + +This panel has no related alerts. + +To see this panel, visit `/-/debug/grafana/d/gitserver/gitserver?viewPanel=100032` on your Sourcegraph instance. + +*Managed by the Sourcegraph Services team.* + +
+Technical details + +Query: + +``` +max(src_repoupdater_repos_not_on_primary) +``` +
+ +
+ ### Git Server: Gitserver (CPU, Memory) Gitserver leverages memory mapping to optimize file reads: it is generally expected to consume all the memory provided to it, if it can. When it finds data that is not available in memory yet, this causes a 'page fault', and the data is loaded into memory from disk. @@ -33061,6 +33083,473 @@ sum(increase(src_telemetrygatewayexporter_usermetadata_exporter_errors_total{job
+## Metering + +

Monitoring credit metering export services in Sourcegraph.

+ +To see this dashboard, visit `/-/debug/grafana/d/metering/metering` on your Sourcegraph instance. + +### Metering: Metering exporter: events export and queue metrics + +#### metering: metering_exporter_queue_size + +

Credit metering events pending export

+ +The number of credit metering events queued to be exported. + +This panel has no related alerts. + +To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100000` on your Sourcegraph instance. + +*Managed by the Sourcegraph Services team.* + +
+Technical details + +Query: + +``` +( + sum( + src_meteringexporter_queue_size + and on(instance, job) + ( + ( + increase(src_meteringexporter_queue_metrics_reporter_total{job=~"^worker.*"}[15m]) > 0 + ) + or on(instance, job) + ( + increase(src_meteringexporter_queue_metrics_reporter_errors_total{job=~"^worker.*"}[15m]) > 0 + ) + ) + ) + ) or on() vector(0) +``` +
+ +
+ +#### metering: metering_exporter_queue_growth + +

Rate of growth of metering export queue over 30m

+ +A positive value indicates the metering export queue is growing. + +Refer to the [alerts reference](alerts#metering-metering_exporter_queue_growth) for 2 alerts related to this panel. + +To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100001` on your Sourcegraph instance. + +*Managed by the Sourcegraph Services team.* + +
+Technical details + +Query: + +``` +( + deriv(( + sum( + src_meteringexporter_queue_size + and on(instance, job) + ( + ( + increase(src_meteringexporter_queue_metrics_reporter_total{job=~"^worker.*"}[15m]) > 0 + ) + or on(instance, job) + ( + increase(src_meteringexporter_queue_metrics_reporter_errors_total{job=~"^worker.*"}[15m]) > 0 + ) + ) + ) + )[30m:]) + ) or on() vector(0) +``` +
+ +
+ +#### metering: metering_exporter_oldest_unexported_event_age + +

Age of the oldest unexported metering event

+ +The age in seconds of the oldest metering event still waiting in the export queue. + +Refer to the [alerts reference](alerts#metering-metering_exporter_oldest_unexported_event_age) for 2 alerts related to this panel. + +To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100010` on your Sourcegraph instance. + +*Managed by the Sourcegraph Services team.* + +
+Technical details + +Query: + +``` +( + max( + src_meteringexporter_oldest_unexported_event + and on(instance, job) + ( + ( + increase(src_meteringexporter_queue_metrics_reporter_total{job=~"^worker.*"}[15m]) > 0 + ) + or on(instance, job) + ( + increase(src_meteringexporter_queue_metrics_reporter_errors_total{job=~"^worker.*"}[15m]) > 0 + ) + ) + ) + ) or on() vector(0) +``` +
+ +
+ +#### metering: metering_exporter_exported_events + +

Metering events exported from queue per hour

+ +The number of metering events exported over the last hour. + +This panel has no related alerts. + +To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100011` on your Sourcegraph instance. + +*Managed by the Sourcegraph Services team.* + +
+Technical details + +Query: + +``` +sum(increase(src_meteringexporter_exported_events[1h])) +``` +
+ +
+ +#### metering: metering_exporter_batch_size + +

Number of metering events exported per batch over 30m

+ +The number of metering events exported in each batch. The largest bucket is the maximum number of events exported per batch. +If the distribution trends to the maximum bucket, then export throughput is at or approaching saturation - please contact customer support. + +This panel has no related alerts. + +To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100020` on your Sourcegraph instance. + +*Managed by the Sourcegraph Services team.* + +
+Technical details + +Query: + +``` +sum by (le) (rate(src_meteringexporter_batch_size_bucket[30m])) +``` +
+ +
+ +#### metering: metering_exporter_pruned_events + +

Metering events pruned from queue per hour

+ +The number of already-exported metering events pruned from the queue over the last hour. + +This panel has no related alerts. + +To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100021` on your Sourcegraph instance. + +*Managed by the Sourcegraph Services team.* + +
+Technical details + +Query: + +``` +sum(increase(src_meteringexporter_events_pruned[1h])) +``` +
+ +
+ +### Metering: Metering Exporter: Events export job operations + +#### metering: meteringexporter_exporter_total + +

Events exporter operations every 30m

+ +This panel has no related alerts. + +To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100100` on your Sourcegraph instance. + +*Managed by the Sourcegraph Services team.* + +
+Technical details + +Query: + +``` +sum(increase(src_meteringexporter_exporter_total{job=~"^worker.*"}[30m])) +``` +
+ +
+ +#### metering: meteringexporter_exporter_99th_percentile_duration + +

Aggregate successful events exporter operation duration distribution over 30m

+ +This panel has no related alerts. + +To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100101` on your Sourcegraph instance. + +*Managed by the Sourcegraph Services team.* + +
+Technical details + +Query: + +``` +sum by (le)(rate(src_meteringexporter_exporter_duration_seconds_bucket{job=~"^worker.*"}[30m])) +``` +
+ +
+ +#### metering: meteringexporter_exporter_errors_total + +

Events exporter operation errors every 30m

+ +Refer to the [alerts reference](alerts#metering-meteringexporter_exporter_errors_total) for 1 alert related to this panel. + +To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100102` on your Sourcegraph instance. + +*Managed by the Sourcegraph Services team.* + +
+Technical details + +Query: + +``` +sum(increase(src_meteringexporter_exporter_errors_total{job=~"^worker.*"}[30m])) +``` +
+ +
+ +#### metering: meteringexporter_exporter_error_rate + +

Events exporter operation error rate over 30m

+ +This panel has no related alerts. + +To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100103` on your Sourcegraph instance. + +*Managed by the Sourcegraph Services team.* + +
+Technical details + +Query: + +``` +sum(increase(src_meteringexporter_exporter_errors_total{job=~"^worker.*"}[30m])) / (sum(increase(src_meteringexporter_exporter_total{job=~"^worker.*"}[30m])) + sum(increase(src_meteringexporter_exporter_errors_total{job=~"^worker.*"}[30m]))) * 100 +``` +
+ +
+ +### Metering: Metering Exporter: Events export queue cleanup job operations + +#### metering: meteringexporter_queue_cleanup_total + +

Events export queue cleanup operations every 30m

+ +This panel has no related alerts. + +To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100200` on your Sourcegraph instance. + +*Managed by the Sourcegraph Services team.* + +
+Technical details + +Query: + +``` +sum(increase(src_meteringexporter_queue_cleanup_total{job=~"^worker.*"}[30m])) +``` +
+ +
+ +#### metering: meteringexporter_queue_cleanup_99th_percentile_duration + +

Aggregate successful events export queue cleanup operation duration distribution over 30m

+ +This panel has no related alerts. + +To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100201` on your Sourcegraph instance. + +*Managed by the Sourcegraph Services team.* + +
+Technical details + +Query: + +``` +sum by (le)(rate(src_meteringexporter_queue_cleanup_duration_seconds_bucket{job=~"^worker.*"}[30m])) +``` +
+ +
+ +#### metering: meteringexporter_queue_cleanup_errors_total + +

Events export queue cleanup operation errors every 30m

+ +Refer to the [alerts reference](alerts#metering-meteringexporter_queue_cleanup_errors_total) for 1 alert related to this panel. + +To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100202` on your Sourcegraph instance. + +*Managed by the Sourcegraph Services team.* + +
+Technical details + +Query: + +``` +sum(increase(src_meteringexporter_queue_cleanup_errors_total{job=~"^worker.*"}[30m])) +``` +
+ +
+ +#### metering: meteringexporter_queue_cleanup_error_rate + +

Events export queue cleanup operation error rate over 30m

+ +This panel has no related alerts. + +To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100203` on your Sourcegraph instance. + +*Managed by the Sourcegraph Services team.* + +
+Technical details + +Query: + +``` +sum(increase(src_meteringexporter_queue_cleanup_errors_total{job=~"^worker.*"}[30m])) / (sum(increase(src_meteringexporter_queue_cleanup_total{job=~"^worker.*"}[30m])) + sum(increase(src_meteringexporter_queue_cleanup_errors_total{job=~"^worker.*"}[30m]))) * 100 +``` +
+ +
+ +### Metering: Metering Exporter: Events export queue metrics reporting job operations + +#### metering: meteringexporter_queue_metrics_reporter_total + +

Events export backlog metrics reporting operations every 30m

+ +This panel has no related alerts. + +To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100300` on your Sourcegraph instance. + +*Managed by the Sourcegraph Services team.* + +
+Technical details + +Query: + +``` +sum(increase(src_meteringexporter_queue_metrics_reporter_total{job=~"^worker.*"}[30m])) +``` +
+ +
+ +#### metering: meteringexporter_queue_metrics_reporter_99th_percentile_duration + +

Aggregate successful events export backlog metrics reporting operation duration distribution over 30m

+ +This panel has no related alerts. + +To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100301` on your Sourcegraph instance. + +*Managed by the Sourcegraph Services team.* + +
+Technical details + +Query: + +``` +sum by (le)(rate(src_meteringexporter_queue_metrics_reporter_duration_seconds_bucket{job=~"^worker.*"}[30m])) +``` +
+ +
+ +#### metering: meteringexporter_queue_metrics_reporter_errors_total + +

Events export backlog metrics reporting operation errors every 30m

+ +Refer to the [alerts reference](alerts#metering-meteringexporter_queue_metrics_reporter_errors_total) for 1 alert related to this panel. + +To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100302` on your Sourcegraph instance. + +*Managed by the Sourcegraph Services team.* + +
+Technical details + +Query: + +``` +sum(increase(src_meteringexporter_queue_metrics_reporter_errors_total{job=~"^worker.*"}[30m])) +``` +
+ +
+ +#### metering: meteringexporter_queue_metrics_reporter_error_rate + +

Events export backlog metrics reporting operation error rate over 30m

+ +This panel has no related alerts. + +To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100303` on your Sourcegraph instance. + +*Managed by the Sourcegraph Services team.* + +
+Technical details + +Query: + +``` +sum(increase(src_meteringexporter_queue_metrics_reporter_errors_total{job=~"^worker.*"}[30m])) / (sum(increase(src_meteringexporter_queue_metrics_reporter_total{job=~"^worker.*"}[30m])) + sum(increase(src_meteringexporter_queue_metrics_reporter_errors_total{job=~"^worker.*"}[30m]))) * 100 +``` +
+ +
+ ## OpenTelemetry Collector

The OpenTelemetry collector ingests OpenTelemetry data from Sourcegraph and exports it to the configured backends.

@@ -33625,11 +34114,11 @@ sum(rate(src_deepsearch_question_processing_errors_total{operation="question"}[5
-#### deepsearch: deepsearch_question_processing_p99_duration +#### deepsearch: deepsearch_question_processing_p95_duration -

99th percentile question processing duration

+

95th percentile question processing duration

-99th percentile time to process a deep search question. +95th percentile time to process a deep search question. This panel has no related alerts. @@ -33643,7 +34132,7 @@ To see this panel, visit `/-/debug/grafana/d/deepsearch/deepsearch?viewPanel=100 Query: ``` -histogram_quantile(0.99, sum(rate(src_deepsearch_question_processing_duration_seconds_bucket{operation="question"}[5m])) by (le)) +histogram_quantile(0.95, sum(rate(src_deepsearch_question_processing_duration_seconds_bucket{operation="question"}[5m])) by (le)) ``` @@ -33673,15 +34162,39 @@ histogram_quantile(0.50, sum(rate(src_deepsearch_question_processing_duration_se
-### Deep Search: LLM streaming +#### deepsearch: deepsearch_slow_questions + +

Questions taking longer than 5 minutes over 15m

+ +Number of deep search questions that took longer than 5 minutes in the last 15 minutes. + +Refer to the [alerts reference](alerts#deepsearch-deepsearch_slow_questions) for 2 alerts related to this panel. + +To see this panel, visit `/-/debug/grafana/d/deepsearch/deepsearch?viewPanel=100030` on your Sourcegraph instance. + +*Managed by the Sourcegraph Code Understanding team.* + +
+Technical details + +Query: + +``` +sum(increase(src_deepsearch_question_processing_duration_seconds_count{operation="question"}[15m])) - sum(increase(src_deepsearch_question_processing_duration_seconds_bucket{le="300",operation="question"}[15m])) +``` +
+ +
+ +### Deep Search: Turn processing -#### deepsearch: deepsearch_llm_stream_fatal_errors +#### deepsearch: deepsearch_turn_p95_duration -

Fatal LLM stream errors over 5m

+

95th percentile turn duration

-Number of fatal errors during LLM streaming in the last 5 minutes. +95th percentile duration of individual agent turns. A turn includes the LLM call and any tool execution. Useful for investigating when question-level alerts fire. -Refer to the [alerts reference](alerts#deepsearch-deepsearch_llm_stream_fatal_errors) for 1 alert related to this panel. +This panel has no related alerts. To see this panel, visit `/-/debug/grafana/d/deepsearch/deepsearch?viewPanel=100100` on your Sourcegraph instance. @@ -33693,17 +34206,17 @@ To see this panel, visit `/-/debug/grafana/d/deepsearch/deepsearch?viewPanel=100 Query: ``` -sum(increase(src_deepsearch_question_processing_errors_total{operation="llm_stream_fatal"}[5m])) +histogram_quantile(0.95, sum(rate(src_deepsearch_turn_duration_seconds_bucket[5m])) by (le)) ```
-#### deepsearch: deepsearch_llm_stream_non_fatal_errors +#### deepsearch: deepsearch_turn_p50_duration -

Non-fatal LLM stream errors over 5m

+

50th percentile turn duration

-Number of non-fatal errors during LLM streaming. +Median duration of individual agent turns. This panel has no related alerts. @@ -33717,7 +34230,129 @@ To see this panel, visit `/-/debug/grafana/d/deepsearch/deepsearch?viewPanel=100 Query: ``` -sum(increase(src_deepsearch_question_processing_errors_total{operation="llm_stream_non_fatal"}[5m])) +histogram_quantile(0.50, sum(rate(src_deepsearch_turn_duration_seconds_bucket[5m])) by (le)) +``` + + +
+ +### Deep Search: Question outcomes + +#### deepsearch: deepsearch_question_outcome_by_reason + +

Question outcomes by reason over 5m

+ +Breakdown of deep search question outcomes by reason and whether the question was billable. + +This panel has no related alerts. + +To see this panel, visit `/-/debug/grafana/d/deepsearch/deepsearch?viewPanel=100200` on your Sourcegraph instance. + +*Managed by the Sourcegraph Code Understanding team.* + +
+Technical details + +Query: + +``` +sum by (outcome, billable) (rate(src_deepsearch_question_outcome_total[5m])) * 60 +``` +
+ +
+ +#### deepsearch: deepsearch_errored_but_billed_rate + +

Errored but billed rate over 5m

+ +Percentage of billed deep search questions that completed with an error. Use alongside count-based panels for cross-referencing scale of issues. + +This panel has no related alerts. + +To see this panel, visit `/-/debug/grafana/d/deepsearch/deepsearch?viewPanel=100201` on your Sourcegraph instance. + +*Managed by the Sourcegraph Code Understanding team.* + +
+Technical details + +Query: + +``` +sum(rate(src_deepsearch_question_outcome_total{billable="true",error="true"}[5m])) / (sum(rate(src_deepsearch_question_outcome_total{billable="true"}[5m])) > 0) * 100 +``` +
+ +
+ +#### deepsearch: deepsearch_billable_failures + +

Billable failures over time

+ +Number of billed deep search questions that completed with an error (token limit exceeded or response truncated) in the last 15 minutes. + +This panel has no related alerts. + +To see this panel, visit `/-/debug/grafana/d/deepsearch/deepsearch?viewPanel=100210` on your Sourcegraph instance. + +*Managed by the Sourcegraph Code Understanding team.* + +
+Technical details + +Query: + +``` +sum by (outcome) (increase(src_deepsearch_question_outcome_total{billable="true",error="true"}[15m])) +``` +
+ +
+ +#### deepsearch: deepsearch_response_truncated + +

Response truncated questions over 15m

+ +Number of deep search questions where the response was truncated due to hitting the output token limit in the last 15 minutes. + +Refer to the [alerts reference](alerts#deepsearch-deepsearch_response_truncated) for 2 alerts related to this panel. + +To see this panel, visit `/-/debug/grafana/d/deepsearch/deepsearch?viewPanel=100220` on your Sourcegraph instance. + +*Managed by the Sourcegraph Code Understanding team.* + +
+Technical details + +Query: + +``` +sum(increase(src_deepsearch_question_outcome_total{outcome="response_truncated"}[15m])) +``` +
+ +
+ +#### deepsearch: deepsearch_token_limit_exceeded + +

Token limit exceeded questions over 15m

+ +Number of deep search questions that exceeded the conversation token limit in the last 15 minutes. + +This panel has no related alerts. + +To see this panel, visit `/-/debug/grafana/d/deepsearch/deepsearch?viewPanel=100221` on your Sourcegraph instance. + +*Managed by the Sourcegraph Code Understanding team.* + +
+Technical details + +Query: + +``` +sum(increase(src_deepsearch_question_outcome_total{outcome="token_limit_exceeded"}[15m])) ```