Add per-tenant alert generator URL template for customizable alert source links

docs/configuration/config-file-reference.md

@@ -4363,6 +4363,17 @@ query_rejection:
 # external labels for alerting rules
 [ruler_external_labels: <map of string (labelName) to string (labelValue)> | default = []]
 
+# Per-tenant external URL for the ruler. If set, it overrides the global
+# -ruler.external.url for this tenant's alert notifications.
+[ruler_external_url: <string> | default = ""]
+
+# Go text/template for alert generator URLs. Available variables: .ExternalURL
+# (resolved external URL) and .Expression (PromQL expression). Built-in
+# functions like urlquery are available. If empty, uses default Prometheus
+# /graph format. Example for a custom explore link:
+# "{{ .ExternalURL }}/explore?expr={{ urlquery .Expression }}"
+[ruler_alert_generator_url_template: <string> | default = ""]
+
 # Enable to allow rules to be evaluated with data from a single zone, if other
 # zones are not available.
 [rules_partial_data: <boolean> | default = false]

docs/getting-started/.env

@@ -2,4 +2,4 @@ CORTEX_VERSION=v1.20.1
 GRAFANA_VERSION=10.4.2
 PROMETHEUS_VERSION=v3.2.1
 SEAWEEDFS_VERSION=3.67
-PERSES_VERSION=v0.49-distroless-debug
+PERSES_VERSION=v0.53.1-distroless-debug

docs/getting-started/cortex-config.yaml

@@ -82,6 +82,14 @@ frontend_worker:
 # https://cortexmetrics.io/docs/configuration/configuration-file/#ruler_config
 ruler:
   enable_api: true
+  external_url: http://localhost:9009
+  alertmanager_url: http://localhost:9009/alertmanager
+
+# Per-tenant runtime configuration (hot-reloaded without restart).
+# This file configures per-tenant overrides such as custom alert generator
+# URL templates for Grafana, Perses, or any metrics explorer.
+runtime_config:
+  file: /config/runtime-config.yaml
 
 # https://cortexmetrics.io/docs/configuration/configuration-file/#ruler_storage_config
 ruler_storage:

docs/getting-started/docker-compose.yaml

@@ -17,6 +17,7 @@ services:
       - -config.file=/config/cortex-config.yaml
     volumes:
       - ./cortex-config.yaml:/config/cortex-config.yaml:ro
+      - ./runtime-config.yaml:/config/runtime-config.yaml:ro
     ports:
       - "9009:9009"
     healthcheck:
@@ -47,6 +48,8 @@ services:
     volumes:
       - ./perses/config.yaml:/etc/perses/config/config.yaml:ro
       - ./perses/datasource.yaml:/etc/perses/resources/datasource.yaml:ro
+      - ./perses/datasource-tenant-a.yaml:/etc/perses/resources/datasource-tenant-a.yaml:ro
+      - ./perses/datasource-tenant-b.yaml:/etc/perses/resources/datasource-tenant-b.yaml:ro
       - ./perses/project.yaml:/etc/perses/resources/project.yaml:ro
       - ./perses/dashboards/cortex-writes.yaml:/etc/perses/resources/cortex-writes.yaml:ro
   prometheus:

docs/getting-started/grafana-datasource-docker.yaml

@@ -5,6 +5,7 @@ apiVersion: 1
 datasources:
   - name: Cortex
     type: prometheus
+    uid: cortex
     access: proxy
     orgId: 1
     url: http://cortex:9009/api/prom
@@ -71,3 +72,25 @@ datasources:
     secureJsonData:
       httpHeaderValue1: cortex
     version: 1
+  - orgId: 1
+    name: Tenant A Alertmanager
+    type: alertmanager
+    access: proxy
+    url: http://cortex:9009/
+    jsonData:
+      httpHeaderName1: X-Scope-OrgID
+      implementation: cortex
+    secureJsonData:
+      httpHeaderValue1: tenant-a
+    version: 1
+  - orgId: 1
+    name: Tenant B Alertmanager
+    type: alertmanager
+    access: proxy
+    url: http://cortex:9009/
+    jsonData:
+      httpHeaderName1: X-Scope-OrgID
+      implementation: cortex
+    secureJsonData:
+      httpHeaderValue1: tenant-b
+    version: 1

docs/getting-started/perses/config.yaml

@@ -8,14 +8,19 @@ security:
 database:
   file:
     extension: yaml
-    folder: /perses
+    folder: /tmp/perses-data
 
 schemas:
   datasources_path: /etc/perses/cue/schemas/datasources
   interval: 5m
   panels_path: /etc/perses/cue/schemas/panels
   queries_path: /etc/perses/cue/schemas/queries
   variables_path: /etc/perses/cue/schemas/variables
+
+frontend:
+  explorer:
+    enable: true
+
 provisioning:
   folders:
     - /etc/perses/resources

docs/getting-started/perses/dashboards/cortex-writes.yaml

@@ -4,7 +4,7 @@ metadata:
     createdAt: 2025-03-24T19:15:47.468680767Z
     updatedAt: 2025-03-24T19:43:53.000136362Z
     version: 12
-    project: default
+    project: cortex
 spec:
     display:
         name: Cortex / Writes

docs/getting-started/perses/datasource-tenant-a.yaml

@@ -0,0 +1,14 @@
+kind: GlobalDatasource
+metadata:
+  name: TenantA
+spec:
+  default: false
+  plugin:
+    kind: PrometheusDatasource
+    spec:
+      proxy:
+        kind: HTTPProxy
+        spec:
+          url: http://cortex:9009/api/prom
+          headers:
+            X-Scope-OrgID: tenant-a

docs/getting-started/perses/datasource-tenant-b.yaml

@@ -0,0 +1,14 @@
+kind: GlobalDatasource
+metadata:
+  name: TenantB
+spec:
+  default: false
+  plugin:
+    kind: PrometheusDatasource
+    spec:
+      proxy:
+        kind: HTTPProxy
+        spec:
+          url: http://cortex:9009/api/prom
+          headers:
+            X-Scope-OrgID: tenant-b

docs/getting-started/perses/project.yaml

@@ -1,6 +1,6 @@
 kind: Project
 metadata:
-  name: default
+  name: cortex
 spec:
   display:
-    name: "default"
+    name: "Cortex"

docs/getting-started/runtime-config.yaml

@@ -0,0 +1,25 @@
+# Runtime configuration with per-tenant overrides.
+# This file is hot-reloaded by Cortex without requiring a restart.
+#
+# The examples below demonstrate per-tenant alert generator URL templates.
+# Each tenant can have a different URL format for alert "Source" links.
+
+overrides:
+  # Tenant using Grafana Explore for alert generator URLs.
+  # Clicking "Source" on an alert in Alertmanager opens Grafana Explore
+  # with the PromQL expression pre-filled.
+  tenant-a:
+    ruler_external_url: "http://localhost:3000"
+    ruler_alert_generator_url_template: >-
+      {{ .ExternalURL }}/explore?schemaVersion=1&panes=%7B%22default%22:%7B%22datasource%22:%22cortex%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22{{ urlquery .Expression }}%22%7D%5D,%22range%22:%7B%22from%22:%22now-1h%22,%22to%22:%22now%22%7D%7D%7D&orgId=1
+
+  # Tenant using Perses for alert generator URLs.
+  # Clicking "Source" on an alert opens Perses explore view with
+  # the PromQL expression pre-filled and the TenantB datasource selected.
+  tenant-b:
+    ruler_external_url: http://localhost:8080
+    ruler_alert_generator_url_template: >-
+      {{ .ExternalURL }}/explore?explorer=Prometheus-PrometheusExplorer&data=%7B%22tab%22%3A%22graph%22%2C%22queries%22%3A%5B%7B%22kind%22%3A%22TimeSeriesQuery%22%2C%22spec%22%3A%7B%22plugin%22%3A%7B%22kind%22%3A%22PrometheusTimeSeriesQuery%22%2C%22spec%22%3A%7B%22datasource%22%3A%7B%22kind%22%3A%22PrometheusDatasource%22%2C%22name%22%3A%22tenantb%22%7D%2C%22query%22%3A%22{{ urlquery .Expression }}%22%7D%7D%7D%7D%5D%7D
+
+  # Tenants without overrides use the global ruler.external.url
+  # and the default Prometheus /graph format.

docs/getting-started/single-binary.md

@@ -214,6 +214,133 @@ docker run --network cortex-docs-getting-started_default \
 
 Configure Alertmanager notification policies in Grafana: [Alerting → Notification policies](http://localhost:3000/alerting/notifications?search=&alertmanager=Cortex%20Alertmanager)
 
+## Step 7: Per-Tenant Alert Generator URLs (Optional)
+
+Cortex supports customizing the "Source" link on alerts per-tenant using Go `text/template` strings. This lets each tenant's alerts link back to their preferred metrics explorer — Grafana Explore, Perses, or any other tool.
+
+The getting-started example includes a `runtime-config.yaml` with two tenant configurations:
+- **tenant-a**: Alert source links point to **Grafana Explore**
+- **tenant-b**: Alert source links point to **Perses**
+
+### How It Works
+
+The `ruler_alert_generator_url_template` field accepts a Go template with two variables:
+- `{{ .ExternalURL }}` — the resolved external URL for this tenant (set via `ruler_external_url`)
+- `{{ .Expression }}` — the PromQL expression that triggered the alert
+
+Built-in Go template functions like `urlquery` are available for URL encoding.
+
+Example for Grafana Explore:
+```yaml
+ruler_external_url: "http://localhost:3000"
+ruler_alert_generator_url_template: >-
+  {{ .ExternalURL }}/explore?expr={{ urlquery .Expression }}
+```
+
+### Try It Out
+
+1. **Load alertmanager configs** for tenant-a and tenant-b:
+
+```sh
+# Upload alertmanager config for tenant-a
+curl -X POST http://localhost:9009/api/v1/alerts \
+  -H "X-Scope-OrgID: tenant-a" \
+  -H "Content-Type: application/yaml" \
+  --data-binary @- <<'EOF'
+alertmanager_config: |
+  receivers:
+    - name: default-receiver
+  route:
+    receiver: default-receiver
+    group_wait: 5s
+    group_interval: 10s
+EOF
+
+# Upload alertmanager config for tenant-b
+curl -X POST http://localhost:9009/api/v1/alerts \
+  -H "X-Scope-OrgID: tenant-b" \
+  -H "Content-Type: application/yaml" \
+  --data-binary @- <<'EOF'
+alertmanager_config: |
+  receivers:
+    - name: default-receiver
+  route:
+    receiver: default-receiver
+    group_wait: 5s
+    group_interval: 10s
+EOF
+```
+
+2. **Load demo alert rules** that fire immediately:
+
+```sh
+# Alert rules for tenant-a
+curl -X POST http://localhost:9009/api/v1/rules/demo \
+  -H "X-Scope-OrgID: tenant-a" \
+  -H "Content-Type: application/yaml" \
+  --data-binary @- <<'EOF'
+name: demo_alerts
+interval: 10s
+rules:
+  - alert: HighMemoryUsage
+    expr: vector(85) > 80
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: "Memory usage is above 80%"
+  - alert: HighErrorRate
+    expr: vector(5.2) > 5
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: "Error rate exceeds 5%"
+EOF
+
+# Alert rules for tenant-b
+curl -X POST http://localhost:9009/api/v1/rules/demo \
+  -H "X-Scope-OrgID: tenant-b" \
+  -H "Content-Type: application/yaml" \
+  --data-binary @- <<'EOF'
+name: demo_alerts
+interval: 10s
+rules:
+  - alert: DiskSpaceLow
+    expr: vector(92) > 90
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: "Disk space usage above 90%"
+  - alert: HighLatency
+    expr: vector(3.5) > 2
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: "P99 latency exceeds 2s"
+EOF
+```
+
+3. **Wait ~30 seconds** for the ruler to evaluate rules and send alerts to the alertmanager.
+
+4. **View alerts in Grafana** at [Alerting → Alert groups](http://localhost:3000/alerting/groups?groupBy=alertname):
+   - Select the **Tenant A Alertmanager** datasource — click "See source" to open Grafana Explore
+   - Select the **Tenant B Alertmanager** datasource — click "See source" to open Perses
+
+5. **Verify generator URLs** via the API:
+
+```sh
+# Tenant A: Grafana Explore URLs
+curl -s "http://localhost:9009/alertmanager/api/v2/alerts" \
+  -H "X-Scope-OrgID: tenant-a" | jq '.[].generatorURL'
+
+# Tenant B: Perses URLs
+curl -s "http://localhost:9009/alertmanager/api/v2/alerts" \
+  -H "X-Scope-OrgID: tenant-b" | jq '.[].generatorURL'
+```
+
 ## Explore and Experiment
 
 Now that everything is running, try these experiments to learn how Cortex works:
@@ -306,6 +433,7 @@ This setup uses several configuration files. Here's what each does:
 |----------------------------------|---------------------------------------------------------------|
 | `docker-compose.yaml`            | Defines all services (Cortex, Prometheus, Grafana, SeaweedFS) |
 | `cortex-config.yaml`             | Cortex configuration (storage, limits, components)            |
+| `runtime-config.yaml`            | Per-tenant runtime overrides (alert generator URL templates)  |
 | `prometheus-config.yaml`         | Prometheus configuration with remote_write to Cortex          |
 | `grafana-datasource-docker.yaml` | Grafana datasource pointing to Cortex                         |
 | `rules.yaml`                     | Example recording rules                                       |

pkg/ruler/compat.go

@@ -19,6 +19,7 @@ import (
 	"github.com/prometheus/prometheus/promql"
 	"github.com/prometheus/prometheus/rules"
 	"github.com/prometheus/prometheus/storage"
+	"github.com/prometheus/prometheus/util/strutil"
 	"github.com/weaveworks/common/httpgrpc"
 	"github.com/weaveworks/common/user"
 
@@ -173,6 +174,8 @@ type RulesLimits interface {
 	RulerQueryOffset(userID string) time.Duration
 	DisabledRuleGroups(userID string) validation.DisabledRuleGroups
 	RulerExternalLabels(userID string) labels.Labels
+	RulerExternalURL(userID string) string
+	RulerAlertGeneratorURLTemplate(userID string) string
 }
 
 type QueryExecutor func(ctx context.Context, qs string, t time.Time) (promql.Vector, error)
@@ -378,11 +381,26 @@ func DefaultTenantManagerFactory(cfg Config, p Pusher, q storage.Queryable, engi
 			Appendable: NewPusherAppendable(p, userID, overrides,
 				evalMetrics.TotalWritesVec.WithLabelValues(userID),
 				evalMetrics.FailedWritesVec.WithLabelValues(userID)),
-			Queryable:              q,
-			QueryFunc:              queryFunc,
-			Context:                prometheusContext,
-			ExternalURL:            cfg.ExternalURL.URL,
-			NotifyFunc:             SendAlerts(notifier, cfg.ExternalURL.URL.String()),
+			Queryable:   q,
+			QueryFunc:   queryFunc,
+			Context:     prometheusContext,
+			ExternalURL: cfg.ExternalURL.URL,
+			NotifyFunc: SendAlerts(notifier, func(expr string) string {
+				externalURL := cfg.ExternalURL.String()
+				if tenantURL := overrides.RulerExternalURL(userID); tenantURL != "" {
+					externalURL = tenantURL
+				}
+				tmplStr := overrides.RulerAlertGeneratorURLTemplate(userID)
+				if tmplStr == "" {
+					return externalURL + strutil.TableLinkForExpression(expr)
+				}
+				result, err := executeGeneratorURLTemplate(tmplStr, externalURL, expr)
+				if err != nil {
+					level.Warn(logger).Log("msg", "failed to execute generator URL template, falling back to prometheus format", "err", err)
+					return externalURL + strutil.TableLinkForExpression(expr)
+				}
+				return result
+			}),
 			Logger:                 util_log.GoKitLogToSlog(log.With(logger, "user", userID)),
 			Registerer:             reg,
 			OutageTolerance:        cfg.OutageTolerance,