Let a TerdutServer turn on metrics from the alerts' Prometheus
terdut-server v0.49.0 graphs the metric behind each alert on the incident page, but only queries hosts listed in TERDUT_PROMETHEUS_ALLOWED_HOSTS, because the URL comes out of a webhook payload. An install this operator creates had no way to set it short of spec.pod.extraEnv. spec.prometheus takes allowedHosts, preroll and timeout and maps them to TERDUT_PROMETHEUS_ALLOWED_HOSTS, _PREROLL and _TIMEOUT. Nothing is emitted unless allowedHosts is non-empty, so existing TerdutServers get the same Deployment as before, and unset durations are left to the server's defaults. There is no CRD default for the host: which Prometheus to trust is the deployer's call, not something this operator should assume. The CRD in config/ is regenerated and the chart's copy carries the same schema; the chart's optional TerdutServer passes the block through. Claude-Session: https://claude.ai/code/session_016mBLURvJoMuUEr9cB2RpUN
This commit is contained in:
@@ -107,6 +107,10 @@ spec:
|
|||||||
fallbackTopic: ""
|
fallbackTopic: ""
|
||||||
repeatEvery: 15m
|
repeatEvery: 15m
|
||||||
tokenSecretRef: {name: "", key: token}
|
tokenSecretRef: {name: "", key: token}
|
||||||
|
prometheus: # terdut-server v0.49.0+: metrics on incidents; off unless allowedHosts is set
|
||||||
|
allowedHosts: [] # e.g. ["kube-prometheus-stack-prometheus.kube-prometheus-stack:9090"]; wildcards: "*.monitoring.svc"
|
||||||
|
preroll: "" # unset = server default (1h)
|
||||||
|
timeout: "" # unset = server default (10s)
|
||||||
oidc:
|
oidc:
|
||||||
enabled: false
|
enabled: false
|
||||||
issuer: ""
|
issuer: ""
|
||||||
|
|||||||
@@ -118,6 +118,32 @@ type NotifySpec struct {
|
|||||||
TokenSecretRef *SecretKeyRef `json:"tokenSecretRef,omitempty"`
|
TokenSecretRef *SecretKeyRef `json:"tokenSecretRef,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PrometheusSpec controls the metrics terdut-server shows on an incident,
|
||||||
|
// fetched from the Prometheus named in each alert's generatorURL
|
||||||
|
// (terdut-server v0.49.0+). Off unless allowedHosts is set: the URL comes
|
||||||
|
// out of a webhook payload, so the server queries only hosts listed here.
|
||||||
|
type PrometheusSpec struct {
|
||||||
|
// allowedHosts are the Prometheus servers terdut-server may query,
|
||||||
|
// passed as TERDUT_PROMETHEUS_ALLOWED_HOSTS. Entries are host names
|
||||||
|
// (with "*" as one DNS label and a leading "**" as one or more), IPs or
|
||||||
|
// CIDR ranges, each with an optional ":port", e.g.
|
||||||
|
// "kube-prometheus-stack-prometheus.kube-prometheus-stack:9090" or
|
||||||
|
// "*.monitoring.svc". Empty turns the feature off. The server refuses to
|
||||||
|
// start on a malformed entry or a bare "*".
|
||||||
|
// +optional
|
||||||
|
AllowedHosts []string `json:"allowedHosts,omitempty"`
|
||||||
|
|
||||||
|
// preroll is how far before an alert started its graph begins (a Go
|
||||||
|
// duration, e.g. "1h"). Unset uses terdut-server's default.
|
||||||
|
// +optional
|
||||||
|
Preroll string `json:"preroll,omitempty"`
|
||||||
|
|
||||||
|
// timeout bounds one request to a Prometheus (a Go duration, e.g.
|
||||||
|
// "10s"). Unset uses terdut-server's default.
|
||||||
|
// +optional
|
||||||
|
Timeout string `json:"timeout,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
// OIDCSpec controls single sign-on.
|
// OIDCSpec controls single sign-on.
|
||||||
type OIDCSpec struct {
|
type OIDCSpec struct {
|
||||||
// +optional
|
// +optional
|
||||||
@@ -325,6 +351,11 @@ type TerdutServerSpec struct {
|
|||||||
// +optional
|
// +optional
|
||||||
OIDC OIDCSpec `json:"oidc,omitempty"`
|
OIDC OIDCSpec `json:"oidc,omitempty"`
|
||||||
|
|
||||||
|
// prometheus enables the metrics on an incident (terdut-server
|
||||||
|
// v0.49.0+; an older image ignores the env vars it produces).
|
||||||
|
// +optional
|
||||||
|
Prometheus PrometheusSpec `json:"prometheus,omitempty"`
|
||||||
|
|
||||||
// passwordLogin: whether a user may sign in, or sign up, with a
|
// passwordLogin: whether a user may sign in, or sign up, with a
|
||||||
// password.
|
// password.
|
||||||
// +kubebuilder:default=true
|
// +kubebuilder:default=true
|
||||||
|
|||||||
@@ -391,6 +391,26 @@ func (in *PostgresClusterRef) DeepCopy() *PostgresClusterRef {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||||
|
func (in *PrometheusSpec) DeepCopyInto(out *PrometheusSpec) {
|
||||||
|
*out = *in
|
||||||
|
if in.AllowedHosts != nil {
|
||||||
|
in, out := &in.AllowedHosts, &out.AllowedHosts
|
||||||
|
*out = make([]string, len(*in))
|
||||||
|
copy(*out, *in)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PrometheusSpec.
|
||||||
|
func (in *PrometheusSpec) DeepCopy() *PrometheusSpec {
|
||||||
|
if in == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := new(PrometheusSpec)
|
||||||
|
in.DeepCopyInto(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||||
func (in *SecretKeyRef) DeepCopyInto(out *SecretKeyRef) {
|
func (in *SecretKeyRef) DeepCopyInto(out *SecretKeyRef) {
|
||||||
*out = *in
|
*out = *in
|
||||||
@@ -606,6 +626,7 @@ func (in *TerdutServerSpec) DeepCopyInto(out *TerdutServerSpec) {
|
|||||||
out.Sweeper = in.Sweeper
|
out.Sweeper = in.Sweeper
|
||||||
in.Notify.DeepCopyInto(&out.Notify)
|
in.Notify.DeepCopyInto(&out.Notify)
|
||||||
in.OIDC.DeepCopyInto(&out.OIDC)
|
in.OIDC.DeepCopyInto(&out.OIDC)
|
||||||
|
in.Prometheus.DeepCopyInto(&out.Prometheus)
|
||||||
in.AllowedTeams.DeepCopyInto(&out.AllowedTeams)
|
in.AllowedTeams.DeepCopyInto(&out.AllowedTeams)
|
||||||
in.Pod.DeepCopyInto(&out.Pod)
|
in.Pod.DeepCopyInto(&out.Pod)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4301,6 +4301,34 @@ spec:
|
|||||||
type: object
|
type: object
|
||||||
type: array
|
type: array
|
||||||
type: object
|
type: object
|
||||||
|
prometheus:
|
||||||
|
description: |-
|
||||||
|
prometheus enables the metrics on an incident (terdut-server
|
||||||
|
v0.49.0+; an older image ignores the env vars it produces).
|
||||||
|
properties:
|
||||||
|
allowedHosts:
|
||||||
|
description: |-
|
||||||
|
allowedHosts are the Prometheus servers terdut-server may query,
|
||||||
|
passed as TERDUT_PROMETHEUS_ALLOWED_HOSTS. Entries are host names
|
||||||
|
(with "*" as one DNS label and a leading "**" as one or more), IPs or
|
||||||
|
CIDR ranges, each with an optional ":port", e.g.
|
||||||
|
"kube-prometheus-stack-prometheus.kube-prometheus-stack:9090" or
|
||||||
|
"*.monitoring.svc". Empty turns the feature off. The server refuses to
|
||||||
|
start on a malformed entry or a bare "*".
|
||||||
|
items:
|
||||||
|
type: string
|
||||||
|
type: array
|
||||||
|
preroll:
|
||||||
|
description: |-
|
||||||
|
preroll is how far before an alert started its graph begins (a Go
|
||||||
|
duration, e.g. "1h"). Unset uses terdut-server's default.
|
||||||
|
type: string
|
||||||
|
timeout:
|
||||||
|
description: |-
|
||||||
|
timeout bounds one request to a Prometheus (a Go duration, e.g.
|
||||||
|
"10s"). Unset uses terdut-server's default.
|
||||||
|
type: string
|
||||||
|
type: object
|
||||||
replicas:
|
replicas:
|
||||||
default: 2
|
default: 2
|
||||||
description: |-
|
description: |-
|
||||||
|
|||||||
@@ -42,6 +42,10 @@ spec:
|
|||||||
oidc:
|
oidc:
|
||||||
{{- toYaml . | nindent 4 }}
|
{{- toYaml . | nindent 4 }}
|
||||||
{{- end }}
|
{{- end }}
|
||||||
|
{{- with .Values.terdutServer.prometheus }}
|
||||||
|
prometheus:
|
||||||
|
{{- toYaml . | nindent 4 }}
|
||||||
|
{{- end }}
|
||||||
passwordLogin: {{ .Values.terdutServer.passwordLogin }}
|
passwordLogin: {{ .Values.terdutServer.passwordLogin }}
|
||||||
{{- with .Values.terdutServer.allowedTeams }}
|
{{- with .Values.terdutServer.allowedTeams }}
|
||||||
allowedTeams:
|
allowedTeams:
|
||||||
|
|||||||
@@ -269,6 +269,12 @@ terdutServer:
|
|||||||
# notify: {}
|
# notify: {}
|
||||||
# oidc: {}
|
# oidc: {}
|
||||||
|
|
||||||
|
## Metrics on incidents (terdut-server v0.49.0+): hosts it may query for the
|
||||||
|
## Prometheus named in an alert's generatorURL. Empty/unset = off.
|
||||||
|
# prometheus:
|
||||||
|
# allowedHosts:
|
||||||
|
# - kube-prometheus-stack-prometheus.kube-prometheus-stack:9090
|
||||||
|
|
||||||
passwordLogin: true
|
passwordLogin: true
|
||||||
|
|
||||||
# allowedTeams: {}
|
# allowedTeams: {}
|
||||||
|
|||||||
@@ -4298,6 +4298,34 @@ spec:
|
|||||||
type: object
|
type: object
|
||||||
type: array
|
type: array
|
||||||
type: object
|
type: object
|
||||||
|
prometheus:
|
||||||
|
description: |-
|
||||||
|
prometheus enables the metrics on an incident (terdut-server
|
||||||
|
v0.49.0+; an older image ignores the env vars it produces).
|
||||||
|
properties:
|
||||||
|
allowedHosts:
|
||||||
|
description: |-
|
||||||
|
allowedHosts are the Prometheus servers terdut-server may query,
|
||||||
|
passed as TERDUT_PROMETHEUS_ALLOWED_HOSTS. Entries are host names
|
||||||
|
(with "*" as one DNS label and a leading "**" as one or more), IPs or
|
||||||
|
CIDR ranges, each with an optional ":port", e.g.
|
||||||
|
"kube-prometheus-stack-prometheus.kube-prometheus-stack:9090" or
|
||||||
|
"*.monitoring.svc". Empty turns the feature off. The server refuses to
|
||||||
|
start on a malformed entry or a bare "*".
|
||||||
|
items:
|
||||||
|
type: string
|
||||||
|
type: array
|
||||||
|
preroll:
|
||||||
|
description: |-
|
||||||
|
preroll is how far before an alert started its graph begins (a Go
|
||||||
|
duration, e.g. "1h"). Unset uses terdut-server's default.
|
||||||
|
type: string
|
||||||
|
timeout:
|
||||||
|
description: |-
|
||||||
|
timeout bounds one request to a Prometheus (a Go duration, e.g.
|
||||||
|
"10s"). Unset uses terdut-server's default.
|
||||||
|
type: string
|
||||||
|
type: object
|
||||||
replicas:
|
replicas:
|
||||||
default: 2
|
default: 2
|
||||||
description: |-
|
description: |-
|
||||||
|
|||||||
@@ -112,6 +112,7 @@ var _ = Describe("TerdutServer Controller", func() {
|
|||||||
Expect(env["TERDUT_DB_DSN"].Value).To(Equal(testDSN))
|
Expect(env["TERDUT_DB_DSN"].Value).To(Equal(testDSN))
|
||||||
Expect(env["TERDUT_OPERATOR_MODE"].Value).To(Equal("true"))
|
Expect(env["TERDUT_OPERATOR_MODE"].Value).To(Equal("true"))
|
||||||
Expect(env).NotTo(HaveKey("TERDUT_DEADMAN_MATCHERS"), "dead man's switches are per team now")
|
Expect(env).NotTo(HaveKey("TERDUT_DEADMAN_MATCHERS"), "dead man's switches are per team now")
|
||||||
|
Expect(env).NotTo(HaveKey("TERDUT_PROMETHEUS_ALLOWED_HOSTS"), "metrics stay off unless spec.prometheus asks")
|
||||||
|
|
||||||
var svc corev1.Service
|
var svc corev1.Service
|
||||||
Expect(k8sClient.Get(ctx, objKey, &svc)).To(Succeed())
|
Expect(k8sClient.Get(ctx, objKey, &svc)).To(Succeed())
|
||||||
@@ -252,6 +253,36 @@ var _ = Describe("TerdutServer Controller", func() {
|
|||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
Describe("spec.prometheus", func() {
|
||||||
|
It("passes the allowlist and tuning to the server as env vars", func(ctx SpecContext) {
|
||||||
|
spec := dsnSpec()
|
||||||
|
spec.Prometheus = terdutv1alpha1.PrometheusSpec{
|
||||||
|
AllowedHosts: []string{"kube-prometheus-stack-prometheus.kube-prometheus-stack:9090", "*.monitoring.svc"},
|
||||||
|
Preroll: "2h",
|
||||||
|
Timeout: "5s",
|
||||||
|
}
|
||||||
|
createServer(ctx, spec)
|
||||||
|
reconcileOnce(ctx)
|
||||||
|
|
||||||
|
env := envOf(ctx)
|
||||||
|
Expect(env["TERDUT_PROMETHEUS_ALLOWED_HOSTS"].Value).To(Equal("kube-prometheus-stack-prometheus.kube-prometheus-stack:9090,*.monitoring.svc"))
|
||||||
|
Expect(env["TERDUT_PROMETHEUS_PREROLL"].Value).To(Equal("2h"))
|
||||||
|
Expect(env["TERDUT_PROMETHEUS_TIMEOUT"].Value).To(Equal("5s"))
|
||||||
|
})
|
||||||
|
|
||||||
|
It("leaves the tuning to the server's defaults when only hosts are given", func(ctx SpecContext) {
|
||||||
|
spec := dsnSpec()
|
||||||
|
spec.Prometheus = terdutv1alpha1.PrometheusSpec{AllowedHosts: []string{"prometheus.monitoring.svc"}}
|
||||||
|
createServer(ctx, spec)
|
||||||
|
reconcileOnce(ctx)
|
||||||
|
|
||||||
|
env := envOf(ctx)
|
||||||
|
Expect(env).To(HaveKey("TERDUT_PROMETHEUS_ALLOWED_HOSTS"))
|
||||||
|
Expect(env).NotTo(HaveKey("TERDUT_PROMETHEUS_PREROLL"))
|
||||||
|
Expect(env).NotTo(HaveKey("TERDUT_PROMETHEUS_TIMEOUT"))
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
Describe("spec.pod", func() {
|
Describe("spec.pod", func() {
|
||||||
It("wires pod-level customization onto the right spot on the Deployment", func(ctx SpecContext) {
|
It("wires pod-level customization onto the right spot on the Deployment", func(ctx SpecContext) {
|
||||||
spec := dsnSpec()
|
spec := dsnSpec()
|
||||||
|
|||||||
@@ -216,6 +216,16 @@ func buildEnv(srv *terdutv1alpha1.TerdutServer, dbEnv []corev1.EnvVar) []corev1.
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if prom := srv.Spec.Prometheus; len(prom.AllowedHosts) > 0 {
|
||||||
|
env = append(env, corev1.EnvVar{Name: "TERDUT_PROMETHEUS_ALLOWED_HOSTS", Value: strings.Join(prom.AllowedHosts, ",")})
|
||||||
|
if prom.Preroll != "" {
|
||||||
|
env = append(env, corev1.EnvVar{Name: "TERDUT_PROMETHEUS_PREROLL", Value: prom.Preroll})
|
||||||
|
}
|
||||||
|
if prom.Timeout != "" {
|
||||||
|
env = append(env, corev1.EnvVar{Name: "TERDUT_PROMETHEUS_TIMEOUT", Value: prom.Timeout})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
publicURL := fmt.Sprintf("https://%s", srv.Spec.Networking.Hostname)
|
publicURL := fmt.Sprintf("https://%s", srv.Spec.Networking.Hostname)
|
||||||
env = append(env,
|
env = append(env,
|
||||||
corev1.EnvVar{Name: "TERDUT_PUBLIC_URL", Value: publicURL},
|
corev1.EnvVar{Name: "TERDUT_PUBLIC_URL", Value: publicURL},
|
||||||
|
|||||||
Reference in New Issue
Block a user