Stage 3: TerdutEscalationRule + TerdutDeadmanSwitch
CI / test (push) Has been cancelled

Both child CRDs resolve their own teamRef -> TerdutTeam.status via the new
shared resolveTeamAndClient helper (childref.go), never chaining up to
TerdutServer (DESIGN.md §5) -- TerdutTeam.status.serverEndpoint, added in
this same stage, is what makes that literally true.

TerdutEscalationRule: one PUT /api/teams/{id}/escalation per reconcile
(an upsert server-side, confirmed against source), resolving each "user"
target's username to a user_id via GET /api/users first and reporting
Ready: False, reason: UnknownUser if it doesn't resolve. No DELETE exists
for this resource, so its delete path PUTs an empty policy as the closest
available undo.

TerdutDeadmanSwitch: real create/update-in-place/delete, using
terdut-server v0.33.0's PUT (added specifically for this operator). No
unique-name constraint server-side, so idempotent-create here is
GET-list-and-match-by-name rather than adopt-on-409.

Extends tdclient with User/GetUserByUsername, the escalation request types
+ SetEscalation, and DeadmanSwitch + its CRUD methods. Also folds
ConditionTeamReady into the single shared ConditionReady constant, since
both were literally "Ready" and Stage 3 would otherwise have needed a
third same-valued constant.

internal/controller/terdutserver_controller_test.go's fakeTerdutServer
grows GET /api/users, PUT .../escalation, and the full dead man's switch
collection/item routes, replacing the old parseTeamPath/handleTeamByID
pair with a more general parseTeamSubPath/handleTeamSubPath dispatcher
that still covers every existing Stage 1/2 route unchanged.

make fmt lint test build all clean; envtest coverage for
internal/controller: 50.5% -> 71.7%.
This commit is contained in:
Niklas Ye
2026-10-01 13:50:08 +02:00
parent fef60caf06
commit fb9e6a38dc
31 changed files with 2399 additions and 51 deletions
+18
View File
@@ -27,4 +27,22 @@ resources:
kind: TerdutTeam
path: git.ryuvia.com/niklas/terdut-operator/api/v1alpha1
version: v1alpha1
- api:
crdVersion: v1
namespaced: true
controller: true
domain: ryuvia.com
group: terdut
kind: TerdutEscalationRule
path: git.ryuvia.com/niklas/terdut-operator/api/v1alpha1
version: v1alpha1
- api:
crdVersion: v1
namespaced: true
controller: true
domain: ryuvia.com
group: terdut
kind: TerdutDeadmanSwitch
path: git.ryuvia.com/niklas/terdut-operator/api/v1alpha1
version: v1alpha1
version: "3"
+98
View File
@@ -0,0 +1,98 @@
package v1alpha1
import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
)
// TerdutDeadmanSwitchSpec defines the desired state of TerdutDeadmanSwitch.
//
// One per switch (DESIGN.md §4.4). Reconciled with real update-in-place
// (terdut-server v0.33.0 added PUT specifically for this, §5) -- but with no
// unique-name constraint server-side, idempotent-create here means
// GET-list-and-match-by-name, not adopt-on-409.
type TerdutDeadmanSwitchSpec struct {
// +required
TeamRef TerdutTeamRef `json:"teamRef"`
// name is optional, same as the API: left empty, terdut-server derives
// it from matcher's own canonical form, and that's what the
// idempotent-create lookup matches against too.
// +optional
Name string `json:"name,omitempty"`
// matcher names the alerts this switch watches, e.g.
// "alertname=Watchdog,cluster=prod". One matcher per switch -- add
// another TerdutDeadmanSwitch instead of separating with ";"
// (terdut-server's own restriction, mirrored here so a bad spec is
// rejected at apply time).
// +required
// +kubebuilder:validation:MinLength=1
// +kubebuilder:validation:XValidation:rule="!self.contains(';')",message="one matcher per switch: add another TerdutDeadmanSwitch instead of separating with ;"
Matcher string `json:"matcher"`
// timeout is a Go duration string, e.g. "15m".
// +required
// +kubebuilder:validation:MinLength=1
Timeout string `json:"timeout"`
// +kubebuilder:validation:Enum=critical;error;warning;info
// +kubebuilder:default=critical
// +optional
Severity string `json:"severity,omitempty"`
}
// TerdutDeadmanSwitchStatus defines the observed state of TerdutDeadmanSwitch.
type TerdutDeadmanSwitchStatus struct {
// +listType=map
// +listMapKey=type
// +optional
Conditions []metav1.Condition `json:"conditions,omitempty"`
// switchID is the server-side id.
// +optional
SwitchID int64 `json:"switchID,omitempty"`
// +optional
ObservedGeneration int64 `json:"observedGeneration,omitempty"`
}
// +kubebuilder:object:root=true
// +kubebuilder:subresource:status
// +kubebuilder:printcolumn:name="Team",type=string,JSONPath=`.spec.teamRef.name`
// +kubebuilder:printcolumn:name="SwitchID",type=integer,JSONPath=`.status.switchID`
// +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].status`
// +kubebuilder:printcolumn:name="Reason",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].reason`
// TerdutDeadmanSwitch is the Schema for the terdutdeadmanswitches API
type TerdutDeadmanSwitch struct {
metav1.TypeMeta `json:",inline"`
// metadata is a standard object metadata
// +optional
metav1.ObjectMeta `json:"metadata,omitzero"`
// spec defines the desired state of TerdutDeadmanSwitch
// +required
Spec TerdutDeadmanSwitchSpec `json:"spec"`
// status defines the observed state of TerdutDeadmanSwitch
// +optional
Status TerdutDeadmanSwitchStatus `json:"status,omitzero"`
}
// +kubebuilder:object:root=true
// TerdutDeadmanSwitchList contains a list of TerdutDeadmanSwitch
type TerdutDeadmanSwitchList struct {
metav1.TypeMeta `json:",inline"`
metav1.ListMeta `json:"metadata,omitzero"`
Items []TerdutDeadmanSwitch `json:"items"`
}
func init() {
SchemeBuilder.Register(func(s *runtime.Scheme) error {
s.AddKnownTypes(SchemeGroupVersion, &TerdutDeadmanSwitch{}, &TerdutDeadmanSwitchList{})
return nil
})
}
+137
View File
@@ -0,0 +1,137 @@
package v1alpha1
import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
)
// TerdutTeamRef names the TerdutTeam this resource belongs to. Always
// same-namespace as the CR itself (DESIGN.md §1: only TerdutTeam.spec.serverRef
// crosses namespaces in v1) -- no namespace field, unlike TerdutServerRef.
type TerdutTeamRef struct {
// +kubebuilder:validation:MinLength=1
Name string `json:"name"`
}
// EscalationTargetKind is who one rung of the ladder pages.
// +kubebuilder:validation:Enum=oncall;user
type EscalationTargetKind string
const (
EscalationTargetOncall EscalationTargetKind = "oncall"
EscalationTargetUser EscalationTargetKind = "user"
)
// EscalationTarget is one page within a level. username is required iff
// kind is "user" (terdut-server's own validation, internal/api/escalation.go's
// handleSetEscalation -- mirrored here as a CEL rule so a bad spec is
// rejected at apply time, not discovered on the next failed PUT).
// +kubebuilder:validation:XValidation:rule="self.kind != 'user' || has(self.username)",message="username is required when kind is user"
// +kubebuilder:validation:XValidation:rule="self.kind != 'oncall' || !has(self.username)",message="username must not be set when kind is oncall"
type EscalationTarget struct {
// +required
Kind EscalationTargetKind `json:"kind"`
// +optional
Username string `json:"username,omitempty"`
}
// EscalationLevel is one rung of the ladder: how long to wait, and who to
// page if nobody's acknowledged by then.
type EscalationLevel struct {
// timeout is a Go duration string, e.g. "5m".
// +required
// +kubebuilder:validation:MinLength=1
Timeout string `json:"timeout"`
// +required
// +kubebuilder:validation:MinItems=1
Targets []EscalationTarget `json:"targets"`
}
// TerdutEscalationRuleSpec defines the desired state of TerdutEscalationRule.
//
// One per team (DESIGN.md §4.3) -- terdut-server models a policy as one row
// with an owned list of levels, reconciled with a single whole-policy PUT.
// Not enforced at admission if two CRs name the same team (no webhooks in
// v1, §1); they would simply clobber each other every reconcile.
type TerdutEscalationRuleSpec struct {
// +required
TeamRef TerdutTeamRef `json:"teamRef"`
// +kubebuilder:validation:Minimum=0
// +kubebuilder:validation:Maximum=10
// +optional
RepeatCount int64 `json:"repeatCount,omitempty"`
// +optional
FallbackTopic string `json:"fallbackTopic,omitempty"`
// +required
// +kubebuilder:validation:MinItems=1
Levels []EscalationLevel `json:"levels"`
}
// Condition reasons shared by TerdutEscalationRule and TerdutDeadmanSwitch
// (both resolve a teamRef the same way, DESIGN.md §5).
const (
// ReasonTeamRefNotFound: spec.teamRef names no TerdutTeam (yet).
ReasonTeamRefNotFound = "TeamRefNotFound"
// ReasonWaitingForTeam: the referenced TerdutTeam exists but isn't
// Ready yet (no status.credentialsSecretRef to read).
ReasonWaitingForTeam = "WaitingForTeam"
// ReasonUnknownUser: an escalation target's username doesn't resolve to
// any user server-side (TerdutEscalationRule only).
ReasonUnknownUser = "UnknownUser"
// ReasonChildAdopted: the happy path, shared by both child kinds.
ReasonChildAdopted = "Adopted"
)
// TerdutEscalationRuleStatus defines the observed state of TerdutEscalationRule.
type TerdutEscalationRuleStatus struct {
// +listType=map
// +listMapKey=type
// +optional
Conditions []metav1.Condition `json:"conditions,omitempty"`
// +optional
ObservedGeneration int64 `json:"observedGeneration,omitempty"`
}
// +kubebuilder:object:root=true
// +kubebuilder:subresource:status
// +kubebuilder:printcolumn:name="Team",type=string,JSONPath=`.spec.teamRef.name`
// +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].status`
// +kubebuilder:printcolumn:name="Reason",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].reason`
// TerdutEscalationRule is the Schema for the terdutescalationrules API
type TerdutEscalationRule struct {
metav1.TypeMeta `json:",inline"`
// metadata is a standard object metadata
// +optional
metav1.ObjectMeta `json:"metadata,omitzero"`
// spec defines the desired state of TerdutEscalationRule
// +required
Spec TerdutEscalationRuleSpec `json:"spec"`
// status defines the observed state of TerdutEscalationRule
// +optional
Status TerdutEscalationRuleStatus `json:"status,omitzero"`
}
// +kubebuilder:object:root=true
// TerdutEscalationRuleList contains a list of TerdutEscalationRule
type TerdutEscalationRuleList struct {
metav1.TypeMeta `json:",inline"`
metav1.ListMeta `json:"metadata,omitzero"`
Items []TerdutEscalationRule `json:"items"`
}
func init() {
SchemeBuilder.Register(func(s *runtime.Scheme) error {
s.AddKnownTypes(SchemeGroupVersion, &TerdutEscalationRule{}, &TerdutEscalationRuleList{})
return nil
})
}
+7 -5
View File
@@ -45,11 +45,6 @@ type TerdutTeamSpec struct {
OIDC TerdutTeamOIDC `json:"oidc,omitempty"`
}
// Condition types this controller sets on TerdutTeam.
const (
ConditionTeamReady = "Ready"
)
// Condition reasons this controller sets.
const (
// ReasonServerRefNotFound: spec.serverRef names no TerdutServer (yet).
@@ -85,6 +80,13 @@ type TerdutTeamStatus struct {
// +optional
CredentialsSecretRef *SecretKeyRef `json:"credentialsSecretRef,omitempty"`
// serverEndpoint is the resolved TerdutServer's base URL, resolved once
// here so no child controller (TerdutEscalationRule, TerdutDeadmanSwitch,
// TerdutAlertSource) ever needs its own RBAC on terdutservers just to
// find out where to send a request (DESIGN.md §5).
// +optional
ServerEndpoint string `json:"serverEndpoint,omitempty"`
// +optional
ObservedGeneration int64 `json:"observedGeneration,omitempty"`
}
+251
View File
@@ -85,6 +85,41 @@ func (in *DeadmanSpec) DeepCopy() *DeadmanSpec {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *EscalationLevel) DeepCopyInto(out *EscalationLevel) {
*out = *in
if in.Targets != nil {
in, out := &in.Targets, &out.Targets
*out = make([]EscalationTarget, len(*in))
copy(*out, *in)
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EscalationLevel.
func (in *EscalationLevel) DeepCopy() *EscalationLevel {
if in == nil {
return nil
}
out := new(EscalationLevel)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *EscalationTarget) DeepCopyInto(out *EscalationTarget) {
*out = *in
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EscalationTarget.
func (in *EscalationTarget) DeepCopy() *EscalationTarget {
if in == nil {
return nil
}
out := new(EscalationTarget)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ImageSpec) DeepCopyInto(out *ImageSpec) {
*out = *in
@@ -205,6 +240,207 @@ func (in *SweeperSpec) DeepCopy() *SweeperSpec {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *TerdutDeadmanSwitch) DeepCopyInto(out *TerdutDeadmanSwitch) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
out.Spec = in.Spec
in.Status.DeepCopyInto(&out.Status)
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TerdutDeadmanSwitch.
func (in *TerdutDeadmanSwitch) DeepCopy() *TerdutDeadmanSwitch {
if in == nil {
return nil
}
out := new(TerdutDeadmanSwitch)
in.DeepCopyInto(out)
return out
}
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
func (in *TerdutDeadmanSwitch) DeepCopyObject() runtime.Object {
if c := in.DeepCopy(); c != nil {
return c
}
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *TerdutDeadmanSwitchList) DeepCopyInto(out *TerdutDeadmanSwitchList) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ListMeta.DeepCopyInto(&out.ListMeta)
if in.Items != nil {
in, out := &in.Items, &out.Items
*out = make([]TerdutDeadmanSwitch, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TerdutDeadmanSwitchList.
func (in *TerdutDeadmanSwitchList) DeepCopy() *TerdutDeadmanSwitchList {
if in == nil {
return nil
}
out := new(TerdutDeadmanSwitchList)
in.DeepCopyInto(out)
return out
}
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
func (in *TerdutDeadmanSwitchList) DeepCopyObject() runtime.Object {
if c := in.DeepCopy(); c != nil {
return c
}
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *TerdutDeadmanSwitchSpec) DeepCopyInto(out *TerdutDeadmanSwitchSpec) {
*out = *in
out.TeamRef = in.TeamRef
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TerdutDeadmanSwitchSpec.
func (in *TerdutDeadmanSwitchSpec) DeepCopy() *TerdutDeadmanSwitchSpec {
if in == nil {
return nil
}
out := new(TerdutDeadmanSwitchSpec)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *TerdutDeadmanSwitchStatus) DeepCopyInto(out *TerdutDeadmanSwitchStatus) {
*out = *in
if in.Conditions != nil {
in, out := &in.Conditions, &out.Conditions
*out = make([]v1.Condition, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TerdutDeadmanSwitchStatus.
func (in *TerdutDeadmanSwitchStatus) DeepCopy() *TerdutDeadmanSwitchStatus {
if in == nil {
return nil
}
out := new(TerdutDeadmanSwitchStatus)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *TerdutEscalationRule) DeepCopyInto(out *TerdutEscalationRule) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
in.Spec.DeepCopyInto(&out.Spec)
in.Status.DeepCopyInto(&out.Status)
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TerdutEscalationRule.
func (in *TerdutEscalationRule) DeepCopy() *TerdutEscalationRule {
if in == nil {
return nil
}
out := new(TerdutEscalationRule)
in.DeepCopyInto(out)
return out
}
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
func (in *TerdutEscalationRule) DeepCopyObject() runtime.Object {
if c := in.DeepCopy(); c != nil {
return c
}
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *TerdutEscalationRuleList) DeepCopyInto(out *TerdutEscalationRuleList) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ListMeta.DeepCopyInto(&out.ListMeta)
if in.Items != nil {
in, out := &in.Items, &out.Items
*out = make([]TerdutEscalationRule, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TerdutEscalationRuleList.
func (in *TerdutEscalationRuleList) DeepCopy() *TerdutEscalationRuleList {
if in == nil {
return nil
}
out := new(TerdutEscalationRuleList)
in.DeepCopyInto(out)
return out
}
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
func (in *TerdutEscalationRuleList) DeepCopyObject() runtime.Object {
if c := in.DeepCopy(); c != nil {
return c
}
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *TerdutEscalationRuleSpec) DeepCopyInto(out *TerdutEscalationRuleSpec) {
*out = *in
out.TeamRef = in.TeamRef
if in.Levels != nil {
in, out := &in.Levels, &out.Levels
*out = make([]EscalationLevel, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TerdutEscalationRuleSpec.
func (in *TerdutEscalationRuleSpec) DeepCopy() *TerdutEscalationRuleSpec {
if in == nil {
return nil
}
out := new(TerdutEscalationRuleSpec)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *TerdutEscalationRuleStatus) DeepCopyInto(out *TerdutEscalationRuleStatus) {
*out = *in
if in.Conditions != nil {
in, out := &in.Conditions, &out.Conditions
*out = make([]v1.Condition, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TerdutEscalationRuleStatus.
func (in *TerdutEscalationRuleStatus) DeepCopy() *TerdutEscalationRuleStatus {
if in == nil {
return nil
}
out := new(TerdutEscalationRuleStatus)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *TerdutServer) DeepCopyInto(out *TerdutServer) {
*out = *in
@@ -403,6 +639,21 @@ func (in *TerdutTeamOIDC) DeepCopy() *TerdutTeamOIDC {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *TerdutTeamRef) DeepCopyInto(out *TerdutTeamRef) {
*out = *in
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TerdutTeamRef.
func (in *TerdutTeamRef) DeepCopy() *TerdutTeamRef {
if in == nil {
return nil
}
out := new(TerdutTeamRef)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *TerdutTeamSpec) DeepCopyInto(out *TerdutTeamSpec) {
*out = *in
+16
View File
@@ -193,6 +193,22 @@ func main() {
setupLog.Error(err, "Failed to create controller", "controller", "terdutteam")
os.Exit(1)
}
if err := (&controller.TerdutEscalationRuleReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
OperatorNamespace: operatorNamespace,
}).SetupWithManager(mgr); err != nil {
setupLog.Error(err, "Failed to create controller", "controller", "terdutescalationrule")
os.Exit(1)
}
if err := (&controller.TerdutDeadmanSwitchReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
OperatorNamespace: operatorNamespace,
}).SetupWithManager(mgr); err != nil {
setupLog.Error(err, "Failed to create controller", "controller", "terdutdeadmanswitch")
os.Exit(1)
}
// +kubebuilder:scaffold:builder
if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
@@ -0,0 +1,180 @@
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.22.0
name: terdutdeadmanswitches.terdut.ryuvia.com
spec:
group: terdut.ryuvia.com
names:
kind: TerdutDeadmanSwitch
listKind: TerdutDeadmanSwitchList
plural: terdutdeadmanswitches
singular: terdutdeadmanswitch
scope: Namespaced
versions:
- additionalPrinterColumns:
- jsonPath: .spec.teamRef.name
name: Team
type: string
- jsonPath: .status.switchID
name: SwitchID
type: integer
- jsonPath: .status.conditions[?(@.type=="Ready")].status
name: Ready
type: string
- jsonPath: .status.conditions[?(@.type=="Ready")].reason
name: Reason
type: string
name: v1alpha1
schema:
openAPIV3Schema:
description: TerdutDeadmanSwitch is the Schema for the terdutdeadmanswitches
API
properties:
apiVersion:
description: |-
APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
type: string
kind:
description: |-
Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
type: string
metadata:
type: object
spec:
description: spec defines the desired state of TerdutDeadmanSwitch
properties:
matcher:
description: |-
matcher names the alerts this switch watches, e.g.
"alertname=Watchdog,cluster=prod". One matcher per switch -- add
another TerdutDeadmanSwitch instead of separating with ";"
(terdut-server's own restriction, mirrored here so a bad spec is
rejected at apply time).
minLength: 1
type: string
x-kubernetes-validations:
- message: 'one matcher per switch: add another TerdutDeadmanSwitch
instead of separating with ;'
rule: '!self.contains('';'')'
name:
description: |-
name is optional, same as the API: left empty, terdut-server derives
it from matcher's own canonical form, and that's what the
idempotent-create lookup matches against too.
type: string
severity:
default: critical
enum:
- critical
- error
- warning
- info
type: string
teamRef:
description: |-
TerdutTeamRef names the TerdutTeam this resource belongs to. Always
same-namespace as the CR itself (DESIGN.md §1: only TerdutTeam.spec.serverRef
crosses namespaces in v1) -- no namespace field, unlike TerdutServerRef.
properties:
name:
minLength: 1
type: string
required:
- name
type: object
timeout:
description: timeout is a Go duration string, e.g. "15m".
minLength: 1
type: string
required:
- matcher
- teamRef
- timeout
type: object
status:
description: status defines the observed state of TerdutDeadmanSwitch
properties:
conditions:
items:
description: Condition contains details for one aspect of the current
state of this API Resource.
properties:
lastTransitionTime:
description: |-
lastTransitionTime is the last time the condition transitioned from one status to another.
This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
format: date-time
type: string
message:
description: |-
message is a human readable message indicating details about the transition.
This may be an empty string.
maxLength: 32768
type: string
observedGeneration:
description: |-
observedGeneration represents the .metadata.generation that the condition was set based upon.
For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
with respect to the current state of the instance.
format: int64
minimum: 0
type: integer
reason:
description: |-
reason contains a programmatic identifier indicating the reason for the condition's last transition.
Producers of specific condition types may define expected values and meanings for this field,
and whether the values are considered a guaranteed API.
The value should be a CamelCase string.
This field may not be empty.
maxLength: 1024
minLength: 1
pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
type: string
status:
description: status of the condition, one of True, False, Unknown.
enum:
- "True"
- "False"
- Unknown
type: string
type:
description: type of condition in CamelCase or in foo.example.com/CamelCase.
maxLength: 316
pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
type: string
required:
- lastTransitionTime
- message
- reason
- status
- type
type: object
type: array
x-kubernetes-list-map-keys:
- type
x-kubernetes-list-type: map
observedGeneration:
format: int64
type: integer
switchID:
description: switchID is the server-side id.
format: int64
type: integer
type: object
required:
- spec
type: object
served: true
storage: true
subresources:
status: {}
@@ -0,0 +1,191 @@
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.22.0
name: terdutescalationrules.terdut.ryuvia.com
spec:
group: terdut.ryuvia.com
names:
kind: TerdutEscalationRule
listKind: TerdutEscalationRuleList
plural: terdutescalationrules
singular: terdutescalationrule
scope: Namespaced
versions:
- additionalPrinterColumns:
- jsonPath: .spec.teamRef.name
name: Team
type: string
- jsonPath: .status.conditions[?(@.type=="Ready")].status
name: Ready
type: string
- jsonPath: .status.conditions[?(@.type=="Ready")].reason
name: Reason
type: string
name: v1alpha1
schema:
openAPIV3Schema:
description: TerdutEscalationRule is the Schema for the terdutescalationrules
API
properties:
apiVersion:
description: |-
APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
type: string
kind:
description: |-
Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
type: string
metadata:
type: object
spec:
description: spec defines the desired state of TerdutEscalationRule
properties:
fallbackTopic:
type: string
levels:
items:
description: |-
EscalationLevel is one rung of the ladder: how long to wait, and who to
page if nobody's acknowledged by then.
properties:
targets:
items:
description: |-
EscalationTarget is one page within a level. username is required iff
kind is "user" (terdut-server's own validation, internal/api/escalation.go's
handleSetEscalation -- mirrored here as a CEL rule so a bad spec is
rejected at apply time, not discovered on the next failed PUT).
properties:
kind:
description: EscalationTargetKind is who one rung of the
ladder pages.
enum:
- oncall
- user
type: string
username:
type: string
required:
- kind
type: object
x-kubernetes-validations:
- message: username is required when kind is user
rule: self.kind != 'user' || has(self.username)
- message: username must not be set when kind is oncall
rule: self.kind != 'oncall' || !has(self.username)
minItems: 1
type: array
timeout:
description: timeout is a Go duration string, e.g. "5m".
minLength: 1
type: string
required:
- targets
- timeout
type: object
minItems: 1
type: array
repeatCount:
format: int64
maximum: 10
minimum: 0
type: integer
teamRef:
description: |-
TerdutTeamRef names the TerdutTeam this resource belongs to. Always
same-namespace as the CR itself (DESIGN.md §1: only TerdutTeam.spec.serverRef
crosses namespaces in v1) -- no namespace field, unlike TerdutServerRef.
properties:
name:
minLength: 1
type: string
required:
- name
type: object
required:
- levels
- teamRef
type: object
status:
description: status defines the observed state of TerdutEscalationRule
properties:
conditions:
items:
description: Condition contains details for one aspect of the current
state of this API Resource.
properties:
lastTransitionTime:
description: |-
lastTransitionTime is the last time the condition transitioned from one status to another.
This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
format: date-time
type: string
message:
description: |-
message is a human readable message indicating details about the transition.
This may be an empty string.
maxLength: 32768
type: string
observedGeneration:
description: |-
observedGeneration represents the .metadata.generation that the condition was set based upon.
For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
with respect to the current state of the instance.
format: int64
minimum: 0
type: integer
reason:
description: |-
reason contains a programmatic identifier indicating the reason for the condition's last transition.
Producers of specific condition types may define expected values and meanings for this field,
and whether the values are considered a guaranteed API.
The value should be a CamelCase string.
This field may not be empty.
maxLength: 1024
minLength: 1
pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
type: string
status:
description: status of the condition, one of True, False, Unknown.
enum:
- "True"
- "False"
- Unknown
type: string
type:
description: type of condition in CamelCase or in foo.example.com/CamelCase.
maxLength: 316
pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
type: string
required:
- lastTransitionTime
- message
- reason
- status
- type
type: object
type: array
x-kubernetes-list-map-keys:
- type
x-kubernetes-list-type: map
observedGeneration:
format: int64
type: integer
type: object
required:
- spec
type: object
served: true
storage: true
subresources:
status: {}
@@ -171,6 +171,13 @@ spec:
observedGeneration:
format: int64
type: integer
serverEndpoint:
description: |-
serverEndpoint is the resolved TerdutServer's base URL, resolved once
here so no child controller (TerdutEscalationRule, TerdutDeadmanSwitch,
TerdutAlertSource) ever needs its own RBAC on terdutservers just to
find out where to send a request (DESIGN.md §5).
type: string
teamID:
description: |-
teamID is the server-side id -- needed by every child object's
+2
View File
@@ -4,6 +4,8 @@
resources:
- bases/terdut.ryuvia.com_terdutservers.yaml
- bases/terdut.ryuvia.com_terdutteams.yaml
- bases/terdut.ryuvia.com_terdutescalationrules.yaml
- bases/terdut.ryuvia.com_terdutdeadmanswitches.yaml
# +kubebuilder:scaffold:crdkustomizeresource
patches:
+6
View File
@@ -22,6 +22,12 @@ resources:
# default, aiding admins in cluster management. Those roles are
# not used by the terdut-operator itself. You can comment the following lines
# if you do not want those helpers be installed with your Project.
- terdutdeadmanswitch_admin_role.yaml
- terdutdeadmanswitch_editor_role.yaml
- terdutdeadmanswitch_viewer_role.yaml
- terdutescalationrule_admin_role.yaml
- terdutescalationrule_editor_role.yaml
- terdutescalationrule_viewer_role.yaml
- terdutteam_admin_role.yaml
- terdutteam_editor_role.yaml
- terdutteam_viewer_role.yaml
+6
View File
@@ -55,6 +55,8 @@ rules:
- apiGroups:
- terdut.ryuvia.com
resources:
- terdutdeadmanswitches
- terdutescalationrules
- terdutservers
- terdutteams
verbs:
@@ -68,6 +70,8 @@ rules:
- apiGroups:
- terdut.ryuvia.com
resources:
- terdutdeadmanswitches/finalizers
- terdutescalationrules/finalizers
- terdutservers/finalizers
- terdutteams/finalizers
verbs:
@@ -75,6 +79,8 @@ rules:
- apiGroups:
- terdut.ryuvia.com
resources:
- terdutdeadmanswitches/status
- terdutescalationrules/status
- terdutservers/status
- terdutteams/status
verbs:
@@ -0,0 +1,27 @@
# This rule is not used by the project terdut-operator itself.
# It is provided to allow the cluster admin to help manage permissions for users.
#
# Grants full permissions ('*') over terdut.ryuvia.com.
# This role is intended for users authorized to modify roles and bindings within the cluster,
# enabling them to delegate specific permissions to other users or groups as needed.
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
app.kubernetes.io/name: terdut-operator
app.kubernetes.io/managed-by: kustomize
name: terdutdeadmanswitch-admin-role
rules:
- apiGroups:
- terdut.ryuvia.com
resources:
- terdutdeadmanswitches
verbs:
- '*'
- apiGroups:
- terdut.ryuvia.com
resources:
- terdutdeadmanswitches/status
verbs:
- get
@@ -0,0 +1,33 @@
# This rule is not used by the project terdut-operator itself.
# It is provided to allow the cluster admin to help manage permissions for users.
#
# Grants permissions to create, update, and delete resources within the terdut.ryuvia.com.
# This role is intended for users who need to manage these resources
# but should not control RBAC or manage permissions for others.
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
app.kubernetes.io/name: terdut-operator
app.kubernetes.io/managed-by: kustomize
name: terdutdeadmanswitch-editor-role
rules:
- apiGroups:
- terdut.ryuvia.com
resources:
- terdutdeadmanswitches
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- terdut.ryuvia.com
resources:
- terdutdeadmanswitches/status
verbs:
- get
@@ -0,0 +1,29 @@
# This rule is not used by the project terdut-operator itself.
# It is provided to allow the cluster admin to help manage permissions for users.
#
# Grants read-only access to terdut.ryuvia.com resources.
# This role is intended for users who need visibility into these resources
# without permissions to modify them. It is ideal for monitoring purposes and limited-access viewing.
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
app.kubernetes.io/name: terdut-operator
app.kubernetes.io/managed-by: kustomize
name: terdutdeadmanswitch-viewer-role
rules:
- apiGroups:
- terdut.ryuvia.com
resources:
- terdutdeadmanswitches
verbs:
- get
- list
- watch
- apiGroups:
- terdut.ryuvia.com
resources:
- terdutdeadmanswitches/status
verbs:
- get
@@ -0,0 +1,27 @@
# This rule is not used by the project terdut-operator itself.
# It is provided to allow the cluster admin to help manage permissions for users.
#
# Grants full permissions ('*') over terdut.ryuvia.com.
# This role is intended for users authorized to modify roles and bindings within the cluster,
# enabling them to delegate specific permissions to other users or groups as needed.
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
app.kubernetes.io/name: terdut-operator
app.kubernetes.io/managed-by: kustomize
name: terdutescalationrule-admin-role
rules:
- apiGroups:
- terdut.ryuvia.com
resources:
- terdutescalationrules
verbs:
- '*'
- apiGroups:
- terdut.ryuvia.com
resources:
- terdutescalationrules/status
verbs:
- get
@@ -0,0 +1,33 @@
# This rule is not used by the project terdut-operator itself.
# It is provided to allow the cluster admin to help manage permissions for users.
#
# Grants permissions to create, update, and delete resources within the terdut.ryuvia.com.
# This role is intended for users who need to manage these resources
# but should not control RBAC or manage permissions for others.
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
app.kubernetes.io/name: terdut-operator
app.kubernetes.io/managed-by: kustomize
name: terdutescalationrule-editor-role
rules:
- apiGroups:
- terdut.ryuvia.com
resources:
- terdutescalationrules
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- terdut.ryuvia.com
resources:
- terdutescalationrules/status
verbs:
- get
@@ -0,0 +1,29 @@
# This rule is not used by the project terdut-operator itself.
# It is provided to allow the cluster admin to help manage permissions for users.
#
# Grants read-only access to terdut.ryuvia.com resources.
# This role is intended for users who need visibility into these resources
# without permissions to modify them. It is ideal for monitoring purposes and limited-access viewing.
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
app.kubernetes.io/name: terdut-operator
app.kubernetes.io/managed-by: kustomize
name: terdutescalationrule-viewer-role
rules:
- apiGroups:
- terdut.ryuvia.com
resources:
- terdutescalationrules
verbs:
- get
- list
- watch
- apiGroups:
- terdut.ryuvia.com
resources:
- terdutescalationrules/status
verbs:
- get
+2
View File
@@ -2,4 +2,6 @@
resources:
- terdut_v1alpha1_terdutserver.yaml
- terdut_v1alpha1_terdutteam.yaml
- terdut_v1alpha1_terdutescalationrule.yaml
- terdut_v1alpha1_terdutdeadmanswitch.yaml
# +kubebuilder:scaffold:manifestskustomizesamples
@@ -0,0 +1,15 @@
apiVersion: terdut.ryuvia.com/v1alpha1
kind: TerdutDeadmanSwitch
metadata:
labels:
app.kubernetes.io/name: terdut-operator
app.kubernetes.io/managed-by: kustomize
name: terdutdeadmanswitch-sample
spec:
teamRef:
name: terdutteam-sample
# name is optional -- left empty, terdut-server derives it from matcher's
# own canonical form (DESIGN.md §4.4).
matcher: "alertname=Watchdog"
timeout: 15m
severity: critical
@@ -0,0 +1,25 @@
apiVersion: terdut.ryuvia.com/v1alpha1
kind: TerdutEscalationRule
metadata:
labels:
app.kubernetes.io/name: terdut-operator
app.kubernetes.io/managed-by: kustomize
name: terdutescalationrule-sample
spec:
# One per team (DESIGN.md §4.3) -- a second TerdutEscalationRule naming
# the same teamRef would simply clobber this one every reconcile, since
# there's no admission-time check for it in v1.
teamRef:
name: terdutteam-sample
repeatCount: 2
fallbackTopic: platform-fallback
levels:
# username is required iff kind is "user", and rejected otherwise --
# enforced at apply time via CEL (api/v1alpha1/terdutescalationrule_types.go).
- timeout: 5m
targets:
- kind: user
username: alice
- timeout: 10m
targets:
- kind: oncall
+54
View File
@@ -0,0 +1,54 @@
package controller
import (
"context"
"fmt"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"sigs.k8s.io/controller-runtime/pkg/client"
terdutv1alpha1 "git.ryuvia.com/niklas/terdut-operator/api/v1alpha1"
"git.ryuvia.com/niklas/terdut-operator/internal/tdclient"
)
// childError carries a condition reason/message, the same role teamError
// and databaseError play for their own controllers: an expected,
// requeue-and-retry outcome, not a reconcile failure.
type childError struct {
reason string
message string
}
func (e *childError) Error() string { return e.message }
// resolveTeamAndClient implements DESIGN.md §5's "every child resolves its
// own teamRef -> TerdutTeam.status, never chains up to TerdutServer"
// rule -- shared by TerdutEscalationRule and TerdutDeadmanSwitch, which
// both need exactly this and nothing else to call terdut-server's API.
func resolveTeamAndClient(
ctx context.Context, c client.Client, operatorNamespace, namespace string,
ref terdutv1alpha1.TerdutTeamRef, newClient func(string) *tdclient.Client,
) (*terdutv1alpha1.TerdutTeam, *tdclient.Client, *childError) {
var team terdutv1alpha1.TerdutTeam
if err := c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: ref.Name}, &team); err != nil {
if apierrors.IsNotFound(err) {
return nil, nil, &childError{
reason: terdutv1alpha1.ReasonTeamRefNotFound,
message: fmt.Sprintf("TerdutTeam %q not found in namespace %q", ref.Name, namespace),
}
}
return nil, nil, &childError{reason: terdutv1alpha1.ReasonTeamRefNotFound, message: err.Error()}
}
if team.Status.CredentialsSecretRef == nil || team.Status.TeamID == 0 {
return nil, nil, &childError{
reason: terdutv1alpha1.ReasonWaitingForTeam,
message: fmt.Sprintf("TerdutTeam %q is not Ready yet", ref.Name),
}
}
teamKey, err := readOperatorSecret(ctx, c, operatorNamespace, team.Status.CredentialsSecretRef)
if err != nil {
return nil, nil, &childError{reason: terdutv1alpha1.ReasonWaitingForTeam, message: err.Error()}
}
return &team, newClient(team.Status.ServerEndpoint).WithToken(teamKey), nil
}
@@ -0,0 +1,199 @@
package controller
import (
"context"
"fmt"
"time"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
logf "sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/recorder"
terdutv1alpha1 "git.ryuvia.com/niklas/terdut-operator/api/v1alpha1"
"git.ryuvia.com/niklas/terdut-operator/internal/tdclient"
)
const deadmanFinalizerName = "terdut.ryuvia.com/terdutdeadmanswitch"
// TerdutDeadmanSwitchReconciler reconciles a TerdutDeadmanSwitch object.
type TerdutDeadmanSwitchReconciler struct {
client.Client
Scheme *runtime.Scheme
OperatorNamespace string
Recorder recorder.EventRecorder
NewClient func(endpoint string) *tdclient.Client
}
// +kubebuilder:rbac:groups=terdut.ryuvia.com,resources=terdutdeadmanswitches,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=terdut.ryuvia.com,resources=terdutdeadmanswitches/status,verbs=get;update;patch
// +kubebuilder:rbac:groups=terdut.ryuvia.com,resources=terdutdeadmanswitches/finalizers,verbs=update
// +kubebuilder:rbac:groups=terdut.ryuvia.com,resources=terdutteams,verbs=get;list;watch
// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch
// +kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=create;patch
func (r *TerdutDeadmanSwitchReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
log := logf.FromContext(ctx)
var sw terdutv1alpha1.TerdutDeadmanSwitch
if err := r.Get(ctx, req.NamespacedName, &sw); err != nil {
if apierrors.IsNotFound(err) {
return ctrl.Result{}, nil
}
return ctrl.Result{}, err
}
newClient := r.NewClient
if newClient == nil {
newClient = tdclient.New
}
if !sw.DeletionTimestamp.IsZero() {
return r.reconcileDeadmanDelete(ctx, &sw, newClient)
}
if !controllerutil.ContainsFinalizer(&sw, deadmanFinalizerName) {
controllerutil.AddFinalizer(&sw, deadmanFinalizerName)
if err := r.Update(ctx, &sw); err != nil {
return ctrl.Result{}, err
}
return ctrl.Result{}, nil
}
team, tc, resolveErr := resolveTeamAndClient(ctx, r.Client, r.OperatorNamespace, sw.Namespace, sw.Spec.TeamRef, newClient)
if resolveErr != nil {
return r.setDeadmanNotReady(ctx, &sw, resolveErr.reason, resolveErr.message, waitInterval)
}
timeout, err := time.ParseDuration(sw.Spec.Timeout)
if err != nil {
return ctrl.Result{}, fmt.Errorf("spec.timeout %q: %w", sw.Spec.Timeout, err)
}
severity := sw.Spec.Severity
if severity == "" {
severity = "critical"
}
if sw.Status.SwitchID == 0 {
if err := r.createOrAdoptDeadmanSwitch(ctx, &sw, tc, team.Status.TeamID, timeout, severity); err != nil {
return ctrl.Result{}, err
}
} else if err := tc.UpdateDeadmanSwitch(ctx, team.Status.TeamID, sw.Status.SwitchID, sw.Spec.Name, sw.Spec.Matcher, int64(timeout.Seconds()), severity); err != nil {
return ctrl.Result{}, fmt.Errorf("PUT /api/teams/%d/deadman/switches/%d: %w", team.Status.TeamID, sw.Status.SwitchID, err)
}
meta.SetStatusCondition(&sw.Status.Conditions, metav1.Condition{
Type: terdutv1alpha1.ConditionReady,
Status: metav1.ConditionTrue,
Reason: terdutv1alpha1.ReasonChildAdopted,
Message: fmt.Sprintf("switch %d applied on team %d", sw.Status.SwitchID, team.Status.TeamID),
})
sw.Status.ObservedGeneration = sw.Generation
if err := r.Status().Update(ctx, &sw); err != nil {
return ctrl.Result{}, err
}
if r.Recorder != nil {
r.Recorder.Eventf(&sw, nil, corev1.EventTypeNormal, terdutv1alpha1.ReasonChildAdopted, terdutv1alpha1.ReasonChildAdopted,
"dead man's switch applied")
}
log.Info("TerdutDeadmanSwitch applied", "name", sw.Name, "switchID", sw.Status.SwitchID)
return ctrl.Result{RequeueAfter: resyncInterval}, nil
}
// createOrAdoptDeadmanSwitch implements this resource's own idempotent-
// create shape (DESIGN.md §4.4, §5): there's no unique-name constraint
// server-side to 409 on, so this lists first and matches by name (the
// server's own derived name, when spec.name is empty) rather than adopting
// after a conflict the API would never actually raise.
func (r *TerdutDeadmanSwitchReconciler) createOrAdoptDeadmanSwitch(
ctx context.Context, sw *terdutv1alpha1.TerdutDeadmanSwitch, tc *tdclient.Client,
teamID int64, timeout time.Duration, severity string,
) error {
existing, err := tc.ListDeadmanSwitches(ctx, teamID)
if err != nil {
return fmt.Errorf("GET /api/teams/%d/deadman/switches: %w", teamID, err)
}
if sw.Spec.Name != "" {
for _, s := range existing {
if s.Name == sw.Spec.Name {
sw.Status.SwitchID = s.ID
return nil
}
}
}
created, err := tc.CreateDeadmanSwitch(ctx, teamID, sw.Spec.Name, sw.Spec.Matcher, int64(timeout.Seconds()), severity)
if err != nil {
return fmt.Errorf("POST /api/teams/%d/deadman/switches: %w", teamID, err)
}
sw.Status.SwitchID = created.ID
return nil
}
func (r *TerdutDeadmanSwitchReconciler) setDeadmanNotReady(
ctx context.Context, sw *terdutv1alpha1.TerdutDeadmanSwitch, reason, message string, d time.Duration,
) (ctrl.Result, error) {
meta.SetStatusCondition(&sw.Status.Conditions, metav1.Condition{
Type: terdutv1alpha1.ConditionReady,
Status: metav1.ConditionFalse,
Reason: reason,
Message: message,
})
sw.Status.ObservedGeneration = sw.Generation
if err := r.Status().Update(ctx, sw); err != nil {
return ctrl.Result{}, err
}
if r.Recorder != nil {
r.Recorder.Eventf(sw, nil, corev1.EventTypeWarning, reason, reason, message)
}
return ctrl.Result{RequeueAfter: d}, nil
}
// reconcileDeadmanDelete calls the real DELETE this resource actually has
// (unlike TerdutEscalationRule) if the team is still resolvable and a
// switch was ever created, then removes the finalizer unconditionally.
func (r *TerdutDeadmanSwitchReconciler) reconcileDeadmanDelete(
ctx context.Context, sw *terdutv1alpha1.TerdutDeadmanSwitch, newClient func(string) *tdclient.Client,
) (ctrl.Result, error) {
if !controllerutil.ContainsFinalizer(sw, deadmanFinalizerName) {
return ctrl.Result{}, nil
}
if sw.Status.SwitchID != 0 {
if team, tc, resolveErr := resolveTeamAndClient(
ctx, r.Client, r.OperatorNamespace, sw.Namespace, sw.Spec.TeamRef, newClient,
); resolveErr == nil {
if err := tc.DeleteDeadmanSwitch(ctx, team.Status.TeamID, sw.Status.SwitchID); err != nil {
if r.Recorder != nil {
r.Recorder.Eventf(sw, nil, corev1.EventTypeWarning, "DeleteFailed", "DeleteFailed", err.Error())
}
return ctrl.Result{}, err
}
}
}
controllerutil.RemoveFinalizer(sw, deadmanFinalizerName)
return ctrl.Result{}, r.Update(ctx, sw)
}
// SetupWithManager sets up the controller with the Manager.
func (r *TerdutDeadmanSwitchReconciler) SetupWithManager(mgr ctrl.Manager) error {
if r.NewClient == nil {
r.NewClient = tdclient.New
}
if r.Recorder == nil {
r.Recorder = mgr.GetEventRecorder("terdutdeadmanswitch-controller")
}
return ctrl.NewControllerManagedBy(mgr).
For(&terdutv1alpha1.TerdutDeadmanSwitch{}).
Named("terdutdeadmanswitch").
Complete(r)
}
@@ -0,0 +1,213 @@
package controller
import (
"context"
"net/http/httptest"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
terdutv1alpha1 "git.ryuvia.com/niklas/terdut-operator/api/v1alpha1"
"git.ryuvia.com/niklas/terdut-operator/internal/tdclient"
)
var _ = Describe("TerdutDeadmanSwitch Controller", func() {
const operatorNamespace = "default"
var (
reconciler *TerdutDeadmanSwitchReconciler
fake *fakeTerdutServer
fakeSrv *httptest.Server
srv *terdutv1alpha1.TerdutServer
team *terdutv1alpha1.TerdutTeam
swName string
swKey types.NamespacedName
)
BeforeEach(func(ctx SpecContext) {
fake, fakeSrv = newFakeTerdutServer()
DeferCleanup(fakeSrv.Close)
srv = bootstrapReadyTerdutServer(ctx, operatorNamespace, uniqueName("dmserver"), fakeSrv.URL)
team = readyTerdutTeam(ctx, operatorNamespace, uniqueName("dmteam"), srv, fakeSrv.URL)
reconciler = &TerdutDeadmanSwitchReconciler{
Client: k8sClient,
Scheme: k8sClient.Scheme(),
OperatorNamespace: operatorNamespace,
NewClient: func(string) *tdclient.Client { return tdclient.New(fakeSrv.URL) },
}
swName = uniqueName("switch")
swKey = types.NamespacedName{Name: swName, Namespace: operatorNamespace}
})
AfterEach(func(ctx SpecContext) {
sw := &terdutv1alpha1.TerdutDeadmanSwitch{}
if err := k8sClient.Get(ctx, swKey, sw); err == nil {
sw.Finalizers = nil
_ = k8sClient.Update(ctx, sw)
_ = k8sClient.Delete(ctx, sw)
}
teamKey := types.NamespacedName{Name: team.Name, Namespace: operatorNamespace}
if err := k8sClient.Get(ctx, teamKey, team); err == nil {
team.Finalizers = nil
_ = k8sClient.Update(ctx, team)
_ = k8sClient.Delete(ctx, team)
}
srvKey := types.NamespacedName{Name: srv.Name, Namespace: operatorNamespace}
if err := k8sClient.Get(ctx, srvKey, srv); err == nil {
srv.Finalizers = nil
_ = k8sClient.Update(ctx, srv)
_ = k8sClient.Delete(ctx, srv)
}
})
createSwitch := func(ctx context.Context, teamRef terdutv1alpha1.TerdutTeamRef, name, matcher, timeout string) {
sw := &terdutv1alpha1.TerdutDeadmanSwitch{
ObjectMeta: metav1.ObjectMeta{Name: swName, Namespace: operatorNamespace},
Spec: terdutv1alpha1.TerdutDeadmanSwitchSpec{
TeamRef: teamRef,
Name: name,
Matcher: matcher,
Timeout: timeout,
},
}
Expect(k8sClient.Create(ctx, sw)).To(Succeed())
}
reconcileOnce := func(ctx context.Context) {
_, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: swKey})
Expect(err).NotTo(HaveOccurred())
}
readyCondition := func(ctx context.Context) metav1.Condition {
sw := &terdutv1alpha1.TerdutDeadmanSwitch{}
Expect(k8sClient.Get(ctx, swKey, sw)).To(Succeed())
c := meta.FindStatusCondition(sw.Status.Conditions, terdutv1alpha1.ConditionReady)
Expect(c).NotTo(BeNil())
return *c
}
sameTeamRef := func() terdutv1alpha1.TerdutTeamRef {
return terdutv1alpha1.TerdutTeamRef{Name: team.Name}
}
Describe("the happy path", func() {
It("creates the switch server-side", func(ctx SpecContext) {
createSwitch(ctx, sameTeamRef(), "heartbeat", "alertname=Watchdog", "15m")
reconcileOnce(ctx) // finalizer
reconcileOnce(ctx) // create
cond := readyCondition(ctx)
Expect(cond.Status).To(Equal(metav1.ConditionTrue))
Expect(cond.Reason).To(Equal(terdutv1alpha1.ReasonChildAdopted))
sw := &terdutv1alpha1.TerdutDeadmanSwitch{}
Expect(k8sClient.Get(ctx, swKey, sw)).To(Succeed())
Expect(sw.Status.SwitchID).NotTo(BeZero())
created, ok := fake.switches[team.Status.TeamID][sw.Status.SwitchID]
Expect(ok).To(BeTrue())
Expect(created.Matcher).To(Equal("alertname=Watchdog"))
Expect(created.TimeoutSeconds).To(Equal(int64(900)))
Expect(created.Severity).To(Equal("critical")) // kubebuilder default
})
})
Describe("list-and-match-by-name adoption", func() {
It("adopts an already-created switch instead of creating a duplicate", func(ctx SpecContext) {
// Simulates a prior, interrupted reconcile that got as far as
// POSTing the switch -- no 409 signal exists for this resource
// (DESIGN.md §4.4), so the recovery path is GET-list-and-match,
// not adopt-on-409.
fake.nextSwitchID = 1
fake.switches[team.Status.TeamID] = map[int64]tdclient.DeadmanSwitch{
1: {ID: 1, Name: "heartbeat", Matcher: "alertname=Watchdog", TimeoutSeconds: 900, Severity: "critical"},
}
createSwitch(ctx, sameTeamRef(), "heartbeat", "alertname=Watchdog", "15m")
reconcileOnce(ctx) // finalizer
reconcileOnce(ctx)
sw := &terdutv1alpha1.TerdutDeadmanSwitch{}
Expect(k8sClient.Get(ctx, swKey, sw)).To(Succeed())
Expect(sw.Status.SwitchID).To(Equal(int64(1)))
Expect(fake.switches[team.Status.TeamID]).To(HaveLen(1), "should not have created a second switch")
})
})
Describe("update-in-place on spec drift", func() {
It("PUTs the new spec rather than creating a second switch", func(ctx SpecContext) {
createSwitch(ctx, sameTeamRef(), "heartbeat", "alertname=Watchdog", "15m")
reconcileOnce(ctx) // finalizer
reconcileOnce(ctx)
Expect(readyCondition(ctx).Status).To(Equal(metav1.ConditionTrue))
sw := &terdutv1alpha1.TerdutDeadmanSwitch{}
Expect(k8sClient.Get(ctx, swKey, sw)).To(Succeed())
switchID := sw.Status.SwitchID
sw.Spec.Timeout = "30m"
Expect(k8sClient.Update(ctx, sw)).To(Succeed())
reconcileOnce(ctx)
Expect(fake.switches[team.Status.TeamID]).To(HaveLen(1), "update-in-place, not a second switch")
Expect(fake.switches[team.Status.TeamID][switchID].TimeoutSeconds).To(Equal(int64(1800)))
})
})
Describe("waiting on the referenced TerdutTeam", func() {
It("reports TeamRefNotFound when the TerdutTeam doesn't exist", func(ctx SpecContext) {
createSwitch(ctx, terdutv1alpha1.TerdutTeamRef{Name: testRefNotFoundName}, "heartbeat", "alertname=Watchdog", "15m")
reconcileOnce(ctx) // finalizer
reconcileOnce(ctx)
Expect(readyCondition(ctx).Reason).To(Equal(terdutv1alpha1.ReasonTeamRefNotFound))
})
It("reports WaitingForTeam when the TerdutTeam exists but isn't Ready yet", func(ctx SpecContext) {
unreadyName := uniqueName("dmteam-unready")
unready := &terdutv1alpha1.TerdutTeam{
ObjectMeta: metav1.ObjectMeta{Name: unreadyName, Namespace: operatorNamespace},
Spec: terdutv1alpha1.TerdutTeamSpec{
ServerRef: terdutv1alpha1.TerdutServerRef{Name: srv.Name},
DisplayName: "unready",
},
}
Expect(k8sClient.Create(ctx, unready)).To(Succeed())
DeferCleanup(func() { _ = k8sClient.Delete(ctx, unready) })
createSwitch(ctx, terdutv1alpha1.TerdutTeamRef{Name: unreadyName}, "heartbeat", "alertname=Watchdog", "15m")
reconcileOnce(ctx) // finalizer
reconcileOnce(ctx)
Expect(readyCondition(ctx).Reason).To(Equal(terdutv1alpha1.ReasonWaitingForTeam))
})
})
Describe("deletion", func() {
It("deletes the switch server-side (the real DELETE this resource has) and removes the finalizer", func(ctx SpecContext) {
createSwitch(ctx, sameTeamRef(), "heartbeat", "alertname=Watchdog", "15m")
reconcileOnce(ctx)
reconcileOnce(ctx)
Expect(readyCondition(ctx).Status).To(Equal(metav1.ConditionTrue))
sw := &terdutv1alpha1.TerdutDeadmanSwitch{}
Expect(k8sClient.Get(ctx, swKey, sw)).To(Succeed())
switchID := sw.Status.SwitchID
Expect(k8sClient.Delete(ctx, sw)).To(Succeed())
reconcileOnce(ctx) // runs the finalizer
Expect(fake.switchDelete[switchID]).To(BeTrue())
err := k8sClient.Get(ctx, swKey, sw)
Expect(err).To(HaveOccurred(), "the TerdutDeadmanSwitch itself should be gone once the finalizer clears")
})
})
})
@@ -0,0 +1,217 @@
package controller
import (
"context"
"fmt"
"time"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
logf "sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/recorder"
terdutv1alpha1 "git.ryuvia.com/niklas/terdut-operator/api/v1alpha1"
"git.ryuvia.com/niklas/terdut-operator/internal/tdclient"
)
// escalationFinalizerName exists for the one undo PUT /api/teams/{teamID}/escalation
// supports, on delete: an empty policy (DESIGN.md §5's general finalizer
// rule expects a server-side counterpart to be undone, but this resource's
// API is GET/PUT-only, with no DELETE at all -- a zeroed PUT is the closest
// equivalent terdut-server itself recognizes as "no policy"
// (escalationPolicy.configured(), confirmed against source: "a policy row
// with no levels is the same as no policy").
const escalationFinalizerName = "terdut.ryuvia.com/terdutescalationrule"
// TerdutEscalationRuleReconciler reconciles a TerdutEscalationRule object.
type TerdutEscalationRuleReconciler struct {
client.Client
Scheme *runtime.Scheme
OperatorNamespace string
Recorder recorder.EventRecorder
NewClient func(endpoint string) *tdclient.Client
}
// +kubebuilder:rbac:groups=terdut.ryuvia.com,resources=terdutescalationrules,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=terdut.ryuvia.com,resources=terdutescalationrules/status,verbs=get;update;patch
// +kubebuilder:rbac:groups=terdut.ryuvia.com,resources=terdutescalationrules/finalizers,verbs=update
// +kubebuilder:rbac:groups=terdut.ryuvia.com,resources=terdutteams,verbs=get;list;watch
// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch
// +kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=create;patch
func (r *TerdutEscalationRuleReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
log := logf.FromContext(ctx)
var rule terdutv1alpha1.TerdutEscalationRule
if err := r.Get(ctx, req.NamespacedName, &rule); err != nil {
if apierrors.IsNotFound(err) {
return ctrl.Result{}, nil
}
return ctrl.Result{}, err
}
newClient := r.NewClient
if newClient == nil {
newClient = tdclient.New
}
if !rule.DeletionTimestamp.IsZero() {
return r.reconcileEscalationDelete(ctx, &rule, newClient)
}
if !controllerutil.ContainsFinalizer(&rule, escalationFinalizerName) {
controllerutil.AddFinalizer(&rule, escalationFinalizerName)
if err := r.Update(ctx, &rule); err != nil {
return ctrl.Result{}, err
}
return ctrl.Result{}, nil
}
team, tc, resolveErr := resolveTeamAndClient(ctx, r.Client, r.OperatorNamespace, rule.Namespace, rule.Spec.TeamRef, newClient)
if resolveErr != nil {
return r.setEscalationNotReady(ctx, &rule, resolveErr.reason, resolveErr.message, waitInterval)
}
body, unknownUser, err := buildEscalationRequest(ctx, tc, rule.Spec)
if err != nil {
return ctrl.Result{}, err
}
if unknownUser != "" {
return r.setEscalationNotReady(ctx, &rule, terdutv1alpha1.ReasonUnknownUser,
fmt.Sprintf("username %q does not resolve to any user", unknownUser), waitInterval)
}
if err := tc.SetEscalation(ctx, team.Status.TeamID, body); err != nil {
return ctrl.Result{}, fmt.Errorf("PUT /api/teams/%d/escalation: %w", team.Status.TeamID, err)
}
meta.SetStatusCondition(&rule.Status.Conditions, metav1.Condition{
Type: terdutv1alpha1.ConditionReady, // "Ready" -- same name, shared across every CRD (DESIGN.md §7)
Status: metav1.ConditionTrue,
Reason: terdutv1alpha1.ReasonChildAdopted,
Message: fmt.Sprintf("escalation policy applied to team %d", team.Status.TeamID),
})
rule.Status.ObservedGeneration = rule.Generation
if err := r.Status().Update(ctx, &rule); err != nil {
return ctrl.Result{}, err
}
if r.Recorder != nil {
r.Recorder.Eventf(&rule, nil, corev1.EventTypeNormal, terdutv1alpha1.ReasonChildAdopted, terdutv1alpha1.ReasonChildAdopted,
"escalation policy applied")
}
log.Info("TerdutEscalationRule applied", "name", rule.Name, "teamID", team.Status.TeamID)
return ctrl.Result{RequeueAfter: resyncInterval}, nil
}
// buildEscalationRequest resolves every "user" target's username to a
// user_id (DESIGN.md §4.3) and translates spec into the wire shape
// SetEscalation sends. Returns the first unresolvable username, if any,
// distinct from a plain error: that's an expected, reportable condition
// (ReasonUnknownUser), not a reconcile failure.
func buildEscalationRequest(
ctx context.Context, tc *tdclient.Client, spec terdutv1alpha1.TerdutEscalationRuleSpec,
) (tdclient.SetEscalationRequest, string, error) {
levels := make([]tdclient.EscalationLevelRequest, len(spec.Levels))
for i, lvl := range spec.Levels {
timeout, err := time.ParseDuration(lvl.Timeout)
if err != nil {
return tdclient.SetEscalationRequest{}, "", fmt.Errorf("spec.levels[%d].timeout %q: %w", i, lvl.Timeout, err)
}
targets := make([]tdclient.EscalationTargetRequest, len(lvl.Targets))
for j, t := range lvl.Targets {
if t.Kind == terdutv1alpha1.EscalationTargetOncall {
targets[j] = tdclient.EscalationTargetRequest{Kind: string(terdutv1alpha1.EscalationTargetOncall)}
continue
}
user, err := tc.GetUserByUsername(ctx, t.Username)
if err != nil {
return tdclient.SetEscalationRequest{}, "", fmt.Errorf("GET /api/users (resolving %q): %w", t.Username, err)
}
if user == nil {
return tdclient.SetEscalationRequest{}, t.Username, nil
}
targets[j] = tdclient.EscalationTargetRequest{Kind: string(terdutv1alpha1.EscalationTargetUser), UserID: &user.ID}
}
levels[i] = tdclient.EscalationLevelRequest{
Position: int64(i + 1),
TimeoutSeconds: int64(timeout.Seconds()),
Targets: targets,
}
}
return tdclient.SetEscalationRequest{
RepeatCount: spec.RepeatCount,
FallbackTopic: spec.FallbackTopic,
Levels: levels,
}, "", nil
}
func (r *TerdutEscalationRuleReconciler) setEscalationNotReady(
ctx context.Context, rule *terdutv1alpha1.TerdutEscalationRule, reason, message string, d time.Duration,
) (ctrl.Result, error) {
meta.SetStatusCondition(&rule.Status.Conditions, metav1.Condition{
Type: terdutv1alpha1.ConditionReady,
Status: metav1.ConditionFalse,
Reason: reason,
Message: message,
})
rule.Status.ObservedGeneration = rule.Generation
if err := r.Status().Update(ctx, rule); err != nil {
return ctrl.Result{}, err
}
if r.Recorder != nil {
r.Recorder.Eventf(rule, nil, corev1.EventTypeWarning, reason, reason, message)
}
return ctrl.Result{RequeueAfter: d}, nil
}
// reconcileEscalationDelete PUTs an empty policy (this resource's only
// available "undo", per this file's own const comment) if the team is
// still resolvable, then removes the finalizer unconditionally -- same
// "the parent's probably going away too" reasoning TerdutTeam's own delete
// path uses for a TerdutServer that's gone.
func (r *TerdutEscalationRuleReconciler) reconcileEscalationDelete(
ctx context.Context, rule *terdutv1alpha1.TerdutEscalationRule, newClient func(string) *tdclient.Client,
) (ctrl.Result, error) {
if !controllerutil.ContainsFinalizer(rule, escalationFinalizerName) {
return ctrl.Result{}, nil
}
if team, tc, resolveErr := resolveTeamAndClient(
ctx, r.Client, r.OperatorNamespace, rule.Namespace, rule.Spec.TeamRef, newClient,
); resolveErr == nil {
if err := tc.SetEscalation(ctx, team.Status.TeamID, tdclient.SetEscalationRequest{Levels: []tdclient.EscalationLevelRequest{}}); err != nil {
if r.Recorder != nil {
r.Recorder.Eventf(rule, nil, corev1.EventTypeWarning, "DeleteFailed", "DeleteFailed", err.Error())
}
return ctrl.Result{}, err
}
}
controllerutil.RemoveFinalizer(rule, escalationFinalizerName)
return ctrl.Result{}, r.Update(ctx, rule)
}
// SetupWithManager sets up the controller with the Manager.
func (r *TerdutEscalationRuleReconciler) SetupWithManager(mgr ctrl.Manager) error {
if r.NewClient == nil {
r.NewClient = tdclient.New
}
if r.Recorder == nil {
r.Recorder = mgr.GetEventRecorder("terdutescalationrule-controller")
}
return ctrl.NewControllerManagedBy(mgr).
For(&terdutv1alpha1.TerdutEscalationRule{}).
Named("terdutescalationrule").
Complete(r)
}
@@ -0,0 +1,206 @@
package controller
import (
"context"
"net/http/httptest"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
terdutv1alpha1 "git.ryuvia.com/niklas/terdut-operator/api/v1alpha1"
"git.ryuvia.com/niklas/terdut-operator/internal/tdclient"
)
var _ = Describe("TerdutEscalationRule Controller", func() {
const operatorNamespace = "default"
var (
reconciler *TerdutEscalationRuleReconciler
fake *fakeTerdutServer
fakeSrv *httptest.Server
srv *terdutv1alpha1.TerdutServer
team *terdutv1alpha1.TerdutTeam
ruleName string
ruleKey types.NamespacedName
)
BeforeEach(func(ctx SpecContext) {
fake, fakeSrv = newFakeTerdutServer()
DeferCleanup(fakeSrv.Close)
srv = bootstrapReadyTerdutServer(ctx, operatorNamespace, uniqueName("erserver"), fakeSrv.URL)
team = readyTerdutTeam(ctx, operatorNamespace, uniqueName("erteam"), srv, fakeSrv.URL)
reconciler = &TerdutEscalationRuleReconciler{
Client: k8sClient,
Scheme: k8sClient.Scheme(),
OperatorNamespace: operatorNamespace,
NewClient: func(string) *tdclient.Client { return tdclient.New(fakeSrv.URL) },
}
ruleName = uniqueName("escalation")
ruleKey = types.NamespacedName{Name: ruleName, Namespace: operatorNamespace}
})
AfterEach(func(ctx SpecContext) {
rule := &terdutv1alpha1.TerdutEscalationRule{}
if err := k8sClient.Get(ctx, ruleKey, rule); err == nil {
rule.Finalizers = nil
_ = k8sClient.Update(ctx, rule)
_ = k8sClient.Delete(ctx, rule)
}
// team/srv carry their own finalizers from readyTerdutTeam/
// bootstrapReadyTerdutServer -- clear them directly the same way
// terdutteam_controller_test.go's own AfterEach does, rather than
// relying on either reconciler to ever run again here.
teamKey := types.NamespacedName{Name: team.Name, Namespace: operatorNamespace}
if err := k8sClient.Get(ctx, teamKey, team); err == nil {
team.Finalizers = nil
_ = k8sClient.Update(ctx, team)
_ = k8sClient.Delete(ctx, team)
}
srvKey := types.NamespacedName{Name: srv.Name, Namespace: operatorNamespace}
if err := k8sClient.Get(ctx, srvKey, srv); err == nil {
srv.Finalizers = nil
_ = k8sClient.Update(ctx, srv)
_ = k8sClient.Delete(ctx, srv)
}
})
createRule := func(ctx context.Context, teamRef terdutv1alpha1.TerdutTeamRef, levels []terdutv1alpha1.EscalationLevel) {
rule := &terdutv1alpha1.TerdutEscalationRule{
ObjectMeta: metav1.ObjectMeta{Name: ruleName, Namespace: operatorNamespace},
Spec: terdutv1alpha1.TerdutEscalationRuleSpec{
TeamRef: teamRef,
Levels: levels,
},
}
Expect(k8sClient.Create(ctx, rule)).To(Succeed())
}
reconcileOnce := func(ctx context.Context) {
_, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: ruleKey})
Expect(err).NotTo(HaveOccurred())
}
readyCondition := func(ctx context.Context) metav1.Condition {
rule := &terdutv1alpha1.TerdutEscalationRule{}
Expect(k8sClient.Get(ctx, ruleKey, rule)).To(Succeed())
c := meta.FindStatusCondition(rule.Status.Conditions, terdutv1alpha1.ConditionReady)
Expect(c).NotTo(BeNil())
return *c
}
sameTeamRef := func() terdutv1alpha1.TerdutTeamRef {
return terdutv1alpha1.TerdutTeamRef{Name: team.Name}
}
Describe("the happy path", func() {
It("resolves usernames and applies the escalation policy", func(ctx SpecContext) {
fake.seedUser("alice")
createRule(ctx, sameTeamRef(), []terdutv1alpha1.EscalationLevel{
{Timeout: "5m", Targets: []terdutv1alpha1.EscalationTarget{
{Kind: terdutv1alpha1.EscalationTargetUser, Username: "alice"},
}},
{Timeout: "10m", Targets: []terdutv1alpha1.EscalationTarget{
{Kind: terdutv1alpha1.EscalationTargetOncall},
}},
})
reconcileOnce(ctx) // finalizer
reconcileOnce(ctx) // resolve + apply
cond := readyCondition(ctx)
Expect(cond.Status).To(Equal(metav1.ConditionTrue))
Expect(cond.Reason).To(Equal(terdutv1alpha1.ReasonChildAdopted))
applied, ok := fake.escalation[team.Status.TeamID]
Expect(ok).To(BeTrue())
Expect(applied.Levels).To(HaveLen(2))
Expect(applied.Levels[0].Targets[0].Kind).To(Equal("user"))
Expect(applied.Levels[0].Targets[0].UserID).NotTo(BeNil())
Expect(*applied.Levels[0].Targets[0].UserID).To(Equal(fake.users["alice"]))
Expect(applied.Levels[1].Targets[0].Kind).To(Equal("oncall"))
Expect(applied.Levels[1].Targets[0].UserID).To(BeNil())
})
})
Describe("an unresolvable username", func() {
It("reports UnknownUser and never calls PUT /api/teams/{id}/escalation", func(ctx SpecContext) {
createRule(ctx, sameTeamRef(), []terdutv1alpha1.EscalationLevel{
{Timeout: "5m", Targets: []terdutv1alpha1.EscalationTarget{
{Kind: terdutv1alpha1.EscalationTargetUser, Username: "ghost"},
}},
})
reconcileOnce(ctx) // finalizer
reconcileOnce(ctx)
cond := readyCondition(ctx)
Expect(cond.Status).To(Equal(metav1.ConditionFalse))
Expect(cond.Reason).To(Equal(terdutv1alpha1.ReasonUnknownUser))
_, applied := fake.escalation[team.Status.TeamID]
Expect(applied).To(BeFalse())
})
})
Describe("waiting on the referenced TerdutTeam", func() {
It("reports TeamRefNotFound when the TerdutTeam doesn't exist", func(ctx SpecContext) {
createRule(ctx, terdutv1alpha1.TerdutTeamRef{Name: testRefNotFoundName}, []terdutv1alpha1.EscalationLevel{
{Timeout: "5m", Targets: []terdutv1alpha1.EscalationTarget{{Kind: terdutv1alpha1.EscalationTargetOncall}}},
})
reconcileOnce(ctx) // finalizer
reconcileOnce(ctx)
Expect(readyCondition(ctx).Reason).To(Equal(terdutv1alpha1.ReasonTeamRefNotFound))
})
It("reports WaitingForTeam when the TerdutTeam exists but isn't Ready yet", func(ctx SpecContext) {
unreadyName := uniqueName("erteam-unready")
unready := &terdutv1alpha1.TerdutTeam{
ObjectMeta: metav1.ObjectMeta{Name: unreadyName, Namespace: operatorNamespace},
Spec: terdutv1alpha1.TerdutTeamSpec{
ServerRef: terdutv1alpha1.TerdutServerRef{Name: srv.Name},
DisplayName: "unready",
},
}
Expect(k8sClient.Create(ctx, unready)).To(Succeed())
DeferCleanup(func() { _ = k8sClient.Delete(ctx, unready) })
createRule(ctx, terdutv1alpha1.TerdutTeamRef{Name: unreadyName}, []terdutv1alpha1.EscalationLevel{
{Timeout: "5m", Targets: []terdutv1alpha1.EscalationTarget{{Kind: terdutv1alpha1.EscalationTargetOncall}}},
})
reconcileOnce(ctx) // finalizer
reconcileOnce(ctx)
Expect(readyCondition(ctx).Reason).To(Equal(terdutv1alpha1.ReasonWaitingForTeam))
})
})
Describe("deletion", func() {
It("PUTs an empty policy (this resource's only available undo) and removes the finalizer", func(ctx SpecContext) {
fake.seedUser("alice")
createRule(ctx, sameTeamRef(), []terdutv1alpha1.EscalationLevel{
{Timeout: "5m", Targets: []terdutv1alpha1.EscalationTarget{
{Kind: terdutv1alpha1.EscalationTargetUser, Username: "alice"},
}},
})
reconcileOnce(ctx)
reconcileOnce(ctx)
Expect(readyCondition(ctx).Status).To(Equal(metav1.ConditionTrue))
Expect(fake.escalation[team.Status.TeamID].Levels).To(HaveLen(1))
rule := &terdutv1alpha1.TerdutEscalationRule{}
Expect(k8sClient.Get(ctx, ruleKey, rule)).To(Succeed())
Expect(k8sClient.Delete(ctx, rule)).To(Succeed())
reconcileOnce(ctx) // runs the finalizer
Expect(fake.escalation[team.Status.TeamID].Levels).To(BeEmpty())
err := k8sClient.Get(ctx, ruleKey, rule)
Expect(err).To(HaveOccurred(), "the TerdutEscalationRule itself should be gone once the finalizer clears")
})
})
})
@@ -6,6 +6,8 @@ import (
"fmt"
"net/http"
"net/http/httptest"
"strconv"
"strings"
"sync"
. "github.com/onsi/ginkgo/v2"
@@ -51,20 +53,54 @@ type fakeTerdutServer struct {
teamNames map[int64]string // id -> current name (renames update this)
teamOIDC map[int64][2]string
teamDelete map[int64]bool // id -> true once DELETEd, for 404-on-redelete
// users backs GET /api/users for TerdutEscalationRule's username
// resolution (DESIGN.md §4.3) -- a fixed, pre-seeded directory, since
// nothing in this controller's own flow ever creates a user.
users map[string]int64 // username -> id
// escalation backs PUT /api/teams/{id}/escalation -- an upsert
// server-side (confirmed against source), so this is just "the last
// body PUT for this team", keyed by teamID, with no separate create
// step to model.
escalation map[int64]tdclient.SetEscalationRequest
// switches/nextSwitchID/switchDelete back the dead man's switch
// endpoints -- no unique-name constraint server-side (DESIGN.md §4.4),
// so switches is keyed by id, not name, same as the real API's own
// GET-list-and-match-by-name idempotent-create shape requires.
nextSwitchID int64
switches map[int64]map[int64]tdclient.DeadmanSwitch // teamID -> switchID -> switch
switchDelete map[int64]bool // switchID -> true once DELETEd, for 404-on-redelete
}
func newFakeTerdutServer() (*fakeTerdutServer, *httptest.Server) {
f := &fakeTerdutServer{
accounts: map[string]int64{},
keyMints: map[int64]int{},
teams: map[string]int64{},
teamNames: map[int64]string{},
teamOIDC: map[int64][2]string{},
teamDelete: map[int64]bool{},
accounts: map[string]int64{},
keyMints: map[int64]int{},
teams: map[string]int64{},
teamNames: map[int64]string{},
teamOIDC: map[int64][2]string{},
teamDelete: map[int64]bool{},
users: map[string]int64{},
escalation: map[int64]tdclient.SetEscalationRequest{},
switches: map[int64]map[int64]tdclient.DeadmanSwitch{},
switchDelete: map[int64]bool{},
}
return f, httptest.NewServer(f)
}
// seedUser registers a username the fake GET /api/users will return --
// called from test setup, before the controller under test ever runs.
// Callers read the assigned id back from f.users themselves, so this has
// nothing left to return.
func (f *fakeTerdutServer) seedUser(username string) {
f.mu.Lock()
defer f.mu.Unlock()
f.nextID++
f.users[username] = f.nextID
}
func (f *fakeTerdutServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
f.mu.Lock()
defer f.mu.Unlock()
@@ -137,6 +173,16 @@ func (f *fakeTerdutServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
writeJSON(w, http.StatusOK, []tdclient.Team{{ID: id, Name: f.teamNames[id]}})
case r.URL.Path == "/api/users" && r.Method == http.MethodGet:
// No query filter -- GetUserByUsername fetches the whole list and
// matches client-side (confirmed against source: no server-side
// filter either), so the fake does the same.
users := make([]tdclient.User, 0, len(f.users))
for name, id := range f.users {
users = append(users, tdclient.User{ID: id, Username: name})
}
writeJSON(w, http.StatusOK, users)
default:
if id, name, ok := parseKeysPath(r.URL.Path); ok && r.Method == http.MethodPost {
f.keyMints[id]++
@@ -146,34 +192,21 @@ func (f *fakeTerdutServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
})
return
}
if id, ok := parseTeamPath(r.URL.Path); ok {
f.handleTeamByID(w, r, id)
if id, rest, ok := parseTeamSubPath(r.URL.Path); ok {
f.handleTeamSubPath(w, r, id, rest)
return
}
w.WriteHeader(http.StatusNotFound)
}
}
// handleTeamByID answers PUT /api/teams/{id}, PUT /api/teams/{id}/oidc-groups
// and DELETE /api/teams/{id}.
func (f *fakeTerdutServer) handleTeamByID(w http.ResponseWriter, r *http.Request, id int64) {
oidcSuffix := fmt.Sprintf("/api/teams/%d/oidc-groups", id)
// handleTeamSubPath answers everything under /api/teams/{id}: PUT (rename),
// DELETE, PUT .../oidc-groups, PUT .../escalation, and the dead man's
// switch collection/item endpoints. rest is whatever parseTeamSubPath found
// after "/api/teams/{id}" -- "" for the bare resource.
func (f *fakeTerdutServer) handleTeamSubPath(w http.ResponseWriter, r *http.Request, id int64, rest string) {
switch {
case r.URL.Path == oidcSuffix && r.Method == http.MethodPut:
var req struct {
MemberGroup string `json:"member_group"`
OwnerGroup string `json:"owner_group"`
}
_ = json.NewDecoder(r.Body).Decode(&req)
if _, exists := f.teamNames[id]; !exists {
w.WriteHeader(http.StatusNotFound)
return
}
f.teamOIDC[id] = [2]string{req.MemberGroup, req.OwnerGroup}
w.WriteHeader(http.StatusNoContent)
case r.URL.Path == fmt.Sprintf("/api/teams/%d", id) && r.Method == http.MethodPut:
case rest == "" && r.Method == http.MethodPut:
var req struct {
Name string `json:"name"`
}
@@ -188,7 +221,7 @@ func (f *fakeTerdutServer) handleTeamByID(w http.ResponseWriter, r *http.Request
f.teams[req.Name] = id
w.WriteHeader(http.StatusNoContent)
case r.URL.Path == fmt.Sprintf("/api/teams/%d", id) && r.Method == http.MethodDelete:
case rest == "" && r.Method == http.MethodDelete:
name, exists := f.teamNames[id]
if !exists {
w.WriteHeader(http.StatusNotFound)
@@ -199,23 +232,132 @@ func (f *fakeTerdutServer) handleTeamByID(w http.ResponseWriter, r *http.Request
f.teamDelete[id] = true
w.WriteHeader(http.StatusNoContent)
case rest == "/oidc-groups" && r.Method == http.MethodPut:
var req struct {
MemberGroup string `json:"member_group"`
OwnerGroup string `json:"owner_group"`
}
_ = json.NewDecoder(r.Body).Decode(&req)
if _, exists := f.teamNames[id]; !exists {
w.WriteHeader(http.StatusNotFound)
return
}
f.teamOIDC[id] = [2]string{req.MemberGroup, req.OwnerGroup}
w.WriteHeader(http.StatusNoContent)
case rest == "/escalation" && r.Method == http.MethodPut:
var req tdclient.SetEscalationRequest
_ = json.NewDecoder(r.Body).Decode(&req)
f.escalation[id] = req
w.WriteHeader(http.StatusNoContent)
case rest == "/deadman/switches" && r.Method == http.MethodGet:
existing := f.switches[id]
out := make([]tdclient.DeadmanSwitch, 0, len(existing))
for _, s := range existing {
out = append(out, s)
}
writeJSON(w, http.StatusOK, out)
case rest == "/deadman/switches" && r.Method == http.MethodPost:
var req deadmanSwitchFakeRequest
_ = json.NewDecoder(r.Body).Decode(&req)
f.nextSwitchID++
switchID := f.nextSwitchID
name := req.Name
if name == "" {
name = "derived-" + req.Matcher
}
sw := tdclient.DeadmanSwitch{
ID: switchID, Name: name, Matcher: req.Matcher,
TimeoutSeconds: req.TimeoutSeconds, Severity: req.Severity,
}
if f.switches[id] == nil {
f.switches[id] = map[int64]tdclient.DeadmanSwitch{}
}
f.switches[id][switchID] = sw
writeJSON(w, http.StatusCreated, sw)
case strings.HasPrefix(rest, "/deadman/switches/") && r.Method == http.MethodPut:
switchID, ok := parseTrailingID(rest, "/deadman/switches/")
if !ok {
w.WriteHeader(http.StatusNotFound)
return
}
if _, exists := f.switches[id][switchID]; !exists {
w.WriteHeader(http.StatusNotFound)
return
}
var req deadmanSwitchFakeRequest
_ = json.NewDecoder(r.Body).Decode(&req)
name := req.Name
if name == "" {
name = f.switches[id][switchID].Name
}
f.switches[id][switchID] = tdclient.DeadmanSwitch{
ID: switchID, Name: name, Matcher: req.Matcher,
TimeoutSeconds: req.TimeoutSeconds, Severity: req.Severity,
}
w.WriteHeader(http.StatusNoContent)
case strings.HasPrefix(rest, "/deadman/switches/") && r.Method == http.MethodDelete:
switchID, ok := parseTrailingID(rest, "/deadman/switches/")
if !ok {
w.WriteHeader(http.StatusNotFound)
return
}
if _, exists := f.switches[id][switchID]; !exists {
w.WriteHeader(http.StatusNotFound)
return
}
delete(f.switches[id], switchID)
f.switchDelete[switchID] = true
w.WriteHeader(http.StatusNoContent)
default:
w.WriteHeader(http.StatusNotFound)
}
}
// parseTeamPath extracts the numeric id from "/api/teams/{id}" or
// "/api/teams/{id}/oidc-groups" -- anything with more or fewer segments
// doesn't match (handleTeamByID's own switch sorts out which of the two).
func parseTeamPath(path string) (id int64, ok bool) {
var parsedID int64
if n, err := fmt.Sscanf(path, "/api/teams/%d/oidc-groups", &parsedID); err == nil && n == 1 {
return parsedID, true
// deadmanSwitchFakeRequest mirrors tdclient's own (unexported)
// deadmanSwitchRequest -- the fake needs its own copy to decode the same
// wire shape without reaching across package boundaries for an internal type.
type deadmanSwitchFakeRequest struct {
Name string `json:"name,omitempty"`
Matcher string `json:"matcher"`
TimeoutSeconds int64 `json:"timeout_seconds"`
Severity string `json:"severity"`
}
// parseTeamSubPath splits "/api/teams/{id}" from anything after it --
// "" for an exact match, "/oidc-groups", "/escalation", "/deadman/switches"
// or "/deadman/switches/{switchID}" otherwise. Doesn't itself validate the
// suffix; handleTeamSubPath's own switch does that.
func parseTeamSubPath(path string) (id int64, rest string, ok bool) {
const prefix = "/api/teams/"
if !strings.HasPrefix(path, prefix) {
return 0, "", false
}
if n, err := fmt.Sscanf(path, "/api/teams/%d", &parsedID); err == nil && n == 1 {
return parsedID, true
trimmed := path[len(prefix):]
parts := strings.SplitN(trimmed, "/", 2)
parsedID, err := strconv.ParseInt(parts[0], 10, 64)
if err != nil {
return 0, "", false
}
return 0, false
if len(parts) == 1 {
return parsedID, "", true
}
return parsedID, "/" + parts[1], true
}
// parseTrailingID parses the numeric id after prefix within rest, e.g.
// parseTrailingID("/deadman/switches/7", "/deadman/switches/") -> 7, true.
func parseTrailingID(rest, prefix string) (id int64, ok bool) {
parsedID, err := strconv.ParseInt(strings.TrimPrefix(rest, prefix), 10, 64)
if err != nil {
return 0, false
}
return parsedID, true
}
func parseKeysPath(path string) (id int64, mintName string, ok bool) {
+6 -4
View File
@@ -95,7 +95,9 @@ func (r *TerdutTeamReconciler) Reconcile(ctx context.Context, req ctrl.Request)
if err != nil {
return ctrl.Result{}, err
}
instanceClient := newClient(serviceURL(srv)).WithToken(instanceKey)
endpoint := serviceURL(srv)
team.Status.ServerEndpoint = endpoint
instanceClient := newClient(endpoint).WithToken(instanceKey)
if team.Status.TeamID == 0 {
if err := r.createOrAdoptTeam(ctx, &team, instanceClient); err != nil {
@@ -113,7 +115,7 @@ func (r *TerdutTeamReconciler) Reconcile(ctx context.Context, req ctrl.Request)
if err != nil {
return ctrl.Result{}, err
}
teamClient := newClient(serviceURL(srv)).WithToken(teamKey)
teamClient := newClient(endpoint).WithToken(teamKey)
// Owner-gated on terdut-server, so this always runs with the
// team-scoped credential just minted above, never the instance-scoped
@@ -131,7 +133,7 @@ func (r *TerdutTeamReconciler) Reconcile(ctx context.Context, req ctrl.Request)
}
meta.SetStatusCondition(&team.Status.Conditions, metav1.Condition{
Type: terdutv1alpha1.ConditionTeamReady,
Type: terdutv1alpha1.ConditionReady,
Status: metav1.ConditionTrue,
Reason: terdutv1alpha1.ReasonTeamAdopted,
Message: fmt.Sprintf("team %d ready, credentials in Secret %q", team.Status.TeamID, team.Status.CredentialsSecretRef.Name),
@@ -208,7 +210,7 @@ func (r *TerdutTeamReconciler) setTeamNotReady(
ctx context.Context, team *terdutv1alpha1.TerdutTeam, reason, message string, d time.Duration,
) (ctrl.Result, error) {
meta.SetStatusCondition(&team.Status.Conditions, metav1.Condition{
Type: terdutv1alpha1.ConditionTeamReady,
Type: terdutv1alpha1.ConditionReady,
Status: metav1.ConditionFalse,
Reason: reason,
Message: message,
@@ -86,7 +86,7 @@ var _ = Describe("TerdutTeam Controller", func() {
readyCondition := func(ctx context.Context) metav1.Condition {
team := &terdutv1alpha1.TerdutTeam{}
Expect(k8sClient.Get(ctx, teamKey, team)).To(Succeed())
c := meta.FindStatusCondition(team.Status.Conditions, terdutv1alpha1.ConditionTeamReady)
c := meta.FindStatusCondition(team.Status.Conditions, terdutv1alpha1.ConditionReady)
Expect(c).NotTo(BeNil())
return *c
}
@@ -120,7 +120,7 @@ var _ = Describe("TerdutTeam Controller", func() {
Describe("waiting on the referenced TerdutServer", func() {
It("reports ServerRefNotFound when the TerdutServer doesn't exist", func(ctx SpecContext) {
createTeam(ctx, "orphan", terdutv1alpha1.TerdutServerRef{Name: "does-not-exist"})
createTeam(ctx, "orphan", terdutv1alpha1.TerdutServerRef{Name: testRefNotFoundName})
reconcileOnce(ctx) // finalizer
reconcileOnce(ctx)
@@ -175,7 +175,7 @@ var _ = Describe("TerdutTeam Controller", func() {
Expect(err).NotTo(HaveOccurred())
Expect(k8sClient.Get(ctx, crossKey, team)).To(Succeed())
cond := meta.FindStatusCondition(team.Status.Conditions, terdutv1alpha1.ConditionTeamReady)
cond := meta.FindStatusCondition(team.Status.Conditions, terdutv1alpha1.ConditionReady)
Expect(cond.Reason).To(Equal(terdutv1alpha1.ReasonRefNotPermitted))
})
@@ -201,7 +201,7 @@ var _ = Describe("TerdutTeam Controller", func() {
Expect(err).NotTo(HaveOccurred())
Expect(k8sClient.Get(ctx, crossKey, team)).To(Succeed())
cond := meta.FindStatusCondition(team.Status.Conditions, terdutv1alpha1.ConditionTeamReady)
cond := meta.FindStatusCondition(team.Status.Conditions, terdutv1alpha1.ConditionReady)
// Past the gate: Adopted (the fake server has no reason to
// reject this), definitely not RefNotPermitted.
Expect(cond.Reason).To(Equal(terdutv1alpha1.ReasonTeamAdopted))
+42
View File
@@ -23,6 +23,13 @@ const (
testImageRepo = "example.invalid/terdut-server"
testImageTag = "test"
testDSN = "postgres://terdut@test-postgres:5432/terdut?sslmode=require"
// testRefNotFoundName is a name no TerdutServer/TerdutTeam ever gets
// created with -- shared by every "...RefNotFound" test across
// terdutteam_controller_test.go, terdutescalationrule_controller_test.go
// and terdutdeadmanswitch_controller_test.go, so goconst doesn't flag
// three independent copies of the same literal.
testRefNotFoundName = "does-not-exist"
)
// bootstrapReadyTerdutServer creates a TerdutServer with a bring-your-own
@@ -70,6 +77,41 @@ func bootstrapReadyTerdutServer(ctx context.Context, namespace, name, fakeURL st
return srv
}
// readyTerdutTeam creates a TerdutTeam under srv (an already-Ready
// TerdutServer, e.g. from bootstrapReadyTerdutServer) and drives it to
// Ready against fakeURL -- shared by TerdutEscalationRule's and
// TerdutDeadmanSwitch's own tests, which both just need a resolvable
// teamRef (DESIGN.md §5), not TerdutTeam's own behavior.
func readyTerdutTeam(ctx context.Context, namespace, name string, srv *terdutv1alpha1.TerdutServer, fakeURL string) *terdutv1alpha1.TerdutTeam {
GinkgoHelper()
reconciler := &TerdutTeamReconciler{
Client: k8sClient,
Scheme: k8sClient.Scheme(),
OperatorNamespace: namespace,
NewClient: func(string) *tdclient.Client { return tdclient.New(fakeURL) },
}
objKey := types.NamespacedName{Name: name, Namespace: namespace}
team := &terdutv1alpha1.TerdutTeam{
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace},
Spec: terdutv1alpha1.TerdutTeamSpec{
ServerRef: terdutv1alpha1.TerdutServerRef{Name: srv.Name},
DisplayName: name,
},
}
Expect(k8sClient.Create(ctx, team)).To(Succeed())
_, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: objKey}) // finalizer
Expect(err).NotTo(HaveOccurred())
_, err = reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: objKey}) // create+mint+apply
Expect(err).NotTo(HaveOccurred())
Expect(k8sClient.Get(ctx, objKey, team)).To(Succeed())
Expect(team.Status.CredentialsSecretRef).NotTo(BeNil(), "test setup: TerdutTeam %s/%s did not reach Ready", namespace, name)
return team
}
// uniqueNameCounter backs uniqueName. GinkgoRandomSeed()/GinkgoParallelProcess()
// are constants for the whole suite run, not per-spec -- an earlier version
// of this helper used them and collided across every spec that called it
+139
View File
@@ -357,3 +357,142 @@ func (c *Client) SetTeamOIDCGroups(ctx context.Context, teamID int64, memberGrou
}
return c.do(req, nil)
}
// User mirrors terdut-server's models.User, minus fields this client never
// reads.
type User struct {
ID int64 `json:"id"`
Username string `json:"username"`
}
// GetUserByUsername calls GET /api/users and finds the one matching exactly
// -- confirmed open to any authenticated caller, not gated by team
// membership or admin (internal/api/router.go's own comment: "readable by
// anyone signed in"), so the team-scoped credential a TerdutEscalationRule's
// controller already holds is enough. There is no server-side filter, so
// this always fetches the whole list; terdut-server's own query has no
// pagination either (confirmed against source), so this matches what the
// server itself considers an acceptable cost. Returns nil, nil on no match.
func (c *Client) GetUserByUsername(ctx context.Context, username string) (*User, error) {
req, err := c.newRequest(ctx, http.MethodGet, "/api/users", nil)
if err != nil {
return nil, err
}
var users []User
if err := c.do(req, &users); err != nil {
return nil, err
}
for _, u := range users {
if u.Username == username {
return &u, nil
}
}
return nil, nil
}
// EscalationTargetRequest/EscalationLevelRequest/SetEscalationRequest mirror
// terdut-server's escalationTargetJSON/escalationLevelJSON/escalationJSON
// (internal/api/escalation.go) -- the PUT body, not the richer GET response
// (escalationView), which this client never needs to decode since the
// controller always computes its own desired state fresh from spec.
type EscalationTargetRequest struct {
Kind string `json:"kind"`
UserID *int64 `json:"user_id,omitempty"`
}
type EscalationLevelRequest struct {
Position int64 `json:"position"`
TimeoutSeconds int64 `json:"timeout_seconds"`
Targets []EscalationTargetRequest `json:"targets"`
}
type SetEscalationRequest struct {
RepeatCount int64 `json:"repeat_count"`
FallbackTopic string `json:"fallback_topic"`
Levels []EscalationLevelRequest `json:"levels"`
}
// SetEscalation calls PUT /api/teams/{teamID}/escalation -- an upsert
// server-side (confirmed against source: `INSERT ... ON CONFLICT (team_id)
// DO UPDATE`), so there is no separate create step for this resource at
// all, unlike Team or the dead man's switch.
func (c *Client) SetEscalation(ctx context.Context, teamID int64, body SetEscalationRequest) error {
req, err := c.newRequest(ctx, http.MethodPut, fmt.Sprintf("/api/teams/%d/escalation", teamID), body)
if err != nil {
return err
}
return c.do(req, nil)
}
// DeadmanSwitch mirrors terdut-server's deadmanSwitchStatus
// (internal/api/deadman.go), minus fields this client never reads.
type DeadmanSwitch struct {
ID int64 `json:"id"`
Name string `json:"name"`
Matcher string `json:"matcher"`
TimeoutSeconds int64 `json:"timeout_seconds"`
Severity string `json:"severity"`
}
// ListDeadmanSwitches calls GET /api/teams/{teamID}/deadman/switches. There
// is no unique-name constraint on this resource server-side (confirmed
// against source), so this is the idempotent-create lookup for it --
// GET-list-and-match-by-name, not adopt-on-409.
func (c *Client) ListDeadmanSwitches(ctx context.Context, teamID int64) ([]DeadmanSwitch, error) {
req, err := c.newRequest(ctx, http.MethodGet, fmt.Sprintf("/api/teams/%d/deadman/switches", teamID), nil)
if err != nil {
return nil, err
}
var switches []DeadmanSwitch
if err := c.do(req, &switches); err != nil {
return nil, err
}
return switches, nil
}
// deadmanSwitchRequest mirrors terdut-server's own deadmanSwitchRequest
// (internal/api/teams.go) -- the same body shape for both create and
// update.
type deadmanSwitchRequest struct {
Name string `json:"name,omitempty"`
Matcher string `json:"matcher"`
TimeoutSeconds int64 `json:"timeout_seconds"`
Severity string `json:"severity"`
}
// CreateDeadmanSwitch calls POST /api/teams/{teamID}/deadman/switches.
func (c *Client) CreateDeadmanSwitch(ctx context.Context, teamID int64, name, matcher string, timeoutSeconds int64, severity string) (*DeadmanSwitch, error) {
req, err := c.newRequest(ctx, http.MethodPost, fmt.Sprintf("/api/teams/%d/deadman/switches", teamID),
deadmanSwitchRequest{Name: name, Matcher: matcher, TimeoutSeconds: timeoutSeconds, Severity: severity})
if err != nil {
return nil, err
}
var sw DeadmanSwitch
if err := c.do(req, &sw); err != nil {
return nil, err
}
return &sw, nil
}
// UpdateDeadmanSwitch calls PUT /api/teams/{teamID}/deadman/switches/{switchID}
// -- real update-in-place, added in terdut-server v0.33.0 specifically for
// this controller (that handler's own doc comment names terdut-operator).
func (c *Client) UpdateDeadmanSwitch(ctx context.Context, teamID, switchID int64, name, matcher string, timeoutSeconds int64, severity string) error {
req, err := c.newRequest(ctx, http.MethodPut,
fmt.Sprintf("/api/teams/%d/deadman/switches/%d", teamID, switchID),
deadmanSwitchRequest{Name: name, Matcher: matcher, TimeoutSeconds: timeoutSeconds, Severity: severity})
if err != nil {
return err
}
return c.do(req, nil)
}
// DeleteDeadmanSwitch calls DELETE /api/teams/{teamID}/deadman/switches/{switchID}.
func (c *Client) DeleteDeadmanSwitch(ctx context.Context, teamID, switchID int64) error {
req, err := c.newRequest(ctx, http.MethodDelete,
fmt.Sprintf("/api/teams/%d/deadman/switches/%d", teamID, switchID), nil)
if err != nil {
return err
}
return c.do(req, nil)
}