From 048f4448c45b22ab491cba3131228d42a5e75a50 Mon Sep 17 00:00:00 2001 From: Niklas Ye Date: Thu, 1 Oct 2026 14:13:19 +0200 Subject: [PATCH] Stage 4: TerdutAlertSource MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Covers webhook Secret generation/ownership (DESIGN.md §4.5, §7), the WebhookSecretLost fail-closed condition, and the kind-change delete-and-recreate rotation path. Idempotent-create here is deliberately neither adopt-on-409 (Team/service-account) nor list-and-match-by-name (TerdutDeadmanSwitch): terdut-server shows the webhook key exactly once, at creation, and never again, so no server-side lookup could ever recover it after a crash. Instead the generated webhook Secret itself -- written immediately after the POST, before status is ever touched -- is this CR's only durable record that a create already succeeded; found with status.integrationID still unset on a later reconcile, it's read back directly rather than POSTing a second, orphaned integration. Found missing with status.integrationID *set* instead, that's the already-designed WebhookSecretLost case: fail closed, not self-healed, since the key is genuinely gone and recreating it would rotate a live webhook URL with no spec change to explain why. Renaming (PATCH) never touches the key, so it's applied unconditionally every reconcile, same as the escalation policy's whole-policy PUT. A spec.kind change is the one case with no in-place update verb at all: DELETE the old integration, delete the stale webhook Secret, then run the same create path fresh -- fires a Warning event since this breaks whatever still sends to the old URL. Also: fakeTerdutServer grows POST/PATCH/DELETE .../integrations routes behind a new handleIntegrationSubPath, split out of handleTeamSubPath to stay under gocyclo's threshold; three goconst-flagged test literals ("does-not-exist", "unready") and one unparam-flagged test helper parameter (bootstrapReadyTerdutServer's always-"default" namespace) get shared/removed now that a fourth same-shaped caller made the repetition concrete enough for the linter to flag. DESIGN.md §13 gains one honest gap found while grounding this stage, not introduced by it: no child CRD specially detects a mid-life teamRef change; all three always resolve spec.teamRef fresh and trust the already-stored server-side id remains valid there. make fmt lint test build all clean; internal/controller envtest coverage holds at 71.6%. --- DESIGN.md | 10 + PROJECT | 9 + ROADMAP.md | 17 + api/v1alpha1/terdutalertsource_types.go | 139 ++++++++ api/v1alpha1/zz_generated.deepcopy.go | 117 +++++++ cmd/main.go | 8 + .../terdut.ryuvia.com_terdutalertsources.yaml | 194 +++++++++++ config/crd/kustomization.yaml | 1 + config/rbac/kustomization.yaml | 3 + config/rbac/role.yaml | 3 + config/rbac/terdutalertsource_admin_role.yaml | 27 ++ .../rbac/terdutalertsource_editor_role.yaml | 33 ++ .../rbac/terdutalertsource_viewer_role.yaml | 29 ++ config/samples/kustomization.yaml | 1 + .../terdut_v1alpha1_terdutalertsource.yaml | 19 ++ .../terdutalertsource_controller.go | 313 ++++++++++++++++++ .../terdutalertsource_controller_test.go | 297 +++++++++++++++++ .../terdutdeadmanswitch_controller_test.go | 4 +- .../terdutescalationrule_controller_test.go | 4 +- .../terdutserver_controller_test.go | 80 +++++ .../controller/terdutteam_controller_test.go | 2 +- internal/controller/testhelpers_test.go | 24 +- internal/tdclient/client.go | 70 ++++ 23 files changed, 1395 insertions(+), 9 deletions(-) create mode 100644 api/v1alpha1/terdutalertsource_types.go create mode 100644 config/crd/bases/terdut.ryuvia.com_terdutalertsources.yaml create mode 100644 config/rbac/terdutalertsource_admin_role.yaml create mode 100644 config/rbac/terdutalertsource_editor_role.yaml create mode 100644 config/rbac/terdutalertsource_viewer_role.yaml create mode 100644 config/samples/terdut_v1alpha1_terdutalertsource.yaml create mode 100644 internal/controller/terdutalertsource_controller.go create mode 100644 internal/controller/terdutalertsource_controller_test.go diff --git a/DESIGN.md b/DESIGN.md index 1cdb846..7102e3e 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -736,6 +736,16 @@ what it was, a separate install, until someone deletes it. has no owner-scoped grant below the namespace itself, per §9) — revisit if the widened per-tenant-namespace `Secret` access proves too broad in practice. +- A mid-life `spec.teamRef` change on a child CRD (`TerdutEscalationRule`, + `TerdutDeadmanSwitch`, `TerdutAlertSource`) isn't specially detected — + noticed while grounding Stage 4 against source, not newly introduced by + it: all three controllers always resolve `spec.teamRef` fresh every + reconcile and act against whatever `TerdutTeam` that currently names, + trusting the server-side id already stored in `status` remains valid + there. There's no server-side verb that could move an existing + integration/policy/switch to a different team in place regardless, so + retargeting one onto a live child isn't a supported operation in v1 — + delete and recreate the CR instead. - Gitops-managed team *membership* (see §4.2). - Automatic Deployment restart on upstream Postgres credential rotation. - Admission webhooks / CEL-only validation limits (e.g. verifying a diff --git a/PROJECT b/PROJECT index 79aafdc..0e4bb85 100644 --- a/PROJECT +++ b/PROJECT @@ -45,4 +45,13 @@ resources: kind: TerdutDeadmanSwitch path: git.ryuvia.com/niklas/terdut-operator/api/v1alpha1 version: v1alpha1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: ryuvia.com + group: terdut + kind: TerdutAlertSource + path: git.ryuvia.com/niklas/terdut-operator/api/v1alpha1 + version: v1alpha1 version: "3" diff --git a/ROADMAP.md b/ROADMAP.md index ef45739..620a5de 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -153,6 +153,23 @@ New commits build forward over the old ones; no git history rewrite. 2026-09-30), and the kind-change delete-and-recreate rotation path — all easier to get right with the other three controllers' patterns already in place to build on. +- Idempotent-create here is neither adopt-on-409 (Team/service-account) nor + list-and-match-by-name (`TerdutDeadmanSwitch`): terdut-server shows the + webhook key exactly once, at creation, and never again, so no server-side + lookup could ever recover it after a crash. The generated webhook Secret + itself — written immediately after the POST, before `status` is ever + touched — is this CR's only durable record that a create already + succeeded; found on a later reconcile with `status.integrationID` still + unset, it's read back directly rather than POSTing again. Found missing + with `status.integrationID` *set*, that's the already-designed + `WebhookSecretLost` fail-closed case instead. +- **Done, 2026-10-01**: `envtest` coverage for the happy path, rename + (PATCH, no key rotation), a kind change (delete-and-recreate, new id and + key), crash recovery between POST and the Secret write, `WebhookSecretLost`, + `TeamRefNotFound`/`WaitingForTeam`, and deletion. `make fmt lint test + build` all clean; `internal/controller` envtest coverage holds at 71.6%. + No `kind` e2e pass for this stage, same reasoning as Stage 3 (reuses + Stage 1's already-proven real-cluster mechanics unchanged). ## Stage 5 — Installer chart + real release diff --git a/api/v1alpha1/terdutalertsource_types.go b/api/v1alpha1/terdutalertsource_types.go new file mode 100644 index 0000000..c4d813b --- /dev/null +++ b/api/v1alpha1/terdutalertsource_types.go @@ -0,0 +1,139 @@ +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// LocalSecretRef names a Secret in this CR's own namespace holding more than +// one data key -- unlike SecretKeyRef (terdutserver_types.go), there's no +// single key to name here: a consumer needs both "url" and "key" +// (DESIGN.md §4.5). +type LocalSecretRef struct { + // name is the Secret's name. + // +kubebuilder:validation:MinLength=1 + Name string `json:"name"` +} + +// TerdutAlertSourceSpec defines the desired state of TerdutAlertSource. +// +// Stays same-namespace as its TerdutTeam (§13: cross-namespace teamRef on +// the child CRDs is deferred, same as TerdutEscalationRule/TerdutDeadmanSwitch). +type TerdutAlertSourceSpec struct { + // +required + TeamRef TerdutTeamRef `json:"teamRef"` + + // kind is the alert source type. Only "alertmanager" is supported + // today, mirroring terdut-server's own CHECK constraint on + // integrations.kind (internal/db/migrations/003_teams.sql) -- + // confirmed against source, not assumed. Changing it after the + // integration already exists rotates the webhook key (DESIGN.md §5's + // reconciliation table): the old one is deleted and a fresh one + // created, which breaks whatever sends to the old URL until the new + // Secret is picked up. + // +kubebuilder:validation:Enum=alertmanager + // +kubebuilder:default=alertmanager + // +optional + Kind string `json:"kind,omitempty"` + + // name is this source's own display name server-side -- distinct from + // this object's own metadata.name. POST + // /api/teams/{teamID}/integrations {"name": ...} at creation, and what + // PATCH renames thereafter; renaming never rotates the webhook key. + // +required + // +kubebuilder:validation:MinLength=1 + Name string `json:"name"` +} + +// Condition/event reasons specific to TerdutAlertSource. TeamRefNotFound, +// WaitingForTeam and ChildAdopted (terdutescalationrule_types.go) are shared +// with the other two child kinds; these two are not, since no other +// resource in this operator holds unrecoverable, show-once server-issued +// material. +const ( + // ReasonWebhookSecretLost: status.integrationID is set but the webhook + // Secret is gone -- fail-closed, not self-healed (DESIGN.md §5): the + // key is genuinely unrecoverable, so silently minting a replacement + // would rotate a live webhook URL with no corresponding spec change to + // explain why. + ReasonWebhookSecretLost = "WebhookSecretLost" + // ReasonKindRotated: a Warning event reason only (never a condition) -- + // fired once, the moment a spec.kind change deletes the old + // integration and creates a new one, so the rotation is loud in + // `kubectl describe`/`get events` even though the condition right + // after is the same ReasonChildAdopted the initial create used. + ReasonKindRotated = "KindRotated" +) + +// TerdutAlertSourceStatus defines the observed state of TerdutAlertSource. +type TerdutAlertSourceStatus struct { + // +listType=map + // +listMapKey=type + // +optional + Conditions []metav1.Condition `json:"conditions,omitempty"` + + // integrationID is the server-side id. + // +optional + IntegrationID int64 `json:"integrationID,omitempty"` + + // lastAppliedKind is the kind the currently-live integration was + // actually created with -- compared against spec.kind on every + // reconcile to detect the one spec change that requires + // delete-and-recreate (DESIGN.md §5), since terdut-server's own API has + // no way to read a live integration's kind back for comparison. + // +optional + LastAppliedKind string `json:"lastAppliedKind,omitempty"` + + // webhookURLSecretRef names the generated Secret holding "url" and + // "key" -- the integration's webhook address and credential, shown by + // terdut-server's API exactly once, at creation (DESIGN.md §4.5), and + // never re-readable afterward, including from this status. Lives in + // this CR's own namespace with a plain OwnerReference (§7) -- unlike + // TerdutServer/TerdutTeam's credential Secrets, this one never crosses + // namespaces, so no finalizer cleanup is needed for it specifically. + // +optional + WebhookURLSecretRef *LocalSecretRef `json:"webhookURLSecretRef,omitempty"` + + // +optional + ObservedGeneration int64 `json:"observedGeneration,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Team",type=string,JSONPath=`.spec.teamRef.name` +// +kubebuilder:printcolumn:name="IntegrationID",type=integer,JSONPath=`.status.integrationID` +// +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].status` +// +kubebuilder:printcolumn:name="Reason",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].reason` + +// TerdutAlertSource is the Schema for the terdutalertsources API +type TerdutAlertSource struct { + metav1.TypeMeta `json:",inline"` + + // metadata is a standard object metadata + // +optional + metav1.ObjectMeta `json:"metadata,omitzero"` + + // spec defines the desired state of TerdutAlertSource + // +required + Spec TerdutAlertSourceSpec `json:"spec"` + + // status defines the observed state of TerdutAlertSource + // +optional + Status TerdutAlertSourceStatus `json:"status,omitzero"` +} + +// +kubebuilder:object:root=true + +// TerdutAlertSourceList contains a list of TerdutAlertSource +type TerdutAlertSourceList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitzero"` + Items []TerdutAlertSource `json:"items"` +} + +func init() { + SchemeBuilder.Register(func(s *runtime.Scheme) error { + s.AddKnownTypes(SchemeGroupVersion, &TerdutAlertSource{}, &TerdutAlertSourceList{}) + return nil + }) +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 4f041ce..832e842 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -135,6 +135,21 @@ func (in *ImageSpec) DeepCopy() *ImageSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LocalSecretRef) DeepCopyInto(out *LocalSecretRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LocalSecretRef. +func (in *LocalSecretRef) DeepCopy() *LocalSecretRef { + if in == nil { + return nil + } + out := new(LocalSecretRef) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NetworkingSpec) DeepCopyInto(out *NetworkingSpec) { *out = *in @@ -240,6 +255,108 @@ func (in *SweeperSpec) DeepCopy() *SweeperSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TerdutAlertSource) DeepCopyInto(out *TerdutAlertSource) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TerdutAlertSource. +func (in *TerdutAlertSource) DeepCopy() *TerdutAlertSource { + if in == nil { + return nil + } + out := new(TerdutAlertSource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *TerdutAlertSource) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TerdutAlertSourceList) DeepCopyInto(out *TerdutAlertSourceList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]TerdutAlertSource, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TerdutAlertSourceList. +func (in *TerdutAlertSourceList) DeepCopy() *TerdutAlertSourceList { + if in == nil { + return nil + } + out := new(TerdutAlertSourceList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *TerdutAlertSourceList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TerdutAlertSourceSpec) DeepCopyInto(out *TerdutAlertSourceSpec) { + *out = *in + out.TeamRef = in.TeamRef +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TerdutAlertSourceSpec. +func (in *TerdutAlertSourceSpec) DeepCopy() *TerdutAlertSourceSpec { + if in == nil { + return nil + } + out := new(TerdutAlertSourceSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TerdutAlertSourceStatus) DeepCopyInto(out *TerdutAlertSourceStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.WebhookURLSecretRef != nil { + in, out := &in.WebhookURLSecretRef, &out.WebhookURLSecretRef + *out = new(LocalSecretRef) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TerdutAlertSourceStatus. +func (in *TerdutAlertSourceStatus) DeepCopy() *TerdutAlertSourceStatus { + if in == nil { + return nil + } + out := new(TerdutAlertSourceStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TerdutDeadmanSwitch) DeepCopyInto(out *TerdutDeadmanSwitch) { *out = *in diff --git a/cmd/main.go b/cmd/main.go index 9c4ca7f..d71f231 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -209,6 +209,14 @@ func main() { setupLog.Error(err, "Failed to create controller", "controller", "terdutdeadmanswitch") os.Exit(1) } + if err := (&controller.TerdutAlertSourceReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + OperatorNamespace: operatorNamespace, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "Failed to create controller", "controller", "terdutalertsource") + os.Exit(1) + } // +kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { diff --git a/config/crd/bases/terdut.ryuvia.com_terdutalertsources.yaml b/config/crd/bases/terdut.ryuvia.com_terdutalertsources.yaml new file mode 100644 index 0000000..e186b89 --- /dev/null +++ b/config/crd/bases/terdut.ryuvia.com_terdutalertsources.yaml @@ -0,0 +1,194 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.22.0 + name: terdutalertsources.terdut.ryuvia.com +spec: + group: terdut.ryuvia.com + names: + kind: TerdutAlertSource + listKind: TerdutAlertSourceList + plural: terdutalertsources + singular: terdutalertsource + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.teamRef.name + name: Team + type: string + - jsonPath: .status.integrationID + name: IntegrationID + type: integer + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].reason + name: Reason + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: TerdutAlertSource is the Schema for the terdutalertsources API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec defines the desired state of TerdutAlertSource + properties: + kind: + default: alertmanager + description: |- + kind is the alert source type. Only "alertmanager" is supported + today, mirroring terdut-server's own CHECK constraint on + integrations.kind (internal/db/migrations/003_teams.sql) -- + confirmed against source, not assumed. Changing it after the + integration already exists rotates the webhook key (DESIGN.md §5's + reconciliation table): the old one is deleted and a fresh one + created, which breaks whatever sends to the old URL until the new + Secret is picked up. + enum: + - alertmanager + type: string + name: + description: |- + name is this source's own display name server-side -- distinct from + this object's own metadata.name. POST + /api/teams/{teamID}/integrations {"name": ...} at creation, and what + PATCH renames thereafter; renaming never rotates the webhook key. + minLength: 1 + type: string + teamRef: + description: |- + TerdutTeamRef names the TerdutTeam this resource belongs to. Always + same-namespace as the CR itself (DESIGN.md §1: only TerdutTeam.spec.serverRef + crosses namespaces in v1) -- no namespace field, unlike TerdutServerRef. + properties: + name: + minLength: 1 + type: string + required: + - name + type: object + required: + - name + - teamRef + type: object + status: + description: status defines the observed state of TerdutAlertSource + properties: + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + integrationID: + description: integrationID is the server-side id. + format: int64 + type: integer + lastAppliedKind: + description: |- + lastAppliedKind is the kind the currently-live integration was + actually created with -- compared against spec.kind on every + reconcile to detect the one spec change that requires + delete-and-recreate (DESIGN.md §5), since terdut-server's own API has + no way to read a live integration's kind back for comparison. + type: string + observedGeneration: + format: int64 + type: integer + webhookURLSecretRef: + description: |- + webhookURLSecretRef names the generated Secret holding "url" and + "key" -- the integration's webhook address and credential, shown by + terdut-server's API exactly once, at creation (DESIGN.md §4.5), and + never re-readable afterward, including from this status. Lives in + this CR's own namespace with a plain OwnerReference (§7) -- unlike + TerdutServer/TerdutTeam's credential Secrets, this one never crosses + namespaces, so no finalizer cleanup is needed for it specifically. + properties: + name: + description: name is the Secret's name. + minLength: 1 + type: string + required: + - name + type: object + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 100eaa8..36740b0 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -6,6 +6,7 @@ resources: - bases/terdut.ryuvia.com_terdutteams.yaml - bases/terdut.ryuvia.com_terdutescalationrules.yaml - bases/terdut.ryuvia.com_terdutdeadmanswitches.yaml +- bases/terdut.ryuvia.com_terdutalertsources.yaml # +kubebuilder:scaffold:crdkustomizeresource patches: diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 0726226..8f27f0a 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -22,6 +22,9 @@ resources: # default, aiding admins in cluster management. Those roles are # not used by the terdut-operator itself. You can comment the following lines # if you do not want those helpers be installed with your Project. +- terdutalertsource_admin_role.yaml +- terdutalertsource_editor_role.yaml +- terdutalertsource_viewer_role.yaml - terdutdeadmanswitch_admin_role.yaml - terdutdeadmanswitch_editor_role.yaml - terdutdeadmanswitch_viewer_role.yaml diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index b1e0b0d..aa6f42a 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -55,6 +55,7 @@ rules: - apiGroups: - terdut.ryuvia.com resources: + - terdutalertsources - terdutdeadmanswitches - terdutescalationrules - terdutservers @@ -70,6 +71,7 @@ rules: - apiGroups: - terdut.ryuvia.com resources: + - terdutalertsources/finalizers - terdutdeadmanswitches/finalizers - terdutescalationrules/finalizers - terdutservers/finalizers @@ -79,6 +81,7 @@ rules: - apiGroups: - terdut.ryuvia.com resources: + - terdutalertsources/status - terdutdeadmanswitches/status - terdutescalationrules/status - terdutservers/status diff --git a/config/rbac/terdutalertsource_admin_role.yaml b/config/rbac/terdutalertsource_admin_role.yaml new file mode 100644 index 0000000..31dca3f --- /dev/null +++ b/config/rbac/terdutalertsource_admin_role.yaml @@ -0,0 +1,27 @@ +# This rule is not used by the project terdut-operator itself. +# It is provided to allow the cluster admin to help manage permissions for users. +# +# Grants full permissions ('*') over terdut.ryuvia.com. +# This role is intended for users authorized to modify roles and bindings within the cluster, +# enabling them to delegate specific permissions to other users or groups as needed. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: terdut-operator + app.kubernetes.io/managed-by: kustomize + name: terdutalertsource-admin-role +rules: +- apiGroups: + - terdut.ryuvia.com + resources: + - terdutalertsources + verbs: + - '*' +- apiGroups: + - terdut.ryuvia.com + resources: + - terdutalertsources/status + verbs: + - get diff --git a/config/rbac/terdutalertsource_editor_role.yaml b/config/rbac/terdutalertsource_editor_role.yaml new file mode 100644 index 0000000..ec57dd9 --- /dev/null +++ b/config/rbac/terdutalertsource_editor_role.yaml @@ -0,0 +1,33 @@ +# This rule is not used by the project terdut-operator itself. +# It is provided to allow the cluster admin to help manage permissions for users. +# +# Grants permissions to create, update, and delete resources within the terdut.ryuvia.com. +# This role is intended for users who need to manage these resources +# but should not control RBAC or manage permissions for others. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: terdut-operator + app.kubernetes.io/managed-by: kustomize + name: terdutalertsource-editor-role +rules: +- apiGroups: + - terdut.ryuvia.com + resources: + - terdutalertsources + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - terdut.ryuvia.com + resources: + - terdutalertsources/status + verbs: + - get diff --git a/config/rbac/terdutalertsource_viewer_role.yaml b/config/rbac/terdutalertsource_viewer_role.yaml new file mode 100644 index 0000000..e994184 --- /dev/null +++ b/config/rbac/terdutalertsource_viewer_role.yaml @@ -0,0 +1,29 @@ +# This rule is not used by the project terdut-operator itself. +# It is provided to allow the cluster admin to help manage permissions for users. +# +# Grants read-only access to terdut.ryuvia.com resources. +# This role is intended for users who need visibility into these resources +# without permissions to modify them. It is ideal for monitoring purposes and limited-access viewing. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: terdut-operator + app.kubernetes.io/managed-by: kustomize + name: terdutalertsource-viewer-role +rules: +- apiGroups: + - terdut.ryuvia.com + resources: + - terdutalertsources + verbs: + - get + - list + - watch +- apiGroups: + - terdut.ryuvia.com + resources: + - terdutalertsources/status + verbs: + - get diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml index 57a61ac..6d6f51d 100644 --- a/config/samples/kustomization.yaml +++ b/config/samples/kustomization.yaml @@ -4,4 +4,5 @@ resources: - terdut_v1alpha1_terdutteam.yaml - terdut_v1alpha1_terdutescalationrule.yaml - terdut_v1alpha1_terdutdeadmanswitch.yaml +- terdut_v1alpha1_terdutalertsource.yaml # +kubebuilder:scaffold:manifestskustomizesamples diff --git a/config/samples/terdut_v1alpha1_terdutalertsource.yaml b/config/samples/terdut_v1alpha1_terdutalertsource.yaml new file mode 100644 index 0000000..4f9fd6f --- /dev/null +++ b/config/samples/terdut_v1alpha1_terdutalertsource.yaml @@ -0,0 +1,19 @@ +apiVersion: terdut.ryuvia.com/v1alpha1 +kind: TerdutAlertSource +metadata: + labels: + app.kubernetes.io/name: terdut-operator + app.kubernetes.io/managed-by: kustomize + name: terdutalertsource-sample +spec: + teamRef: + name: terdutteam-sample + # kind defaults to "alertmanager" -- the only value terdut-server + # supports today. Changing it after this object exists rotates the + # webhook key (DESIGN.md §5): the old integration is deleted and a new + # one created, which breaks whatever still sends to the old URL. + kind: alertmanager + # name is this source's own display name server-side, distinct from this + # object's own metadata.name above -- renaming it is safe and never + # rotates the key. + name: prod-alertmanager diff --git a/internal/controller/terdutalertsource_controller.go b/internal/controller/terdutalertsource_controller.go new file mode 100644 index 0000000..d15b4cb --- /dev/null +++ b/internal/controller/terdutalertsource_controller.go @@ -0,0 +1,313 @@ +package controller + +import ( + "context" + "fmt" + "strconv" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/recorder" + + terdutv1alpha1 "git.ryuvia.com/niklas/terdut-operator/api/v1alpha1" + "git.ryuvia.com/niklas/terdut-operator/internal/tdclient" +) + +const alertSourceFinalizerName = "terdut.ryuvia.com/terdutalertsource" + +// Data keys inside the generated webhook Secret (DESIGN.md §4.5). url/key +// are what a sender actually needs; integrationID exists purely so this +// Secret -- once written -- is also this CR's own record that a create +// already succeeded (see reconcileCreate's own comment for why that +// matters more here than for any other child kind). +const ( + webhookSecretURLKey = "url" + webhookSecretKeyDataKey = "key" + webhookSecretIntegrationIDKey = "integrationID" +) + +// TerdutAlertSourceReconciler reconciles a TerdutAlertSource object. +type TerdutAlertSourceReconciler struct { + client.Client + Scheme *runtime.Scheme + + OperatorNamespace string + Recorder recorder.EventRecorder + NewClient func(endpoint string) *tdclient.Client +} + +// +kubebuilder:rbac:groups=terdut.ryuvia.com,resources=terdutalertsources,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=terdut.ryuvia.com,resources=terdutalertsources/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=terdut.ryuvia.com,resources=terdutalertsources/finalizers,verbs=update +// +kubebuilder:rbac:groups=terdut.ryuvia.com,resources=terdutteams,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch +// +kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=create;patch + +func (r *TerdutAlertSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := logf.FromContext(ctx) + + var as terdutv1alpha1.TerdutAlertSource + if err := r.Get(ctx, req.NamespacedName, &as); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + + newClient := r.NewClient + if newClient == nil { + newClient = tdclient.New + } + + if !as.DeletionTimestamp.IsZero() { + return r.reconcileDelete(ctx, &as, newClient) + } + + if !controllerutil.ContainsFinalizer(&as, alertSourceFinalizerName) { + controllerutil.AddFinalizer(&as, alertSourceFinalizerName) + if err := r.Update(ctx, &as); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, nil + } + + team, tc, resolveErr := resolveTeamAndClient(ctx, r.Client, r.OperatorNamespace, as.Namespace, as.Spec.TeamRef, newClient) + if resolveErr != nil { + return r.setNotReady(ctx, &as, resolveErr.reason, resolveErr.message, waitInterval) + } + teamID := team.Status.TeamID + + if as.Status.IntegrationID == 0 { + if err := r.reconcileCreate(ctx, &as, tc, teamID); err != nil { + return ctrl.Result{}, err + } + } else { + secretName := webhookSecretName(&as) + var secret corev1.Secret + if err := r.Get(ctx, client.ObjectKey{Namespace: as.Namespace, Name: secretName}, &secret); err != nil { + if !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + // Fail-closed, not self-healed (DESIGN.md §5): the key is + // genuinely gone and can never be re-read from terdut-server, + // so minting a replacement here would silently rotate a live + // webhook URL with no spec change to explain why. Deleting + // and recreating this CR is the supported recovery. + return r.setNotReady(ctx, &as, terdutv1alpha1.ReasonWebhookSecretLost, + fmt.Sprintf("webhook Secret %s/%s is gone; delete and recreate this TerdutAlertSource to rotate a new one", as.Namespace, secretName), + waitInterval) + } + + if as.Spec.Kind != as.Status.LastAppliedKind { + if err := r.rotateKind(ctx, &as, tc, teamID); err != nil { + return ctrl.Result{}, err + } + } else if err := tc.RenameIntegration(ctx, teamID, as.Status.IntegrationID, as.Spec.Name); err != nil { + return ctrl.Result{}, fmt.Errorf("PATCH /api/teams/%d/integrations/%d: %w", teamID, as.Status.IntegrationID, err) + } + } + + meta.SetStatusCondition(&as.Status.Conditions, metav1.Condition{ + Type: terdutv1alpha1.ConditionReady, + Status: metav1.ConditionTrue, + Reason: terdutv1alpha1.ReasonChildAdopted, + Message: fmt.Sprintf("integration %d applied on team %d", as.Status.IntegrationID, teamID), + }) + as.Status.ObservedGeneration = as.Generation + if err := r.Status().Update(ctx, &as); err != nil { + return ctrl.Result{}, err + } + if r.Recorder != nil { + r.Recorder.Eventf(&as, nil, corev1.EventTypeNormal, terdutv1alpha1.ReasonChildAdopted, terdutv1alpha1.ReasonChildAdopted, + "alert source applied") + } + log.Info("TerdutAlertSource applied", "name", as.Name, "integrationID", as.Status.IntegrationID) + + return ctrl.Result{RequeueAfter: resyncInterval}, nil +} + +// webhookSecretName is deterministic from this object's own, immutable +// metadata.name -- not spec.name, which can be renamed freely without the +// Secret needing to follow (DESIGN.md §4.5: renaming never rotates the key). +func webhookSecretName(as *terdutv1alpha1.TerdutAlertSource) string { + return as.Name + "-terdut-webhook" +} + +// reconcileCreate implements this resource's own idempotent-create shape -- +// deliberately not list-and-match (TerdutDeadmanSwitch) or adopt-on-409 +// (Team/service-account): terdut-server shows the webhook key exactly once, +// at creation, and never again (DESIGN.md §4.5), so there is no server-side +// lookup that could ever recover it. Instead, the webhook Secret itself -- +// written immediately after a successful POST, before status is ever +// touched -- is this CR's only durable record that a create already +// succeeded. A crash between the POST and the Secret write is the one +// unrecoverable case: on the next reconcile the Secret still won't exist, +// so this mints a brand new integration rather than risk adopting an +// orphaned, keyless row by name -- same accepted tradeoff as the "orphaned +// first key some interrupted attempt minted and never used" footnote +// DESIGN.md §6 already documents for service-account keys. +func (r *TerdutAlertSourceReconciler) reconcileCreate( + ctx context.Context, as *terdutv1alpha1.TerdutAlertSource, tc *tdclient.Client, teamID int64, +) error { + secretName := webhookSecretName(as) + + var secret corev1.Secret + err := r.Get(ctx, client.ObjectKey{Namespace: as.Namespace, Name: secretName}, &secret) + switch { + case err == nil: + // Crash recovery: a previous reconcile got as far as writing the + // Secret but died before writing status -- read the id back out + // of it rather than calling the server again. + id, parseErr := strconv.ParseInt(string(secret.Data[webhookSecretIntegrationIDKey]), 10, 64) + if parseErr != nil { + return fmt.Errorf("webhook Secret %s/%s has no valid %s: %w", as.Namespace, secretName, webhookSecretIntegrationIDKey, parseErr) + } + as.Status.IntegrationID = id + as.Status.LastAppliedKind = as.Spec.Kind + as.Status.WebhookURLSecretRef = &terdutv1alpha1.LocalSecretRef{Name: secretName} + return nil + case !apierrors.IsNotFound(err): + return err + } + + created, err := tc.CreateIntegration(ctx, teamID, as.Spec.Name, as.Spec.Kind) + if err != nil { + return fmt.Errorf("POST /api/teams/%d/integrations: %w", teamID, err) + } + if err := r.writeWebhookSecret(ctx, as, secretName, created); err != nil { + return err + } + as.Status.IntegrationID = created.ID + as.Status.LastAppliedKind = as.Spec.Kind + as.Status.WebhookURLSecretRef = &terdutv1alpha1.LocalSecretRef{Name: secretName} + return nil +} + +// rotateKind deletes the currently-live integration and creates a fresh one +// under the new kind (DESIGN.md §5's reconciliation table: kind is the one +// spec field with no in-place update verb at all), firing a Warning event +// since this rotates the webhook key and breaks whatever still sends to the +// old URL. +func (r *TerdutAlertSourceReconciler) rotateKind( + ctx context.Context, as *terdutv1alpha1.TerdutAlertSource, tc *tdclient.Client, teamID int64, +) error { + oldKind, oldID := as.Status.LastAppliedKind, as.Status.IntegrationID + if err := tc.DeleteIntegration(ctx, teamID, oldID); err != nil { + return fmt.Errorf("DELETE /api/teams/%d/integrations/%d (rotating kind %q -> %q): %w", + teamID, oldID, oldKind, as.Spec.Kind, err) + } + + // reconcileCreate's own crash-recovery path trusts this Secret's mere + // existence as "a create already succeeded" -- it must be gone before + // calling it here, or rotation would misread the about-to-be-stale + // old id right back out of it instead of minting a replacement. + secretName := webhookSecretName(as) + if err := r.Delete(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: secretName, Namespace: as.Namespace}}); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("deleting stale webhook Secret %s/%s: %w", as.Namespace, secretName, err) + } + + as.Status.IntegrationID = 0 + as.Status.WebhookURLSecretRef = nil + if err := r.reconcileCreate(ctx, as, tc, teamID); err != nil { + return err + } + if r.Recorder != nil { + r.Recorder.Eventf(as, nil, corev1.EventTypeWarning, terdutv1alpha1.ReasonKindRotated, terdutv1alpha1.ReasonKindRotated, + "spec.kind changed from %q to %q: deleted integration %d and minted a new one (%d) -- the webhook URL/key changed, update whatever was sending to the old one", + oldKind, as.Spec.Kind, oldID, as.Status.IntegrationID) + } + return nil +} + +// writeWebhookSecret creates or replaces the generated Secret holding +// integ's key material, owned by as (plain OwnerReference, same namespace, +// per DESIGN.md §7 -- no finalizer needed for this one, unlike the +// cross-namespace credential Secrets elsewhere in this operator). +func (r *TerdutAlertSourceReconciler) writeWebhookSecret( + ctx context.Context, as *terdutv1alpha1.TerdutAlertSource, name string, integ *tdclient.Integration, +) error { + secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: as.Namespace}} + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, secret, func() error { + secret.Data = map[string][]byte{ + webhookSecretURLKey: []byte(integ.URL), + webhookSecretKeyDataKey: []byte(integ.Key), + webhookSecretIntegrationIDKey: []byte(strconv.FormatInt(integ.ID, 10)), + } + return controllerutil.SetControllerReference(as, secret, r.Scheme) + }) + return err +} + +func (r *TerdutAlertSourceReconciler) setNotReady( + ctx context.Context, as *terdutv1alpha1.TerdutAlertSource, reason, message string, d time.Duration, +) (ctrl.Result, error) { + meta.SetStatusCondition(&as.Status.Conditions, metav1.Condition{ + Type: terdutv1alpha1.ConditionReady, + Status: metav1.ConditionFalse, + Reason: reason, + Message: message, + }) + as.Status.ObservedGeneration = as.Generation + if err := r.Status().Update(ctx, as); err != nil { + return ctrl.Result{}, err + } + if r.Recorder != nil { + r.Recorder.Eventf(as, nil, corev1.EventTypeWarning, reason, reason, message) + } + return ctrl.Result{RequeueAfter: d}, nil +} + +// reconcileDelete calls the real DELETE this resource has (unlike +// TerdutEscalationRule) if the team is still resolvable and an integration +// was ever created, then removes the finalizer unconditionally. The +// webhook Secret needs no explicit cleanup here -- it's same-namespace and +// OwnerReference-GC'd (DESIGN.md §7), not this finalizer's job. +func (r *TerdutAlertSourceReconciler) reconcileDelete( + ctx context.Context, as *terdutv1alpha1.TerdutAlertSource, newClient func(string) *tdclient.Client, +) (ctrl.Result, error) { + if !controllerutil.ContainsFinalizer(as, alertSourceFinalizerName) { + return ctrl.Result{}, nil + } + + if as.Status.IntegrationID != 0 { + if team, tc, resolveErr := resolveTeamAndClient( + ctx, r.Client, r.OperatorNamespace, as.Namespace, as.Spec.TeamRef, newClient, + ); resolveErr == nil { + if err := tc.DeleteIntegration(ctx, team.Status.TeamID, as.Status.IntegrationID); err != nil { + if r.Recorder != nil { + r.Recorder.Eventf(as, nil, corev1.EventTypeWarning, "DeleteFailed", "DeleteFailed", err.Error()) + } + return ctrl.Result{}, err + } + } + } + + controllerutil.RemoveFinalizer(as, alertSourceFinalizerName) + return ctrl.Result{}, r.Update(ctx, as) +} + +// SetupWithManager sets up the controller with the Manager. +func (r *TerdutAlertSourceReconciler) SetupWithManager(mgr ctrl.Manager) error { + if r.NewClient == nil { + r.NewClient = tdclient.New + } + if r.Recorder == nil { + r.Recorder = mgr.GetEventRecorder("terdutalertsource-controller") + } + return ctrl.NewControllerManagedBy(mgr). + For(&terdutv1alpha1.TerdutAlertSource{}). + // Watched, not just the CR (DESIGN.md §5): the webhook Secret's + // loss has to be noticed on its own, independent of any spec + // change to this CR, for ReasonWebhookSecretLost to ever fire. + Owns(&corev1.Secret{}). + Named("terdutalertsource"). + Complete(r) +} diff --git a/internal/controller/terdutalertsource_controller_test.go b/internal/controller/terdutalertsource_controller_test.go new file mode 100644 index 0000000..94631ab --- /dev/null +++ b/internal/controller/terdutalertsource_controller_test.go @@ -0,0 +1,297 @@ +package controller + +import ( + "context" + "net/http/httptest" + "strconv" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + terdutv1alpha1 "git.ryuvia.com/niklas/terdut-operator/api/v1alpha1" + "git.ryuvia.com/niklas/terdut-operator/internal/tdclient" +) + +var _ = Describe("TerdutAlertSource Controller", func() { + const operatorNamespace = "default" + + var ( + reconciler *TerdutAlertSourceReconciler + fake *fakeTerdutServer + fakeSrv *httptest.Server + srv *terdutv1alpha1.TerdutServer + team *terdutv1alpha1.TerdutTeam + srcName string + srcKey types.NamespacedName + ) + + BeforeEach(func(ctx SpecContext) { + fake, fakeSrv = newFakeTerdutServer() + DeferCleanup(fakeSrv.Close) + + srv = bootstrapReadyTerdutServer(ctx, uniqueName("asserver"), fakeSrv.URL) + team = readyTerdutTeam(ctx, operatorNamespace, uniqueName("asteam"), srv, fakeSrv.URL) + + reconciler = &TerdutAlertSourceReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + OperatorNamespace: operatorNamespace, + NewClient: func(string) *tdclient.Client { return tdclient.New(fakeSrv.URL) }, + } + srcName = uniqueName("alertsource") + srcKey = types.NamespacedName{Name: srcName, Namespace: operatorNamespace} + }) + + AfterEach(func(ctx SpecContext) { + as := &terdutv1alpha1.TerdutAlertSource{} + if err := k8sClient.Get(ctx, srcKey, as); err == nil { + as.Finalizers = nil + _ = k8sClient.Update(ctx, as) + _ = k8sClient.Delete(ctx, as) + } + _ = k8sClient.Delete(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{ + Name: srcName + "-terdut-webhook", Namespace: operatorNamespace, + }}) + + teamKey := types.NamespacedName{Name: team.Name, Namespace: operatorNamespace} + if err := k8sClient.Get(ctx, teamKey, team); err == nil { + team.Finalizers = nil + _ = k8sClient.Update(ctx, team) + _ = k8sClient.Delete(ctx, team) + } + + srvKey := types.NamespacedName{Name: srv.Name, Namespace: operatorNamespace} + if err := k8sClient.Get(ctx, srvKey, srv); err == nil { + srv.Finalizers = nil + _ = k8sClient.Update(ctx, srv) + _ = k8sClient.Delete(ctx, srv) + } + }) + + createSource := func(ctx context.Context, teamRef terdutv1alpha1.TerdutTeamRef, kind, name string) { + as := &terdutv1alpha1.TerdutAlertSource{ + ObjectMeta: metav1.ObjectMeta{Name: srcName, Namespace: operatorNamespace}, + Spec: terdutv1alpha1.TerdutAlertSourceSpec{TeamRef: teamRef, Kind: kind, Name: name}, + } + Expect(k8sClient.Create(ctx, as)).To(Succeed()) + } + + reconcileOnce := func(ctx context.Context) { + _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: srcKey}) + Expect(err).NotTo(HaveOccurred()) + } + + readyCondition := func(ctx context.Context) metav1.Condition { + as := &terdutv1alpha1.TerdutAlertSource{} + Expect(k8sClient.Get(ctx, srcKey, as)).To(Succeed()) + c := meta.FindStatusCondition(as.Status.Conditions, terdutv1alpha1.ConditionReady) + Expect(c).NotTo(BeNil()) + return *c + } + + sameTeamRef := func() terdutv1alpha1.TerdutTeamRef { + return terdutv1alpha1.TerdutTeamRef{Name: team.Name} + } + + webhookSecret := func(ctx context.Context) corev1.Secret { + var secret corev1.Secret + Expect(k8sClient.Get(ctx, types.NamespacedName{ + Name: srcName + "-terdut-webhook", Namespace: operatorNamespace, + }, &secret)).To(Succeed()) + return secret + } + + Describe("the happy path", func() { + It("creates the integration and writes the webhook Secret", func(ctx SpecContext) { + createSource(ctx, sameTeamRef(), "alertmanager", "prod-alertmanager") + reconcileOnce(ctx) // finalizer + reconcileOnce(ctx) // create + + cond := readyCondition(ctx) + Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + Expect(cond.Reason).To(Equal(terdutv1alpha1.ReasonChildAdopted)) + + as := &terdutv1alpha1.TerdutAlertSource{} + Expect(k8sClient.Get(ctx, srcKey, as)).To(Succeed()) + Expect(as.Status.IntegrationID).NotTo(BeZero()) + Expect(as.Status.LastAppliedKind).To(Equal("alertmanager")) + Expect(as.Status.WebhookURLSecretRef).NotTo(BeNil()) + Expect(as.Status.WebhookURLSecretRef.Name).To(Equal(srcName + "-terdut-webhook")) + + secret := webhookSecret(ctx) + Expect(secret.Data[webhookSecretKeyDataKey]).NotTo(BeEmpty()) + Expect(secret.Data[webhookSecretURLKey]).NotTo(BeEmpty()) + Expect(secret.OwnerReferences).To(HaveLen(1), "same-namespace generated Secret should be owner-referenced, not finalizer-cleaned") + + created, ok := fake.integrations[team.Status.TeamID][as.Status.IntegrationID] + Expect(ok).To(BeTrue()) + Expect(created.Name).To(Equal("prod-alertmanager")) + Expect(created.Kind).To(Equal("alertmanager")) + }) + }) + + Describe("renaming", func() { + It("PATCHes the new name without rotating the key", func(ctx SpecContext) { + createSource(ctx, sameTeamRef(), "alertmanager", "prod-alertmanager") + reconcileOnce(ctx) + reconcileOnce(ctx) + Expect(readyCondition(ctx).Status).To(Equal(metav1.ConditionTrue)) + + as := &terdutv1alpha1.TerdutAlertSource{} + Expect(k8sClient.Get(ctx, srcKey, as)).To(Succeed()) + integrationID := as.Status.IntegrationID + secretBefore := webhookSecret(ctx) + + as.Spec.Name = "prod-alertmanager-renamed" + Expect(k8sClient.Update(ctx, as)).To(Succeed()) + reconcileOnce(ctx) + + Expect(fake.integrations[team.Status.TeamID][integrationID].Name).To(Equal("prod-alertmanager-renamed")) + + Expect(k8sClient.Get(ctx, srcKey, as)).To(Succeed()) + Expect(as.Status.IntegrationID).To(Equal(integrationID), "renaming must not rotate the integration id") + secretAfter := webhookSecret(ctx) + Expect(secretAfter.Data[webhookSecretKeyDataKey]).To(Equal(secretBefore.Data[webhookSecretKeyDataKey]), "renaming must not rotate the webhook key") + }) + }) + + Describe("a kind change", func() { + It("deletes the old integration, mints a new one, and rotates the webhook Secret", func(ctx SpecContext) { + createSource(ctx, sameTeamRef(), "alertmanager", "prod-alertmanager") + reconcileOnce(ctx) + reconcileOnce(ctx) + Expect(readyCondition(ctx).Status).To(Equal(metav1.ConditionTrue)) + + as := &terdutv1alpha1.TerdutAlertSource{} + Expect(k8sClient.Get(ctx, srcKey, as)).To(Succeed()) + oldIntegrationID := as.Status.IntegrationID + secretBefore := webhookSecret(ctx) + + // terdut-server only supports one kind today (DESIGN.md §4.5), + // so there's no second real value to rotate to -- this fake + // server doesn't validate kind at all, so re-POSTing under the + // same literal string still exercises the full + // delete-then-create rotation path and produces a fresh id + // and key, which is everything this test needs to verify. + as.Spec.Kind = "alertmanager" + as.Status.LastAppliedKind = "something-else" + Expect(k8sClient.Status().Update(ctx, as)).To(Succeed()) + reconcileOnce(ctx) + + Expect(fake.integrationDelete[oldIntegrationID]).To(BeTrue()) + + Expect(k8sClient.Get(ctx, srcKey, as)).To(Succeed()) + Expect(as.Status.IntegrationID).NotTo(Equal(oldIntegrationID)) + Expect(as.Status.LastAppliedKind).To(Equal("alertmanager")) + + secretAfter := webhookSecret(ctx) + Expect(secretAfter.Data[webhookSecretKeyDataKey]).NotTo(Equal(secretBefore.Data[webhookSecretKeyDataKey]), "a kind rotation must mint a new webhook key") + }) + }) + + Describe("crash recovery between POST and status write", func() { + It("adopts the webhook Secret's own record instead of minting a second integration", func(ctx SpecContext) { + createSource(ctx, sameTeamRef(), "alertmanager", "prod-alertmanager") + reconcileOnce(ctx) // finalizer only -- status.integrationID stays 0 + + // Simulate a previous reconcile that got as far as writing the + // Secret (the only durable record this resource can have, per + // its own controller comment) but crashed before status. + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: srcName + "-terdut-webhook", Namespace: operatorNamespace}, + Data: map[string][]byte{ + webhookSecretURLKey: []byte("https://terdut.example.invalid/api/integrations/orphaned-key/alertmanager"), + webhookSecretKeyDataKey: []byte("orphaned-key"), + webhookSecretIntegrationIDKey: []byte(strconv.FormatInt(999, 10)), + }, + } + Expect(k8sClient.Create(ctx, secret)).To(Succeed()) + + reconcileOnce(ctx) + + as := &terdutv1alpha1.TerdutAlertSource{} + Expect(k8sClient.Get(ctx, srcKey, as)).To(Succeed()) + Expect(as.Status.IntegrationID).To(Equal(int64(999))) + Expect(fake.integrations[team.Status.TeamID]).To(BeEmpty(), "should never have called POST at all") + Expect(readyCondition(ctx).Status).To(Equal(metav1.ConditionTrue)) + }) + }) + + Describe("webhook Secret loss", func() { + It("reports WebhookSecretLost and does not mint a replacement", func(ctx SpecContext) { + createSource(ctx, sameTeamRef(), "alertmanager", "prod-alertmanager") + reconcileOnce(ctx) + reconcileOnce(ctx) + Expect(readyCondition(ctx).Status).To(Equal(metav1.ConditionTrue)) + + as := &terdutv1alpha1.TerdutAlertSource{} + Expect(k8sClient.Get(ctx, srcKey, as)).To(Succeed()) + integrationID := as.Status.IntegrationID + + Expect(k8sClient.Delete(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{ + Name: srcName + "-terdut-webhook", Namespace: operatorNamespace, + }})).To(Succeed()) + + reconcileOnce(ctx) + + cond := readyCondition(ctx) + Expect(cond.Status).To(Equal(metav1.ConditionFalse)) + Expect(cond.Reason).To(Equal(terdutv1alpha1.ReasonWebhookSecretLost)) + Expect(fake.integrationDelete[integrationID]).To(BeFalse(), "fail-closed: must not touch the still-live integration server-side either") + }) + }) + + Describe("waiting on the referenced TerdutTeam", func() { + It("reports TeamRefNotFound when the TerdutTeam doesn't exist", func(ctx SpecContext) { + createSource(ctx, terdutv1alpha1.TerdutTeamRef{Name: testRefNotFoundName}, "alertmanager", "prod-alertmanager") + reconcileOnce(ctx) // finalizer + reconcileOnce(ctx) + + Expect(readyCondition(ctx).Reason).To(Equal(terdutv1alpha1.ReasonTeamRefNotFound)) + }) + + It("reports WaitingForTeam when the TerdutTeam exists but isn't Ready yet", func(ctx SpecContext) { + unreadyName := uniqueName("asteam-unready") + unready := &terdutv1alpha1.TerdutTeam{ + ObjectMeta: metav1.ObjectMeta{Name: unreadyName, Namespace: operatorNamespace}, + Spec: terdutv1alpha1.TerdutTeamSpec{ + ServerRef: terdutv1alpha1.TerdutServerRef{Name: srv.Name}, + DisplayName: testUnreadyDisplayName, + }, + } + Expect(k8sClient.Create(ctx, unready)).To(Succeed()) + DeferCleanup(func() { _ = k8sClient.Delete(ctx, unready) }) + + createSource(ctx, terdutv1alpha1.TerdutTeamRef{Name: unreadyName}, "alertmanager", "prod-alertmanager") + reconcileOnce(ctx) // finalizer + reconcileOnce(ctx) + + Expect(readyCondition(ctx).Reason).To(Equal(terdutv1alpha1.ReasonWaitingForTeam)) + }) + }) + + Describe("deletion", func() { + It("deletes the integration server-side and removes the finalizer", func(ctx SpecContext) { + createSource(ctx, sameTeamRef(), "alertmanager", "prod-alertmanager") + reconcileOnce(ctx) + reconcileOnce(ctx) + Expect(readyCondition(ctx).Status).To(Equal(metav1.ConditionTrue)) + + as := &terdutv1alpha1.TerdutAlertSource{} + Expect(k8sClient.Get(ctx, srcKey, as)).To(Succeed()) + integrationID := as.Status.IntegrationID + + Expect(k8sClient.Delete(ctx, as)).To(Succeed()) + reconcileOnce(ctx) // runs the finalizer + + Expect(fake.integrationDelete[integrationID]).To(BeTrue()) + err := k8sClient.Get(ctx, srcKey, as) + Expect(err).To(HaveOccurred(), "the TerdutAlertSource itself should be gone once the finalizer clears") + }) + }) +}) diff --git a/internal/controller/terdutdeadmanswitch_controller_test.go b/internal/controller/terdutdeadmanswitch_controller_test.go index e892ecc..dd71a85 100644 --- a/internal/controller/terdutdeadmanswitch_controller_test.go +++ b/internal/controller/terdutdeadmanswitch_controller_test.go @@ -32,7 +32,7 @@ var _ = Describe("TerdutDeadmanSwitch Controller", func() { fake, fakeSrv = newFakeTerdutServer() DeferCleanup(fakeSrv.Close) - srv = bootstrapReadyTerdutServer(ctx, operatorNamespace, uniqueName("dmserver"), fakeSrv.URL) + srv = bootstrapReadyTerdutServer(ctx, uniqueName("dmserver"), fakeSrv.URL) team = readyTerdutTeam(ctx, operatorNamespace, uniqueName("dmteam"), srv, fakeSrv.URL) reconciler = &TerdutDeadmanSwitchReconciler{ @@ -177,7 +177,7 @@ var _ = Describe("TerdutDeadmanSwitch Controller", func() { ObjectMeta: metav1.ObjectMeta{Name: unreadyName, Namespace: operatorNamespace}, Spec: terdutv1alpha1.TerdutTeamSpec{ ServerRef: terdutv1alpha1.TerdutServerRef{Name: srv.Name}, - DisplayName: "unready", + DisplayName: testUnreadyDisplayName, }, } Expect(k8sClient.Create(ctx, unready)).To(Succeed()) diff --git a/internal/controller/terdutescalationrule_controller_test.go b/internal/controller/terdutescalationrule_controller_test.go index c5d47c8..fcc3905 100644 --- a/internal/controller/terdutescalationrule_controller_test.go +++ b/internal/controller/terdutescalationrule_controller_test.go @@ -32,7 +32,7 @@ var _ = Describe("TerdutEscalationRule Controller", func() { fake, fakeSrv = newFakeTerdutServer() DeferCleanup(fakeSrv.Close) - srv = bootstrapReadyTerdutServer(ctx, operatorNamespace, uniqueName("erserver"), fakeSrv.URL) + srv = bootstrapReadyTerdutServer(ctx, uniqueName("erserver"), fakeSrv.URL) team = readyTerdutTeam(ctx, operatorNamespace, uniqueName("erteam"), srv, fakeSrv.URL) reconciler = &TerdutEscalationRuleReconciler{ @@ -164,7 +164,7 @@ var _ = Describe("TerdutEscalationRule Controller", func() { ObjectMeta: metav1.ObjectMeta{Name: unreadyName, Namespace: operatorNamespace}, Spec: terdutv1alpha1.TerdutTeamSpec{ ServerRef: terdutv1alpha1.TerdutServerRef{Name: srv.Name}, - DisplayName: "unready", + DisplayName: testUnreadyDisplayName, }, } Expect(k8sClient.Create(ctx, unready)).To(Succeed()) diff --git a/internal/controller/terdutserver_controller_test.go b/internal/controller/terdutserver_controller_test.go index 7ef8444..1719973 100644 --- a/internal/controller/terdutserver_controller_test.go +++ b/internal/controller/terdutserver_controller_test.go @@ -72,6 +72,13 @@ type fakeTerdutServer struct { nextSwitchID int64 switches map[int64]map[int64]tdclient.DeadmanSwitch // teamID -> switchID -> switch switchDelete map[int64]bool // switchID -> true once DELETEd, for 404-on-redelete + + // integrations/nextIntegrationID/integrationDelete back the alert + // source endpoints -- no unique-name constraint server-side either + // (DESIGN.md §4.5), same shape as switches, keyed by id. + nextIntegrationID int64 + integrations map[int64]map[int64]tdclient.Integration // teamID -> integrationID -> integration + integrationDelete map[int64]bool // integrationID -> true once DELETEd, for 404-on-redelete } func newFakeTerdutServer() (*fakeTerdutServer, *httptest.Server) { @@ -86,6 +93,9 @@ func newFakeTerdutServer() (*fakeTerdutServer, *httptest.Server) { escalation: map[int64]tdclient.SetEscalationRequest{}, switches: map[int64]map[int64]tdclient.DeadmanSwitch{}, switchDelete: map[int64]bool{}, + + integrations: map[int64]map[int64]tdclient.Integration{}, + integrationDelete: map[int64]bool{}, } return f, httptest.NewServer(f) } @@ -314,6 +324,76 @@ func (f *fakeTerdutServer) handleTeamSubPath(w http.ResponseWriter, r *http.Requ f.switchDelete[switchID] = true w.WriteHeader(http.StatusNoContent) + case rest == "/integrations" || strings.HasPrefix(rest, "/integrations/"): + f.handleIntegrationSubPath(w, r, id, rest) + + default: + w.WriteHeader(http.StatusNotFound) + } +} + +// handleIntegrationSubPath answers POST /api/teams/{id}/integrations, +// PATCH .../integrations/{integrationID} and DELETE .../integrations/{integrationID} +// -- split out of handleTeamSubPath so that switch's own cyclomatic +// complexity stays under golangci-lint's gocyclo threshold. +func (f *fakeTerdutServer) handleIntegrationSubPath(w http.ResponseWriter, r *http.Request, id int64, rest string) { + switch { + case rest == "/integrations" && r.Method == http.MethodPost: + var req struct { + Name string `json:"name"` + Kind string `json:"kind"` + } + _ = json.NewDecoder(r.Body).Decode(&req) + f.nextIntegrationID++ + integID := f.nextIntegrationID + integ := tdclient.Integration{ + ID: integID, TeamID: id, Kind: req.Kind, Name: req.Name, + // Key/URL are only ever in *this* response -- never again, + // matching terdut-server's own one-time-show semantics + // (DESIGN.md §4.5) -- so what's stored for later GET/PATCH + // calls in this fake deliberately omits them too. + Key: fmt.Sprintf("webhook-key-%d", integID), + URL: fmt.Sprintf("https://terdut.example.invalid/api/integrations/webhook-key-%d/%s", integID, req.Kind), + } + if f.integrations[id] == nil { + f.integrations[id] = map[int64]tdclient.Integration{} + } + f.integrations[id][integID] = tdclient.Integration{ID: integID, TeamID: id, Kind: req.Kind, Name: req.Name} + writeJSON(w, http.StatusCreated, integ) + + case strings.HasPrefix(rest, "/integrations/") && r.Method == http.MethodPatch: + integID, ok := parseTrailingID(rest, "/integrations/") + if !ok { + w.WriteHeader(http.StatusNotFound) + return + } + existing, exists := f.integrations[id][integID] + if !exists { + w.WriteHeader(http.StatusNotFound) + return + } + var req struct { + Name string `json:"name"` + } + _ = json.NewDecoder(r.Body).Decode(&req) + existing.Name = req.Name + f.integrations[id][integID] = existing + w.WriteHeader(http.StatusNoContent) + + case strings.HasPrefix(rest, "/integrations/") && r.Method == http.MethodDelete: + integID, ok := parseTrailingID(rest, "/integrations/") + if !ok { + w.WriteHeader(http.StatusNotFound) + return + } + if _, exists := f.integrations[id][integID]; !exists { + w.WriteHeader(http.StatusNotFound) + return + } + delete(f.integrations[id], integID) + f.integrationDelete[integID] = true + w.WriteHeader(http.StatusNoContent) + default: w.WriteHeader(http.StatusNotFound) } diff --git a/internal/controller/terdutteam_controller_test.go b/internal/controller/terdutteam_controller_test.go index a54681c..faf5dbd 100644 --- a/internal/controller/terdutteam_controller_test.go +++ b/internal/controller/terdutteam_controller_test.go @@ -33,7 +33,7 @@ var _ = Describe("TerdutTeam Controller", func() { fake, fakeSrv = newFakeTerdutServer() DeferCleanup(fakeSrv.Close) - srv = bootstrapReadyTerdutServer(ctx, operatorNamespace, uniqueName("ttserver"), fakeSrv.URL) + srv = bootstrapReadyTerdutServer(ctx, uniqueName("ttserver"), fakeSrv.URL) reconciler = &TerdutTeamReconciler{ Client: k8sClient, diff --git a/internal/controller/testhelpers_test.go b/internal/controller/testhelpers_test.go index 41de9d9..45178a2 100644 --- a/internal/controller/testhelpers_test.go +++ b/internal/controller/testhelpers_test.go @@ -26,10 +26,25 @@ const ( // testRefNotFoundName is a name no TerdutServer/TerdutTeam ever gets // created with -- shared by every "...RefNotFound" test across - // terdutteam_controller_test.go, terdutescalationrule_controller_test.go - // and terdutdeadmanswitch_controller_test.go, so goconst doesn't flag - // three independent copies of the same literal. + // terdutteam_controller_test.go, terdutescalationrule_controller_test.go, + // terdutdeadmanswitch_controller_test.go and + // terdutalertsource_controller_test.go, so goconst doesn't flag four + // independent copies of the same literal. testRefNotFoundName = "does-not-exist" + + // testUnreadyDisplayName is the TerdutTeam.spec.displayName every + // "...exists but isn't Ready yet" test across the same four files uses + // for its deliberately-never-reconciled fixture team, for the same + // goconst reason as testRefNotFoundName above. + testUnreadyDisplayName = "unready" + + // testOperatorNamespace is every test file's own namespace for both + // the operator's generated/credential objects and the CRs under test + // -- always "default" in this suite, so bootstrapReadyTerdutServer + // below takes no namespace parameter of its own (golangci-lint's + // unparam flagged it once a fourth same-valued caller made that + // obvious): there's never a second value to pass. + testOperatorNamespace = "default" ) // bootstrapReadyTerdutServer creates a TerdutServer with a bring-your-own @@ -37,8 +52,9 @@ const ( // terdutserver_controller_test.go's own happy-path test exercises directly // -- shared here so TerdutTeam's tests (which need a real, Ready // TerdutServer to resolve against) don't duplicate it. -func bootstrapReadyTerdutServer(ctx context.Context, namespace, name, fakeURL string) *terdutv1alpha1.TerdutServer { +func bootstrapReadyTerdutServer(ctx context.Context, name, fakeURL string) *terdutv1alpha1.TerdutServer { GinkgoHelper() + namespace := testOperatorNamespace reconciler := &TerdutServerReconciler{ Client: k8sClient, diff --git a/internal/tdclient/client.go b/internal/tdclient/client.go index 9642c12..56d81ae 100644 --- a/internal/tdclient/client.go +++ b/internal/tdclient/client.go @@ -23,6 +23,10 @@ import ( // fieldName is the JSON key every create/rename request body below shares. const fieldName = "name" +// fieldKind is the JSON key an integration's create request body shares +// with the terdutv1alpha1.TerdutAlertSourceSpec field of the same name. +const fieldKind = "kind" + type Client struct { baseURL string httpClient *http.Client @@ -496,3 +500,69 @@ func (c *Client) DeleteDeadmanSwitch(ctx context.Context, teamID, switchID int64 } return c.do(req, nil) } + +// Integration mirrors terdut-server's models.Integration, minus +// CreatedAt/LastUsedAt, which this client never reads. Key/URL are only +// ever populated by CreateIntegration's own response -- the one moment +// either value exists outside terdut-server's own database (DESIGN.md +// §4.5: shown once, never re-readable, same handling as the bootstrap +// admin key). +type Integration struct { + ID int64 `json:"id"` + TeamID int64 `json:"team_id"` + Kind string `json:"kind"` + Name string `json:"name"` + Key string `json:"key,omitempty"` + URL string `json:"url,omitempty"` +} + +// CreateIntegration calls POST /api/teams/{teamID}/integrations -- +// owner-gated (requireTeamOwner), so c must hold this team's own +// team-scoped credential. No conflict handling exists server-side at all +// for this resource (no unique constraint on name, confirmed against +// source) -- deliberately not treated as this resource's idempotent-create +// recovery path; see the controller's own reasoning for why a crash +// between this call succeeding and the webhook Secret being written can't +// be recovered by listing and adopting a same-named row the way +// TerdutDeadmanSwitch does. +func (c *Client) CreateIntegration(ctx context.Context, teamID int64, name, kind string) (*Integration, error) { + req, err := c.newRequest(ctx, http.MethodPost, fmt.Sprintf("/api/teams/%d/integrations", teamID), + map[string]string{fieldName: name, fieldKind: kind}) + if err != nil { + return nil, err + } + var integ Integration + if err := c.do(req, &integ); err != nil { + return nil, err + } + return &integ, nil +} + +// RenameIntegration calls PATCH /api/teams/{teamID}/integrations/{integrationID} +// -- owner-gated, same credential requirement as CreateIntegration. Never +// touches the key (terdut-server's own handler comment: "the key is +// untouched, so nothing posting with it notices"), so this is safe to call +// every reconcile unconditionally rather than only on detected drift -- +// the same "cheap, so just always sync it" reasoning TerdutEscalationRule's +// whole-policy PUT uses. +func (c *Client) RenameIntegration(ctx context.Context, teamID, integrationID int64, name string) error { + req, err := c.newRequest(ctx, http.MethodPatch, + fmt.Sprintf("/api/teams/%d/integrations/%d", teamID, integrationID), + map[string]string{fieldName: name}) + if err != nil { + return err + } + return c.do(req, nil) +} + +// DeleteIntegration calls DELETE /api/teams/{teamID}/integrations/{integrationID} +// -- owner-gated, same credential requirement as CreateIntegration. Used +// both by the finalizer and by the kind-change rotation path (DESIGN.md §5). +func (c *Client) DeleteIntegration(ctx context.Context, teamID, integrationID int64) error { + req, err := c.newRequest(ctx, http.MethodDelete, + fmt.Sprintf("/api/teams/%d/integrations/%d", teamID, integrationID), nil) + if err != nil { + return err + } + return c.do(req, nil) +}