Authenticate with a seeded operator key; fold escalation and switches into TerdutTeam
Credentials: the TerdutServer controller generates <name>-operator-key in the server's own namespace (owned by it) and hands it to the pods as TERDUT_OPERATOR_KEY; the server creates its instance-scoped account from it at every start. A replaced Secret rolls the pods. The bootstrap handshake, the checkpoint Secret, per-team service accounts and credentials Secrets, BootstrapStateLost and credentials.deletionPolicy are gone. CRDs: TerdutServer, TerdutTeam and TerdutAlertSource. TerdutEscalationRule and TerdutDeadmanSwitch become spec.escalation and spec.deadmanSwitches[] on the team (matched by name, extras removed); team invites are removed. A team is created under the identity <namespace>/<name> (external_id), so a retry, a lost status or a deleted team heal by repeating the same call, and a display name owned by another team is TeamNameTaken instead of an adoption. The server resolves escalation usernames (UnknownUser condition). OIDC claim names and trustEmail are spec fields. Fixes: query values are URL-escaped; every delete treats 404 as success; deleting a team no longer depends on allowedTeams consent; a switch or integration deleted on the server is recreated; unnamed switches take the CR's name. Cleanup: scaffold e2e test, AGENTS.md, devcontainer, unused config/ pieces and Client.Version() removed; DESIGN.md, README, ROADMAP and the demo (run-demo.sh, manifests) rewritten for the new design. Secret RBAC stays cluster-wide, now stated in DESIGN.md section 9. Claude-Session: https://claude.ai/code/session_016mBLURvJoMuUEr9cB2RpUN
This commit is contained in:
Niklas Ye
@@ -5,14 +5,15 @@ import (
|
||||
"fmt"
|
||||
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/api/meta"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
terdutv1alpha1 "git.ryuvia.com/niklas/terdut-operator/api/v1alpha1"
|
||||
"git.ryuvia.com/niklas/terdut-operator/internal/tdclient"
|
||||
)
|
||||
|
||||
// childError carries a condition reason/message, the same role teamError
|
||||
// and databaseError play for their own controllers: an expected,
|
||||
// childError carries a condition reason/message, the same role teamError and
|
||||
// databaseError play for their own controllers: an expected,
|
||||
// requeue-and-retry outcome, not a reconcile failure.
|
||||
type childError struct {
|
||||
reason string
|
||||
@@ -21,12 +22,13 @@ type childError struct {
|
||||
|
||||
func (e *childError) Error() string { return e.message }
|
||||
|
||||
// resolveTeamAndClient implements DESIGN.md §5's "every child resolves its
|
||||
// own teamRef -> TerdutTeam.status, never chains up to TerdutServer"
|
||||
// rule -- shared by TerdutEscalationRule and TerdutDeadmanSwitch, which
|
||||
// both need exactly this and nothing else to call terdut-server's API.
|
||||
// resolveTeamAndClient finds the TerdutTeam a child (TerdutAlertSource) names,
|
||||
// requires it to be Ready, and returns a client for its server authenticated
|
||||
// with that server's operator key. The server is read from the team's own
|
||||
// serverRef: consent was already checked when the team went Ready, and does not
|
||||
// gate a resource the team owns.
|
||||
func resolveTeamAndClient(
|
||||
ctx context.Context, c client.Client, operatorNamespace, namespace string,
|
||||
ctx context.Context, c client.Client, namespace string,
|
||||
ref terdutv1alpha1.TerdutTeamRef, newClient func(string) *tdclient.Client,
|
||||
) (*terdutv1alpha1.TerdutTeam, *tdclient.Client, *childError) {
|
||||
var team terdutv1alpha1.TerdutTeam
|
||||
@@ -39,16 +41,24 @@ func resolveTeamAndClient(
|
||||
}
|
||||
return nil, nil, &childError{reason: terdutv1alpha1.ReasonTeamRefNotFound, message: err.Error()}
|
||||
}
|
||||
if team.Status.CredentialsSecretRef == nil || team.Status.TeamID == 0 {
|
||||
if team.Status.TeamID == 0 || !meta.IsStatusConditionTrue(team.Status.Conditions, terdutv1alpha1.ConditionReady) {
|
||||
return nil, nil, &childError{
|
||||
reason: terdutv1alpha1.ReasonWaitingForTeam,
|
||||
message: fmt.Sprintf("TerdutTeam %q is not Ready yet", ref.Name),
|
||||
}
|
||||
}
|
||||
|
||||
teamKey, err := readOperatorSecret(ctx, c, operatorNamespace, team.Status.CredentialsSecretRef)
|
||||
srvNS := team.Spec.ServerRef.Namespace
|
||||
if srvNS == "" {
|
||||
srvNS = team.Namespace
|
||||
}
|
||||
var srv terdutv1alpha1.TerdutServer
|
||||
if err := c.Get(ctx, client.ObjectKey{Namespace: srvNS, Name: team.Spec.ServerRef.Name}, &srv); err != nil {
|
||||
return nil, nil, &childError{reason: terdutv1alpha1.ReasonWaitingForTeam, message: err.Error()}
|
||||
}
|
||||
key, err := readOperatorKey(ctx, c, &srv)
|
||||
if err != nil {
|
||||
return nil, nil, &childError{reason: terdutv1alpha1.ReasonWaitingForTeam, message: err.Error()}
|
||||
}
|
||||
return &team, newClient(team.Status.ServerEndpoint).WithToken(teamKey), nil
|
||||
return &team, newClient(serviceURL(&srv)).WithToken(key), nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,375 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.ryuvia.com/niklas/terdut-operator/internal/tdclient"
|
||||
)
|
||||
|
||||
// errJSONKey is the key every error body the fake writes uses, and
|
||||
// deadmanSwitchesPath the literal shared by its dead man's switch routes --
|
||||
// goconst would otherwise flag the repeats.
|
||||
const (
|
||||
errJSONKey = "error"
|
||||
deadmanSwitchesPath = "/deadman/switches"
|
||||
)
|
||||
|
||||
// fakeTerdutServer reproduces the parts of terdut-server's API the controllers
|
||||
// call, with the same status codes and idempotency rules (POST /api/teams
|
||||
// keyed by external_id, unique team names, unique switch names per team), so a
|
||||
// controller test exercises the real contract and not a canned reply.
|
||||
type fakeTerdutServer struct {
|
||||
mu sync.Mutex
|
||||
|
||||
// expectKey, when set, makes every request without that bearer token a 401.
|
||||
expectKey string
|
||||
|
||||
nextTeamID int64
|
||||
teams map[string]int64 // name -> id
|
||||
teamNames map[int64]string // id -> current name
|
||||
teamExt map[string]int64 // external_id -> id
|
||||
teamOIDC map[int64][2]string
|
||||
teamDelete map[int64]bool // id -> true once DELETEd
|
||||
|
||||
// unknownUsers are usernames the escalation PUT answers 400 "unknown user" for.
|
||||
unknownUsers map[string]bool
|
||||
escalation map[int64]tdclient.SetEscalationRequest
|
||||
|
||||
nextSwitchID int64
|
||||
switches map[int64]map[int64]tdclient.DeadmanSwitch // teamID -> switchID -> switch
|
||||
switchDelete map[int64]bool
|
||||
|
||||
nextIntegrationID int64
|
||||
integrations map[int64]map[int64]tdclient.Integration
|
||||
integrationDelete map[int64]bool
|
||||
}
|
||||
|
||||
func newFakeTerdutServer() (*fakeTerdutServer, *httptest.Server) {
|
||||
f := &fakeTerdutServer{
|
||||
teams: map[string]int64{},
|
||||
teamNames: map[int64]string{},
|
||||
teamExt: map[string]int64{},
|
||||
teamOIDC: map[int64][2]string{},
|
||||
teamDelete: map[int64]bool{},
|
||||
unknownUsers: map[string]bool{},
|
||||
escalation: map[int64]tdclient.SetEscalationRequest{},
|
||||
switches: map[int64]map[int64]tdclient.DeadmanSwitch{},
|
||||
switchDelete: map[int64]bool{},
|
||||
integrations: map[int64]map[int64]tdclient.Integration{},
|
||||
integrationDelete: map[int64]bool{},
|
||||
}
|
||||
return f, httptest.NewServer(f)
|
||||
}
|
||||
|
||||
func (f *fakeTerdutServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
|
||||
if f.expectKey != "" && r.Header.Get("Authorization") != "Bearer "+f.expectKey {
|
||||
writeJSON(w, http.StatusUnauthorized, map[string]string{errJSONKey: "invalid or expired API key"})
|
||||
return
|
||||
}
|
||||
|
||||
switch {
|
||||
case r.URL.Path == "/api/version":
|
||||
writeJSON(w, http.StatusOK, map[string]string{"version": "test"})
|
||||
|
||||
case r.URL.Path == "/api/teams" && r.Method == http.MethodPost:
|
||||
var req struct {
|
||||
Name string `json:"name"`
|
||||
ExternalID string `json:"external_id"`
|
||||
}
|
||||
_ = json.NewDecoder(r.Body).Decode(&req)
|
||||
if id, ok := f.teamExt[req.ExternalID]; ok && req.ExternalID != "" {
|
||||
writeJSON(w, http.StatusOK, tdclient.Team{ID: id, Name: f.teamNames[id]})
|
||||
return
|
||||
}
|
||||
if _, exists := f.teams[req.Name]; exists {
|
||||
writeJSON(w, http.StatusConflict, map[string]string{errJSONKey: "a team with that name already exists"})
|
||||
return
|
||||
}
|
||||
f.nextTeamID++
|
||||
id := f.nextTeamID
|
||||
f.teams[req.Name] = id
|
||||
f.teamNames[id] = req.Name
|
||||
if req.ExternalID != "" {
|
||||
f.teamExt[req.ExternalID] = id
|
||||
}
|
||||
writeJSON(w, http.StatusCreated, tdclient.Team{ID: id, Name: req.Name})
|
||||
|
||||
default:
|
||||
if id, rest, ok := parseTeamSubPath(r.URL.Path); ok {
|
||||
f.handleTeamSubPath(w, r, id, rest)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}
|
||||
|
||||
// handleTeamSubPath answers everything under /api/teams/{id}. rest is whatever
|
||||
// parseTeamSubPath found after "/api/teams/{id}" -- "" for the bare resource.
|
||||
func (f *fakeTerdutServer) handleTeamSubPath(w http.ResponseWriter, r *http.Request, id int64, rest string) {
|
||||
switch {
|
||||
case rest == "" && r.Method == http.MethodPut:
|
||||
var req struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
_ = json.NewDecoder(r.Body).Decode(&req)
|
||||
oldName, exists := f.teamNames[id]
|
||||
if !exists {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
if other, taken := f.teams[req.Name]; taken && other != id {
|
||||
writeJSON(w, http.StatusConflict, map[string]string{errJSONKey: "a team with that name already exists"})
|
||||
return
|
||||
}
|
||||
delete(f.teams, oldName)
|
||||
f.teamNames[id] = req.Name
|
||||
f.teams[req.Name] = id
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
|
||||
case rest == "" && r.Method == http.MethodDelete:
|
||||
name, exists := f.teamNames[id]
|
||||
if !exists {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
delete(f.teams, name)
|
||||
delete(f.teamNames, id)
|
||||
f.teamDelete[id] = true
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
|
||||
case rest == "/oidc-groups" && r.Method == http.MethodPut:
|
||||
var req struct {
|
||||
MemberGroup string `json:"member_group"`
|
||||
OwnerGroup string `json:"owner_group"`
|
||||
}
|
||||
_ = json.NewDecoder(r.Body).Decode(&req)
|
||||
if _, exists := f.teamNames[id]; !exists {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
f.teamOIDC[id] = [2]string{req.MemberGroup, req.OwnerGroup}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
|
||||
case rest == "/escalation" && r.Method == http.MethodPut:
|
||||
var req tdclient.SetEscalationRequest
|
||||
_ = json.NewDecoder(r.Body).Decode(&req)
|
||||
for _, l := range req.Levels {
|
||||
for _, t := range l.Targets {
|
||||
if t.Username != "" && f.unknownUsers[t.Username] {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{errJSONKey: fmt.Sprintf("unknown user %q", t.Username)})
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
f.escalation[id] = req
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
|
||||
case rest == deadmanSwitchesPath || strings.HasPrefix(rest, deadmanSwitchesPath+"/"):
|
||||
f.handleDeadmanSubPath(w, r, id, rest)
|
||||
|
||||
case rest == "/integrations" || strings.HasPrefix(rest, "/integrations/"):
|
||||
f.handleIntegrationSubPath(w, r, id, rest)
|
||||
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}
|
||||
|
||||
// handleDeadmanSubPath answers GET/POST /api/teams/{id}/deadman/switches and
|
||||
// PUT/DELETE .../deadman/switches/{switchID} -- split out of
|
||||
// handleTeamSubPath for the same gocyclo reason as handleIntegrationSubPath.
|
||||
func (f *fakeTerdutServer) handleDeadmanSubPath(w http.ResponseWriter, r *http.Request, id int64, rest string) {
|
||||
switch {
|
||||
case rest == deadmanSwitchesPath && r.Method == http.MethodGet:
|
||||
existing := f.switches[id]
|
||||
out := make([]tdclient.DeadmanSwitch, 0, len(existing))
|
||||
for _, s := range existing {
|
||||
out = append(out, s)
|
||||
}
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
|
||||
case rest == deadmanSwitchesPath && r.Method == http.MethodPost:
|
||||
var req deadmanSwitchFakeRequest
|
||||
_ = json.NewDecoder(r.Body).Decode(&req)
|
||||
name := req.Name
|
||||
for _, s := range f.switches[id] {
|
||||
if s.Name == name {
|
||||
writeJSON(w, http.StatusConflict, map[string]string{errJSONKey: "a switch with that name already exists in this team"})
|
||||
return
|
||||
}
|
||||
}
|
||||
f.nextSwitchID++
|
||||
switchID := f.nextSwitchID
|
||||
sw := tdclient.DeadmanSwitch{
|
||||
ID: switchID, Name: name, Matcher: req.Matcher,
|
||||
TimeoutSeconds: req.TimeoutSeconds, Severity: req.Severity,
|
||||
}
|
||||
if f.switches[id] == nil {
|
||||
f.switches[id] = map[int64]tdclient.DeadmanSwitch{}
|
||||
}
|
||||
f.switches[id][switchID] = sw
|
||||
writeJSON(w, http.StatusCreated, sw)
|
||||
|
||||
case strings.HasPrefix(rest, "/deadman/switches/") && r.Method == http.MethodPut:
|
||||
switchID, ok := parseTrailingID(rest, "/deadman/switches/")
|
||||
if !ok {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
if _, exists := f.switches[id][switchID]; !exists {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
var req deadmanSwitchFakeRequest
|
||||
_ = json.NewDecoder(r.Body).Decode(&req)
|
||||
name := req.Name
|
||||
if name == "" {
|
||||
name = f.switches[id][switchID].Name
|
||||
}
|
||||
f.switches[id][switchID] = tdclient.DeadmanSwitch{
|
||||
ID: switchID, Name: name, Matcher: req.Matcher,
|
||||
TimeoutSeconds: req.TimeoutSeconds, Severity: req.Severity,
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
|
||||
case strings.HasPrefix(rest, "/deadman/switches/") && r.Method == http.MethodDelete:
|
||||
switchID, ok := parseTrailingID(rest, "/deadman/switches/")
|
||||
if !ok {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
if _, exists := f.switches[id][switchID]; !exists {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
delete(f.switches[id], switchID)
|
||||
f.switchDelete[switchID] = true
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}
|
||||
|
||||
// handleIntegrationSubPath answers POST /api/teams/{id}/integrations,
|
||||
// PATCH .../integrations/{integrationID} and DELETE .../integrations/{integrationID}
|
||||
// -- split out of handleTeamSubPath so that switch's own cyclomatic
|
||||
// complexity stays under golangci-lint's gocyclo threshold.
|
||||
func (f *fakeTerdutServer) handleIntegrationSubPath(w http.ResponseWriter, r *http.Request, id int64, rest string) {
|
||||
switch {
|
||||
case rest == "/integrations" && r.Method == http.MethodPost:
|
||||
var req struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
}
|
||||
_ = json.NewDecoder(r.Body).Decode(&req)
|
||||
f.nextIntegrationID++
|
||||
integID := f.nextIntegrationID
|
||||
integ := tdclient.Integration{
|
||||
ID: integID, TeamID: id, Kind: req.Kind, Name: req.Name,
|
||||
// Key/URL are only ever in *this* response -- never again,
|
||||
// matching terdut-server's own one-time-show semantics
|
||||
// (DESIGN.md §4.5) -- so what's stored for later GET/PATCH
|
||||
// calls in this fake deliberately omits them too.
|
||||
Key: fmt.Sprintf("webhook-key-%d", integID),
|
||||
URL: fmt.Sprintf("https://terdut.example.invalid/api/integrations/webhook-key-%d/%s", integID, req.Kind),
|
||||
}
|
||||
if f.integrations[id] == nil {
|
||||
f.integrations[id] = map[int64]tdclient.Integration{}
|
||||
}
|
||||
f.integrations[id][integID] = tdclient.Integration{ID: integID, TeamID: id, Kind: req.Kind, Name: req.Name}
|
||||
writeJSON(w, http.StatusCreated, integ)
|
||||
|
||||
case strings.HasPrefix(rest, "/integrations/") && r.Method == http.MethodPatch:
|
||||
integID, ok := parseTrailingID(rest, "/integrations/")
|
||||
if !ok {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
existing, exists := f.integrations[id][integID]
|
||||
if !exists {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
var req struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
_ = json.NewDecoder(r.Body).Decode(&req)
|
||||
existing.Name = req.Name
|
||||
f.integrations[id][integID] = existing
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
|
||||
case strings.HasPrefix(rest, "/integrations/") && r.Method == http.MethodDelete:
|
||||
integID, ok := parseTrailingID(rest, "/integrations/")
|
||||
if !ok {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
if _, exists := f.integrations[id][integID]; !exists {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
delete(f.integrations[id], integID)
|
||||
f.integrationDelete[integID] = true
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}
|
||||
|
||||
// deadmanSwitchFakeRequest mirrors tdclient's own (unexported)
|
||||
// deadmanSwitchRequest -- the fake needs its own copy to decode the same
|
||||
// wire shape without reaching across package boundaries for an internal type.
|
||||
type deadmanSwitchFakeRequest struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
Matcher string `json:"matcher"`
|
||||
TimeoutSeconds int64 `json:"timeout_seconds"`
|
||||
Severity string `json:"severity"`
|
||||
}
|
||||
|
||||
// parseTeamSubPath splits "/api/teams/{id}" from anything after it --
|
||||
// "" for an exact match, "/oidc-groups", "/escalation", "/deadman/switches"
|
||||
// or "/deadman/switches/{switchID}" otherwise. Doesn't itself validate the
|
||||
// suffix; handleTeamSubPath's own switch does that.
|
||||
func parseTeamSubPath(path string) (id int64, rest string, ok bool) {
|
||||
const prefix = "/api/teams/"
|
||||
if !strings.HasPrefix(path, prefix) {
|
||||
return 0, "", false
|
||||
}
|
||||
trimmed := path[len(prefix):]
|
||||
parts := strings.SplitN(trimmed, "/", 2)
|
||||
parsedID, err := strconv.ParseInt(parts[0], 10, 64)
|
||||
if err != nil {
|
||||
return 0, "", false
|
||||
}
|
||||
if len(parts) == 1 {
|
||||
return parsedID, "", true
|
||||
}
|
||||
return parsedID, "/" + parts[1], true
|
||||
}
|
||||
|
||||
// parseTrailingID parses the numeric id after prefix within rest, e.g.
|
||||
// parseTrailingID("/deadman/switches/7", "/deadman/switches/") -> 7, true.
|
||||
func parseTrailingID(rest, prefix string) (id int64, ok bool) {
|
||||
parsedID, err := strconv.ParseInt(strings.TrimPrefix(rest, prefix), 10, 64)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
return parsedID, true
|
||||
}
|
||||
|
||||
func writeJSON(w http.ResponseWriter, status int, v any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(v)
|
||||
}
|
||||
@@ -1,49 +0,0 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
terdutv1alpha1 "git.ryuvia.com/niklas/terdut-operator/api/v1alpha1"
|
||||
)
|
||||
|
||||
// credentialsSecretDataKey is the fixed data key every generated
|
||||
// credentials Secret this controller writes uses — "whatever a human chose"
|
||||
// only applied to the bring-your-own input an earlier design draft had and
|
||||
// removed (DESIGN.md §6); every Secret any controller in this repo
|
||||
// generates itself uses this one key.
|
||||
const credentialsSecretDataKey = "token"
|
||||
|
||||
// writeOperatorSecret creates or replaces a Secret in namespace (always the
|
||||
// operator's own, DESIGN.md §6) holding one raw value under
|
||||
// credentialsSecretDataKey. Shared by every controller that generates a
|
||||
// credential there -- TerdutServer's instance-scoped key and the bootstrap
|
||||
// admin-key checkpoint, TerdutTeam's team-scoped key.
|
||||
func writeOperatorSecret(ctx context.Context, c client.Client, namespace, name, rawValue string) error {
|
||||
secret := &corev1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace},
|
||||
Data: map[string][]byte{credentialsSecretDataKey: []byte(rawValue)},
|
||||
}
|
||||
if err := c.Create(ctx, secret); err != nil {
|
||||
if apierrors.IsAlreadyExists(err) {
|
||||
return c.Update(ctx, secret)
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// readOperatorSecret reads one generated credential back, by the
|
||||
// SecretKeyRef a controller's own status stores (always resolved in the
|
||||
// operator's own namespace, never the referencing CR's -- DESIGN.md §6).
|
||||
func readOperatorSecret(ctx context.Context, c client.Client, namespace string, ref *terdutv1alpha1.SecretKeyRef) (string, error) {
|
||||
var secret corev1.Secret
|
||||
if err := c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: ref.Name}, &secret); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(secret.Data[ref.Key]), nil
|
||||
}
|
||||
@@ -2,7 +2,9 @@ package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
@@ -39,9 +41,8 @@ type TerdutAlertSourceReconciler struct {
|
||||
client.Client
|
||||
Scheme *runtime.Scheme
|
||||
|
||||
OperatorNamespace string
|
||||
Recorder recorder.EventRecorder
|
||||
NewClient func(endpoint string) *tdclient.Client
|
||||
Recorder recorder.EventRecorder
|
||||
NewClient func(endpoint string) *tdclient.Client
|
||||
}
|
||||
|
||||
// +kubebuilder:rbac:groups=terdut.ryuvia.com,resources=terdutalertsources,verbs=get;list;watch;create;update;patch;delete
|
||||
@@ -79,7 +80,7 @@ func (r *TerdutAlertSourceReconciler) Reconcile(ctx context.Context, req ctrl.Re
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
|
||||
team, tc, resolveErr := resolveTeamAndClient(ctx, r.Client, r.OperatorNamespace, as.Namespace, as.Spec.TeamRef, newClient)
|
||||
team, tc, resolveErr := resolveTeamAndClient(ctx, r.Client, as.Namespace, as.Spec.TeamRef, newClient)
|
||||
if resolveErr != nil {
|
||||
return r.setNotReady(ctx, &as, resolveErr.reason, resolveErr.message, waitInterval)
|
||||
}
|
||||
@@ -111,7 +112,27 @@ func (r *TerdutAlertSourceReconciler) Reconcile(ctx context.Context, req ctrl.Re
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
} else if err := tc.RenameIntegration(ctx, teamID, as.Status.IntegrationID, as.Spec.Name); err != nil {
|
||||
return ctrl.Result{}, fmt.Errorf("PATCH /api/teams/%d/integrations/%d: %w", teamID, as.Status.IntegrationID, err)
|
||||
se, ok := errors.AsType[*tdclient.StatusError](err)
|
||||
if !ok || se.Code != http.StatusNotFound {
|
||||
return ctrl.Result{}, fmt.Errorf("PATCH /api/teams/%d/integrations/%d: %w", teamID, as.Status.IntegrationID, err)
|
||||
}
|
||||
// Deleted on the server behind our back, so the webhook URL
|
||||
// already stopped working: recreate it (a new URL, in the same
|
||||
// Secret) rather than fail forever. reconcileCreate trusts the
|
||||
// Secret's existence as "already created", so drop it first.
|
||||
lostID := as.Status.IntegrationID
|
||||
if err := r.Delete(ctx, &secret); err != nil && !apierrors.IsNotFound(err) {
|
||||
return ctrl.Result{}, fmt.Errorf("deleting stale webhook Secret %s/%s: %w", as.Namespace, secretName, err)
|
||||
}
|
||||
as.Status.IntegrationID = 0
|
||||
if err := r.reconcileCreate(ctx, &as, tc, teamID); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
if r.Recorder != nil {
|
||||
r.Recorder.Eventf(&as, nil, corev1.EventTypeWarning, "IntegrationRecreated", "IntegrationRecreated",
|
||||
"integration %d no longer exists on the server; created %d with a new webhook URL (Secret %s)",
|
||||
lostID, as.Status.IntegrationID, secretName)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -279,7 +300,7 @@ func (r *TerdutAlertSourceReconciler) reconcileDelete(
|
||||
|
||||
if as.Status.IntegrationID != 0 {
|
||||
if team, tc, resolveErr := resolveTeamAndClient(
|
||||
ctx, r.Client, r.OperatorNamespace, as.Namespace, as.Spec.TeamRef, newClient,
|
||||
ctx, r.Client, as.Namespace, as.Spec.TeamRef, newClient,
|
||||
); resolveErr == nil {
|
||||
if err := tc.DeleteIntegration(ctx, team.Status.TeamID, as.Status.IntegrationID); err != nil {
|
||||
if r.Recorder != nil {
|
||||
|
||||
@@ -34,14 +34,13 @@ var _ = Describe("TerdutAlertSource Controller", func() {
|
||||
fake, fakeSrv = newFakeTerdutServer()
|
||||
DeferCleanup(fakeSrv.Close)
|
||||
|
||||
srv = bootstrapReadyTerdutServer(ctx, uniqueName("asserver"), fakeSrv.URL)
|
||||
srv = readyTerdutServer(ctx, uniqueName("asserver"))
|
||||
team = readyTerdutTeam(ctx, operatorNamespace, uniqueName("asteam"), srv, fakeSrv.URL)
|
||||
|
||||
reconciler = &TerdutAlertSourceReconciler{
|
||||
Client: k8sClient,
|
||||
Scheme: k8sClient.Scheme(),
|
||||
OperatorNamespace: operatorNamespace,
|
||||
NewClient: func(string) *tdclient.Client { return tdclient.New(fakeSrv.URL) },
|
||||
Client: k8sClient,
|
||||
Scheme: k8sClient.Scheme(),
|
||||
NewClient: func(string) *tdclient.Client { return tdclient.New(fakeSrv.URL) },
|
||||
}
|
||||
srcName = uniqueName("alertsource")
|
||||
srcKey = types.NamespacedName{Name: srcName, Namespace: operatorNamespace}
|
||||
|
||||
@@ -1,199 +0,0 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/api/meta"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
|
||||
logf "sigs.k8s.io/controller-runtime/pkg/log"
|
||||
"sigs.k8s.io/controller-runtime/pkg/recorder"
|
||||
|
||||
terdutv1alpha1 "git.ryuvia.com/niklas/terdut-operator/api/v1alpha1"
|
||||
"git.ryuvia.com/niklas/terdut-operator/internal/tdclient"
|
||||
)
|
||||
|
||||
const deadmanFinalizerName = "terdut.ryuvia.com/terdutdeadmanswitch"
|
||||
|
||||
// TerdutDeadmanSwitchReconciler reconciles a TerdutDeadmanSwitch object.
|
||||
type TerdutDeadmanSwitchReconciler struct {
|
||||
client.Client
|
||||
Scheme *runtime.Scheme
|
||||
|
||||
OperatorNamespace string
|
||||
Recorder recorder.EventRecorder
|
||||
NewClient func(endpoint string) *tdclient.Client
|
||||
}
|
||||
|
||||
// +kubebuilder:rbac:groups=terdut.ryuvia.com,resources=terdutdeadmanswitches,verbs=get;list;watch;create;update;patch;delete
|
||||
// +kubebuilder:rbac:groups=terdut.ryuvia.com,resources=terdutdeadmanswitches/status,verbs=get;update;patch
|
||||
// +kubebuilder:rbac:groups=terdut.ryuvia.com,resources=terdutdeadmanswitches/finalizers,verbs=update
|
||||
// +kubebuilder:rbac:groups=terdut.ryuvia.com,resources=terdutteams,verbs=get;list;watch
|
||||
// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch
|
||||
// +kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=create;patch
|
||||
|
||||
func (r *TerdutDeadmanSwitchReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
|
||||
log := logf.FromContext(ctx)
|
||||
|
||||
var sw terdutv1alpha1.TerdutDeadmanSwitch
|
||||
if err := r.Get(ctx, req.NamespacedName, &sw); err != nil {
|
||||
if apierrors.IsNotFound(err) {
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
|
||||
newClient := r.NewClient
|
||||
if newClient == nil {
|
||||
newClient = tdclient.New
|
||||
}
|
||||
|
||||
if !sw.DeletionTimestamp.IsZero() {
|
||||
return r.reconcileDeadmanDelete(ctx, &sw, newClient)
|
||||
}
|
||||
|
||||
if !controllerutil.ContainsFinalizer(&sw, deadmanFinalizerName) {
|
||||
controllerutil.AddFinalizer(&sw, deadmanFinalizerName)
|
||||
if err := r.Update(ctx, &sw); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
|
||||
team, tc, resolveErr := resolveTeamAndClient(ctx, r.Client, r.OperatorNamespace, sw.Namespace, sw.Spec.TeamRef, newClient)
|
||||
if resolveErr != nil {
|
||||
return r.setDeadmanNotReady(ctx, &sw, resolveErr.reason, resolveErr.message, waitInterval)
|
||||
}
|
||||
|
||||
timeout, err := time.ParseDuration(sw.Spec.Timeout)
|
||||
if err != nil {
|
||||
return ctrl.Result{}, fmt.Errorf("spec.timeout %q: %w", sw.Spec.Timeout, err)
|
||||
}
|
||||
severity := sw.Spec.Severity
|
||||
if severity == "" {
|
||||
severity = "critical"
|
||||
}
|
||||
|
||||
if sw.Status.SwitchID == 0 {
|
||||
if err := r.createOrAdoptDeadmanSwitch(ctx, &sw, tc, team.Status.TeamID, timeout, severity); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
} else if err := tc.UpdateDeadmanSwitch(ctx, team.Status.TeamID, sw.Status.SwitchID, sw.Spec.Name, sw.Spec.Matcher, int64(timeout.Seconds()), severity); err != nil {
|
||||
return ctrl.Result{}, fmt.Errorf("PUT /api/teams/%d/deadman/switches/%d: %w", team.Status.TeamID, sw.Status.SwitchID, err)
|
||||
}
|
||||
|
||||
meta.SetStatusCondition(&sw.Status.Conditions, metav1.Condition{
|
||||
Type: terdutv1alpha1.ConditionReady,
|
||||
Status: metav1.ConditionTrue,
|
||||
Reason: terdutv1alpha1.ReasonChildAdopted,
|
||||
Message: fmt.Sprintf("switch %d applied on team %d", sw.Status.SwitchID, team.Status.TeamID),
|
||||
})
|
||||
sw.Status.ObservedGeneration = sw.Generation
|
||||
if err := r.Status().Update(ctx, &sw); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
if r.Recorder != nil {
|
||||
r.Recorder.Eventf(&sw, nil, corev1.EventTypeNormal, terdutv1alpha1.ReasonChildAdopted, terdutv1alpha1.ReasonChildAdopted,
|
||||
"dead man's switch applied")
|
||||
}
|
||||
log.Info("TerdutDeadmanSwitch applied", "name", sw.Name, "switchID", sw.Status.SwitchID)
|
||||
|
||||
return ctrl.Result{RequeueAfter: resyncInterval}, nil
|
||||
}
|
||||
|
||||
// createOrAdoptDeadmanSwitch implements this resource's own idempotent-
|
||||
// create shape (DESIGN.md §4.4, §5): there's no unique-name constraint
|
||||
// server-side to 409 on, so this lists first and matches by name (the
|
||||
// server's own derived name, when spec.name is empty) rather than adopting
|
||||
// after a conflict the API would never actually raise.
|
||||
func (r *TerdutDeadmanSwitchReconciler) createOrAdoptDeadmanSwitch(
|
||||
ctx context.Context, sw *terdutv1alpha1.TerdutDeadmanSwitch, tc *tdclient.Client,
|
||||
teamID int64, timeout time.Duration, severity string,
|
||||
) error {
|
||||
existing, err := tc.ListDeadmanSwitches(ctx, teamID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("GET /api/teams/%d/deadman/switches: %w", teamID, err)
|
||||
}
|
||||
if sw.Spec.Name != "" {
|
||||
for _, s := range existing {
|
||||
if s.Name == sw.Spec.Name {
|
||||
sw.Status.SwitchID = s.ID
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
created, err := tc.CreateDeadmanSwitch(ctx, teamID, sw.Spec.Name, sw.Spec.Matcher, int64(timeout.Seconds()), severity)
|
||||
if err != nil {
|
||||
return fmt.Errorf("POST /api/teams/%d/deadman/switches: %w", teamID, err)
|
||||
}
|
||||
sw.Status.SwitchID = created.ID
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *TerdutDeadmanSwitchReconciler) setDeadmanNotReady(
|
||||
ctx context.Context, sw *terdutv1alpha1.TerdutDeadmanSwitch, reason, message string, d time.Duration,
|
||||
) (ctrl.Result, error) {
|
||||
meta.SetStatusCondition(&sw.Status.Conditions, metav1.Condition{
|
||||
Type: terdutv1alpha1.ConditionReady,
|
||||
Status: metav1.ConditionFalse,
|
||||
Reason: reason,
|
||||
Message: message,
|
||||
})
|
||||
sw.Status.ObservedGeneration = sw.Generation
|
||||
if err := r.Status().Update(ctx, sw); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
if r.Recorder != nil {
|
||||
r.Recorder.Eventf(sw, nil, corev1.EventTypeWarning, reason, reason, message)
|
||||
}
|
||||
return ctrl.Result{RequeueAfter: d}, nil
|
||||
}
|
||||
|
||||
// reconcileDeadmanDelete calls the real DELETE this resource actually has
|
||||
// (unlike TerdutEscalationRule) if the team is still resolvable and a
|
||||
// switch was ever created, then removes the finalizer unconditionally.
|
||||
func (r *TerdutDeadmanSwitchReconciler) reconcileDeadmanDelete(
|
||||
ctx context.Context, sw *terdutv1alpha1.TerdutDeadmanSwitch, newClient func(string) *tdclient.Client,
|
||||
) (ctrl.Result, error) {
|
||||
if !controllerutil.ContainsFinalizer(sw, deadmanFinalizerName) {
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
|
||||
if sw.Status.SwitchID != 0 {
|
||||
if team, tc, resolveErr := resolveTeamAndClient(
|
||||
ctx, r.Client, r.OperatorNamespace, sw.Namespace, sw.Spec.TeamRef, newClient,
|
||||
); resolveErr == nil {
|
||||
if err := tc.DeleteDeadmanSwitch(ctx, team.Status.TeamID, sw.Status.SwitchID); err != nil {
|
||||
if r.Recorder != nil {
|
||||
r.Recorder.Eventf(sw, nil, corev1.EventTypeWarning, "DeleteFailed", "DeleteFailed", err.Error())
|
||||
}
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
controllerutil.RemoveFinalizer(sw, deadmanFinalizerName)
|
||||
return ctrl.Result{}, r.Update(ctx, sw)
|
||||
}
|
||||
|
||||
// SetupWithManager sets up the controller with the Manager.
|
||||
func (r *TerdutDeadmanSwitchReconciler) SetupWithManager(mgr ctrl.Manager) error {
|
||||
if r.NewClient == nil {
|
||||
r.NewClient = tdclient.New
|
||||
}
|
||||
if r.Recorder == nil {
|
||||
r.Recorder = mgr.GetEventRecorder("terdutdeadmanswitch-controller")
|
||||
}
|
||||
return ctrl.NewControllerManagedBy(mgr).
|
||||
For(&terdutv1alpha1.TerdutDeadmanSwitch{}).
|
||||
Named("terdutdeadmanswitch").
|
||||
Complete(r)
|
||||
}
|
||||
@@ -1,213 +0,0 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http/httptest"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"k8s.io/apimachinery/pkg/api/meta"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
||||
|
||||
terdutv1alpha1 "git.ryuvia.com/niklas/terdut-operator/api/v1alpha1"
|
||||
"git.ryuvia.com/niklas/terdut-operator/internal/tdclient"
|
||||
)
|
||||
|
||||
var _ = Describe("TerdutDeadmanSwitch Controller", func() {
|
||||
const operatorNamespace = "default"
|
||||
|
||||
var (
|
||||
reconciler *TerdutDeadmanSwitchReconciler
|
||||
fake *fakeTerdutServer
|
||||
fakeSrv *httptest.Server
|
||||
srv *terdutv1alpha1.TerdutServer
|
||||
team *terdutv1alpha1.TerdutTeam
|
||||
swName string
|
||||
swKey types.NamespacedName
|
||||
)
|
||||
|
||||
BeforeEach(func(ctx SpecContext) {
|
||||
fake, fakeSrv = newFakeTerdutServer()
|
||||
DeferCleanup(fakeSrv.Close)
|
||||
|
||||
srv = bootstrapReadyTerdutServer(ctx, uniqueName("dmserver"), fakeSrv.URL)
|
||||
team = readyTerdutTeam(ctx, operatorNamespace, uniqueName("dmteam"), srv, fakeSrv.URL)
|
||||
|
||||
reconciler = &TerdutDeadmanSwitchReconciler{
|
||||
Client: k8sClient,
|
||||
Scheme: k8sClient.Scheme(),
|
||||
OperatorNamespace: operatorNamespace,
|
||||
NewClient: func(string) *tdclient.Client { return tdclient.New(fakeSrv.URL) },
|
||||
}
|
||||
swName = uniqueName("switch")
|
||||
swKey = types.NamespacedName{Name: swName, Namespace: operatorNamespace}
|
||||
})
|
||||
|
||||
AfterEach(func(ctx SpecContext) {
|
||||
sw := &terdutv1alpha1.TerdutDeadmanSwitch{}
|
||||
if err := k8sClient.Get(ctx, swKey, sw); err == nil {
|
||||
sw.Finalizers = nil
|
||||
_ = k8sClient.Update(ctx, sw)
|
||||
_ = k8sClient.Delete(ctx, sw)
|
||||
}
|
||||
|
||||
teamKey := types.NamespacedName{Name: team.Name, Namespace: operatorNamespace}
|
||||
if err := k8sClient.Get(ctx, teamKey, team); err == nil {
|
||||
team.Finalizers = nil
|
||||
_ = k8sClient.Update(ctx, team)
|
||||
_ = k8sClient.Delete(ctx, team)
|
||||
}
|
||||
|
||||
srvKey := types.NamespacedName{Name: srv.Name, Namespace: operatorNamespace}
|
||||
if err := k8sClient.Get(ctx, srvKey, srv); err == nil {
|
||||
srv.Finalizers = nil
|
||||
_ = k8sClient.Update(ctx, srv)
|
||||
_ = k8sClient.Delete(ctx, srv)
|
||||
}
|
||||
})
|
||||
|
||||
createSwitch := func(ctx context.Context, teamRef terdutv1alpha1.TerdutTeamRef, name, matcher, timeout string) {
|
||||
sw := &terdutv1alpha1.TerdutDeadmanSwitch{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: swName, Namespace: operatorNamespace},
|
||||
Spec: terdutv1alpha1.TerdutDeadmanSwitchSpec{
|
||||
TeamRef: teamRef,
|
||||
Name: name,
|
||||
Matcher: matcher,
|
||||
Timeout: timeout,
|
||||
},
|
||||
}
|
||||
Expect(k8sClient.Create(ctx, sw)).To(Succeed())
|
||||
}
|
||||
|
||||
reconcileOnce := func(ctx context.Context) {
|
||||
_, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: swKey})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
}
|
||||
|
||||
readyCondition := func(ctx context.Context) metav1.Condition {
|
||||
sw := &terdutv1alpha1.TerdutDeadmanSwitch{}
|
||||
Expect(k8sClient.Get(ctx, swKey, sw)).To(Succeed())
|
||||
c := meta.FindStatusCondition(sw.Status.Conditions, terdutv1alpha1.ConditionReady)
|
||||
Expect(c).NotTo(BeNil())
|
||||
return *c
|
||||
}
|
||||
|
||||
sameTeamRef := func() terdutv1alpha1.TerdutTeamRef {
|
||||
return terdutv1alpha1.TerdutTeamRef{Name: team.Name}
|
||||
}
|
||||
|
||||
Describe("the happy path", func() {
|
||||
It("creates the switch server-side", func(ctx SpecContext) {
|
||||
createSwitch(ctx, sameTeamRef(), "heartbeat", "alertname=Watchdog", "15m")
|
||||
reconcileOnce(ctx) // finalizer
|
||||
reconcileOnce(ctx) // create
|
||||
|
||||
cond := readyCondition(ctx)
|
||||
Expect(cond.Status).To(Equal(metav1.ConditionTrue))
|
||||
Expect(cond.Reason).To(Equal(terdutv1alpha1.ReasonChildAdopted))
|
||||
|
||||
sw := &terdutv1alpha1.TerdutDeadmanSwitch{}
|
||||
Expect(k8sClient.Get(ctx, swKey, sw)).To(Succeed())
|
||||
Expect(sw.Status.SwitchID).NotTo(BeZero())
|
||||
|
||||
created, ok := fake.switches[team.Status.TeamID][sw.Status.SwitchID]
|
||||
Expect(ok).To(BeTrue())
|
||||
Expect(created.Matcher).To(Equal("alertname=Watchdog"))
|
||||
Expect(created.TimeoutSeconds).To(Equal(int64(900)))
|
||||
Expect(created.Severity).To(Equal("critical")) // kubebuilder default
|
||||
})
|
||||
})
|
||||
|
||||
Describe("list-and-match-by-name adoption", func() {
|
||||
It("adopts an already-created switch instead of creating a duplicate", func(ctx SpecContext) {
|
||||
// Simulates a prior, interrupted reconcile that got as far as
|
||||
// POSTing the switch -- no 409 signal exists for this resource
|
||||
// (DESIGN.md §4.4), so the recovery path is GET-list-and-match,
|
||||
// not adopt-on-409.
|
||||
fake.nextSwitchID = 1
|
||||
fake.switches[team.Status.TeamID] = map[int64]tdclient.DeadmanSwitch{
|
||||
1: {ID: 1, Name: "heartbeat", Matcher: "alertname=Watchdog", TimeoutSeconds: 900, Severity: "critical"},
|
||||
}
|
||||
|
||||
createSwitch(ctx, sameTeamRef(), "heartbeat", "alertname=Watchdog", "15m")
|
||||
reconcileOnce(ctx) // finalizer
|
||||
reconcileOnce(ctx)
|
||||
|
||||
sw := &terdutv1alpha1.TerdutDeadmanSwitch{}
|
||||
Expect(k8sClient.Get(ctx, swKey, sw)).To(Succeed())
|
||||
Expect(sw.Status.SwitchID).To(Equal(int64(1)))
|
||||
Expect(fake.switches[team.Status.TeamID]).To(HaveLen(1), "should not have created a second switch")
|
||||
})
|
||||
})
|
||||
|
||||
Describe("update-in-place on spec drift", func() {
|
||||
It("PUTs the new spec rather than creating a second switch", func(ctx SpecContext) {
|
||||
createSwitch(ctx, sameTeamRef(), "heartbeat", "alertname=Watchdog", "15m")
|
||||
reconcileOnce(ctx) // finalizer
|
||||
reconcileOnce(ctx)
|
||||
Expect(readyCondition(ctx).Status).To(Equal(metav1.ConditionTrue))
|
||||
|
||||
sw := &terdutv1alpha1.TerdutDeadmanSwitch{}
|
||||
Expect(k8sClient.Get(ctx, swKey, sw)).To(Succeed())
|
||||
switchID := sw.Status.SwitchID
|
||||
|
||||
sw.Spec.Timeout = "30m"
|
||||
Expect(k8sClient.Update(ctx, sw)).To(Succeed())
|
||||
reconcileOnce(ctx)
|
||||
|
||||
Expect(fake.switches[team.Status.TeamID]).To(HaveLen(1), "update-in-place, not a second switch")
|
||||
Expect(fake.switches[team.Status.TeamID][switchID].TimeoutSeconds).To(Equal(int64(1800)))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("waiting on the referenced TerdutTeam", func() {
|
||||
It("reports TeamRefNotFound when the TerdutTeam doesn't exist", func(ctx SpecContext) {
|
||||
createSwitch(ctx, terdutv1alpha1.TerdutTeamRef{Name: testRefNotFoundName}, "heartbeat", "alertname=Watchdog", "15m")
|
||||
reconcileOnce(ctx) // finalizer
|
||||
reconcileOnce(ctx)
|
||||
|
||||
Expect(readyCondition(ctx).Reason).To(Equal(terdutv1alpha1.ReasonTeamRefNotFound))
|
||||
})
|
||||
|
||||
It("reports WaitingForTeam when the TerdutTeam exists but isn't Ready yet", func(ctx SpecContext) {
|
||||
unreadyName := uniqueName("dmteam-unready")
|
||||
unready := &terdutv1alpha1.TerdutTeam{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: unreadyName, Namespace: operatorNamespace},
|
||||
Spec: terdutv1alpha1.TerdutTeamSpec{
|
||||
ServerRef: terdutv1alpha1.TerdutServerRef{Name: srv.Name},
|
||||
DisplayName: testUnreadyDisplayName,
|
||||
},
|
||||
}
|
||||
Expect(k8sClient.Create(ctx, unready)).To(Succeed())
|
||||
DeferCleanup(func() { _ = k8sClient.Delete(ctx, unready) })
|
||||
|
||||
createSwitch(ctx, terdutv1alpha1.TerdutTeamRef{Name: unreadyName}, "heartbeat", "alertname=Watchdog", "15m")
|
||||
reconcileOnce(ctx) // finalizer
|
||||
reconcileOnce(ctx)
|
||||
|
||||
Expect(readyCondition(ctx).Reason).To(Equal(terdutv1alpha1.ReasonWaitingForTeam))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("deletion", func() {
|
||||
It("deletes the switch server-side (the real DELETE this resource has) and removes the finalizer", func(ctx SpecContext) {
|
||||
createSwitch(ctx, sameTeamRef(), "heartbeat", "alertname=Watchdog", "15m")
|
||||
reconcileOnce(ctx)
|
||||
reconcileOnce(ctx)
|
||||
Expect(readyCondition(ctx).Status).To(Equal(metav1.ConditionTrue))
|
||||
|
||||
sw := &terdutv1alpha1.TerdutDeadmanSwitch{}
|
||||
Expect(k8sClient.Get(ctx, swKey, sw)).To(Succeed())
|
||||
switchID := sw.Status.SwitchID
|
||||
|
||||
Expect(k8sClient.Delete(ctx, sw)).To(Succeed())
|
||||
reconcileOnce(ctx) // runs the finalizer
|
||||
|
||||
Expect(fake.switchDelete[switchID]).To(BeTrue())
|
||||
err := k8sClient.Get(ctx, swKey, sw)
|
||||
Expect(err).To(HaveOccurred(), "the TerdutDeadmanSwitch itself should be gone once the finalizer clears")
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -1,217 +0,0 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/api/meta"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
|
||||
logf "sigs.k8s.io/controller-runtime/pkg/log"
|
||||
"sigs.k8s.io/controller-runtime/pkg/recorder"
|
||||
|
||||
terdutv1alpha1 "git.ryuvia.com/niklas/terdut-operator/api/v1alpha1"
|
||||
"git.ryuvia.com/niklas/terdut-operator/internal/tdclient"
|
||||
)
|
||||
|
||||
// escalationFinalizerName exists for the one undo PUT /api/teams/{teamID}/escalation
|
||||
// supports, on delete: an empty policy (DESIGN.md §5's general finalizer
|
||||
// rule expects a server-side counterpart to be undone, but this resource's
|
||||
// API is GET/PUT-only, with no DELETE at all -- a zeroed PUT is the closest
|
||||
// equivalent terdut-server itself recognizes as "no policy"
|
||||
// (escalationPolicy.configured(), confirmed against source: "a policy row
|
||||
// with no levels is the same as no policy").
|
||||
const escalationFinalizerName = "terdut.ryuvia.com/terdutescalationrule"
|
||||
|
||||
// TerdutEscalationRuleReconciler reconciles a TerdutEscalationRule object.
|
||||
type TerdutEscalationRuleReconciler struct {
|
||||
client.Client
|
||||
Scheme *runtime.Scheme
|
||||
|
||||
OperatorNamespace string
|
||||
Recorder recorder.EventRecorder
|
||||
NewClient func(endpoint string) *tdclient.Client
|
||||
}
|
||||
|
||||
// +kubebuilder:rbac:groups=terdut.ryuvia.com,resources=terdutescalationrules,verbs=get;list;watch;create;update;patch;delete
|
||||
// +kubebuilder:rbac:groups=terdut.ryuvia.com,resources=terdutescalationrules/status,verbs=get;update;patch
|
||||
// +kubebuilder:rbac:groups=terdut.ryuvia.com,resources=terdutescalationrules/finalizers,verbs=update
|
||||
// +kubebuilder:rbac:groups=terdut.ryuvia.com,resources=terdutteams,verbs=get;list;watch
|
||||
// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch
|
||||
// +kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=create;patch
|
||||
|
||||
func (r *TerdutEscalationRuleReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
|
||||
log := logf.FromContext(ctx)
|
||||
|
||||
var rule terdutv1alpha1.TerdutEscalationRule
|
||||
if err := r.Get(ctx, req.NamespacedName, &rule); err != nil {
|
||||
if apierrors.IsNotFound(err) {
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
|
||||
newClient := r.NewClient
|
||||
if newClient == nil {
|
||||
newClient = tdclient.New
|
||||
}
|
||||
|
||||
if !rule.DeletionTimestamp.IsZero() {
|
||||
return r.reconcileEscalationDelete(ctx, &rule, newClient)
|
||||
}
|
||||
|
||||
if !controllerutil.ContainsFinalizer(&rule, escalationFinalizerName) {
|
||||
controllerutil.AddFinalizer(&rule, escalationFinalizerName)
|
||||
if err := r.Update(ctx, &rule); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
|
||||
team, tc, resolveErr := resolveTeamAndClient(ctx, r.Client, r.OperatorNamespace, rule.Namespace, rule.Spec.TeamRef, newClient)
|
||||
if resolveErr != nil {
|
||||
return r.setEscalationNotReady(ctx, &rule, resolveErr.reason, resolveErr.message, waitInterval)
|
||||
}
|
||||
|
||||
body, unknownUser, err := buildEscalationRequest(ctx, tc, rule.Spec)
|
||||
if err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
if unknownUser != "" {
|
||||
return r.setEscalationNotReady(ctx, &rule, terdutv1alpha1.ReasonUnknownUser,
|
||||
fmt.Sprintf("username %q does not resolve to any user", unknownUser), waitInterval)
|
||||
}
|
||||
|
||||
if err := tc.SetEscalation(ctx, team.Status.TeamID, body); err != nil {
|
||||
return ctrl.Result{}, fmt.Errorf("PUT /api/teams/%d/escalation: %w", team.Status.TeamID, err)
|
||||
}
|
||||
|
||||
meta.SetStatusCondition(&rule.Status.Conditions, metav1.Condition{
|
||||
Type: terdutv1alpha1.ConditionReady, // "Ready" -- same name, shared across every CRD (DESIGN.md §7)
|
||||
Status: metav1.ConditionTrue,
|
||||
Reason: terdutv1alpha1.ReasonChildAdopted,
|
||||
Message: fmt.Sprintf("escalation policy applied to team %d", team.Status.TeamID),
|
||||
})
|
||||
rule.Status.ObservedGeneration = rule.Generation
|
||||
if err := r.Status().Update(ctx, &rule); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
if r.Recorder != nil {
|
||||
r.Recorder.Eventf(&rule, nil, corev1.EventTypeNormal, terdutv1alpha1.ReasonChildAdopted, terdutv1alpha1.ReasonChildAdopted,
|
||||
"escalation policy applied")
|
||||
}
|
||||
log.Info("TerdutEscalationRule applied", "name", rule.Name, "teamID", team.Status.TeamID)
|
||||
|
||||
return ctrl.Result{RequeueAfter: resyncInterval}, nil
|
||||
}
|
||||
|
||||
// buildEscalationRequest resolves every "user" target's username to a
|
||||
// user_id (DESIGN.md §4.3) and translates spec into the wire shape
|
||||
// SetEscalation sends. Returns the first unresolvable username, if any,
|
||||
// distinct from a plain error: that's an expected, reportable condition
|
||||
// (ReasonUnknownUser), not a reconcile failure.
|
||||
func buildEscalationRequest(
|
||||
ctx context.Context, tc *tdclient.Client, spec terdutv1alpha1.TerdutEscalationRuleSpec,
|
||||
) (tdclient.SetEscalationRequest, string, error) {
|
||||
levels := make([]tdclient.EscalationLevelRequest, len(spec.Levels))
|
||||
for i, lvl := range spec.Levels {
|
||||
timeout, err := time.ParseDuration(lvl.Timeout)
|
||||
if err != nil {
|
||||
return tdclient.SetEscalationRequest{}, "", fmt.Errorf("spec.levels[%d].timeout %q: %w", i, lvl.Timeout, err)
|
||||
}
|
||||
|
||||
targets := make([]tdclient.EscalationTargetRequest, len(lvl.Targets))
|
||||
for j, t := range lvl.Targets {
|
||||
if t.Kind == terdutv1alpha1.EscalationTargetOncall {
|
||||
targets[j] = tdclient.EscalationTargetRequest{Kind: string(terdutv1alpha1.EscalationTargetOncall)}
|
||||
continue
|
||||
}
|
||||
user, err := tc.GetUserByUsername(ctx, t.Username)
|
||||
if err != nil {
|
||||
return tdclient.SetEscalationRequest{}, "", fmt.Errorf("GET /api/users (resolving %q): %w", t.Username, err)
|
||||
}
|
||||
if user == nil {
|
||||
return tdclient.SetEscalationRequest{}, t.Username, nil
|
||||
}
|
||||
targets[j] = tdclient.EscalationTargetRequest{Kind: string(terdutv1alpha1.EscalationTargetUser), UserID: &user.ID}
|
||||
}
|
||||
|
||||
levels[i] = tdclient.EscalationLevelRequest{
|
||||
Position: int64(i + 1),
|
||||
TimeoutSeconds: int64(timeout.Seconds()),
|
||||
Targets: targets,
|
||||
}
|
||||
}
|
||||
|
||||
return tdclient.SetEscalationRequest{
|
||||
RepeatCount: spec.RepeatCount,
|
||||
FallbackTopic: spec.FallbackTopic,
|
||||
Levels: levels,
|
||||
}, "", nil
|
||||
}
|
||||
|
||||
func (r *TerdutEscalationRuleReconciler) setEscalationNotReady(
|
||||
ctx context.Context, rule *terdutv1alpha1.TerdutEscalationRule, reason, message string, d time.Duration,
|
||||
) (ctrl.Result, error) {
|
||||
meta.SetStatusCondition(&rule.Status.Conditions, metav1.Condition{
|
||||
Type: terdutv1alpha1.ConditionReady,
|
||||
Status: metav1.ConditionFalse,
|
||||
Reason: reason,
|
||||
Message: message,
|
||||
})
|
||||
rule.Status.ObservedGeneration = rule.Generation
|
||||
if err := r.Status().Update(ctx, rule); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
if r.Recorder != nil {
|
||||
r.Recorder.Eventf(rule, nil, corev1.EventTypeWarning, reason, reason, message)
|
||||
}
|
||||
return ctrl.Result{RequeueAfter: d}, nil
|
||||
}
|
||||
|
||||
// reconcileEscalationDelete PUTs an empty policy (this resource's only
|
||||
// available "undo", per this file's own const comment) if the team is
|
||||
// still resolvable, then removes the finalizer unconditionally -- same
|
||||
// "the parent's probably going away too" reasoning TerdutTeam's own delete
|
||||
// path uses for a TerdutServer that's gone.
|
||||
func (r *TerdutEscalationRuleReconciler) reconcileEscalationDelete(
|
||||
ctx context.Context, rule *terdutv1alpha1.TerdutEscalationRule, newClient func(string) *tdclient.Client,
|
||||
) (ctrl.Result, error) {
|
||||
if !controllerutil.ContainsFinalizer(rule, escalationFinalizerName) {
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
|
||||
if team, tc, resolveErr := resolveTeamAndClient(
|
||||
ctx, r.Client, r.OperatorNamespace, rule.Namespace, rule.Spec.TeamRef, newClient,
|
||||
); resolveErr == nil {
|
||||
if err := tc.SetEscalation(ctx, team.Status.TeamID, tdclient.SetEscalationRequest{Levels: []tdclient.EscalationLevelRequest{}}); err != nil {
|
||||
if r.Recorder != nil {
|
||||
r.Recorder.Eventf(rule, nil, corev1.EventTypeWarning, "DeleteFailed", "DeleteFailed", err.Error())
|
||||
}
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
}
|
||||
|
||||
controllerutil.RemoveFinalizer(rule, escalationFinalizerName)
|
||||
return ctrl.Result{}, r.Update(ctx, rule)
|
||||
}
|
||||
|
||||
// SetupWithManager sets up the controller with the Manager.
|
||||
func (r *TerdutEscalationRuleReconciler) SetupWithManager(mgr ctrl.Manager) error {
|
||||
if r.NewClient == nil {
|
||||
r.NewClient = tdclient.New
|
||||
}
|
||||
if r.Recorder == nil {
|
||||
r.Recorder = mgr.GetEventRecorder("terdutescalationrule-controller")
|
||||
}
|
||||
return ctrl.NewControllerManagedBy(mgr).
|
||||
For(&terdutv1alpha1.TerdutEscalationRule{}).
|
||||
Named("terdutescalationrule").
|
||||
Complete(r)
|
||||
}
|
||||
@@ -1,206 +0,0 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http/httptest"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"k8s.io/apimachinery/pkg/api/meta"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
||||
|
||||
terdutv1alpha1 "git.ryuvia.com/niklas/terdut-operator/api/v1alpha1"
|
||||
"git.ryuvia.com/niklas/terdut-operator/internal/tdclient"
|
||||
)
|
||||
|
||||
var _ = Describe("TerdutEscalationRule Controller", func() {
|
||||
const operatorNamespace = "default"
|
||||
|
||||
var (
|
||||
reconciler *TerdutEscalationRuleReconciler
|
||||
fake *fakeTerdutServer
|
||||
fakeSrv *httptest.Server
|
||||
srv *terdutv1alpha1.TerdutServer
|
||||
team *terdutv1alpha1.TerdutTeam
|
||||
ruleName string
|
||||
ruleKey types.NamespacedName
|
||||
)
|
||||
|
||||
BeforeEach(func(ctx SpecContext) {
|
||||
fake, fakeSrv = newFakeTerdutServer()
|
||||
DeferCleanup(fakeSrv.Close)
|
||||
|
||||
srv = bootstrapReadyTerdutServer(ctx, uniqueName("erserver"), fakeSrv.URL)
|
||||
team = readyTerdutTeam(ctx, operatorNamespace, uniqueName("erteam"), srv, fakeSrv.URL)
|
||||
|
||||
reconciler = &TerdutEscalationRuleReconciler{
|
||||
Client: k8sClient,
|
||||
Scheme: k8sClient.Scheme(),
|
||||
OperatorNamespace: operatorNamespace,
|
||||
NewClient: func(string) *tdclient.Client { return tdclient.New(fakeSrv.URL) },
|
||||
}
|
||||
ruleName = uniqueName("escalation")
|
||||
ruleKey = types.NamespacedName{Name: ruleName, Namespace: operatorNamespace}
|
||||
})
|
||||
|
||||
AfterEach(func(ctx SpecContext) {
|
||||
rule := &terdutv1alpha1.TerdutEscalationRule{}
|
||||
if err := k8sClient.Get(ctx, ruleKey, rule); err == nil {
|
||||
rule.Finalizers = nil
|
||||
_ = k8sClient.Update(ctx, rule)
|
||||
_ = k8sClient.Delete(ctx, rule)
|
||||
}
|
||||
|
||||
// team/srv carry their own finalizers from readyTerdutTeam/
|
||||
// bootstrapReadyTerdutServer -- clear them directly the same way
|
||||
// terdutteam_controller_test.go's own AfterEach does, rather than
|
||||
// relying on either reconciler to ever run again here.
|
||||
teamKey := types.NamespacedName{Name: team.Name, Namespace: operatorNamespace}
|
||||
if err := k8sClient.Get(ctx, teamKey, team); err == nil {
|
||||
team.Finalizers = nil
|
||||
_ = k8sClient.Update(ctx, team)
|
||||
_ = k8sClient.Delete(ctx, team)
|
||||
}
|
||||
|
||||
srvKey := types.NamespacedName{Name: srv.Name, Namespace: operatorNamespace}
|
||||
if err := k8sClient.Get(ctx, srvKey, srv); err == nil {
|
||||
srv.Finalizers = nil
|
||||
_ = k8sClient.Update(ctx, srv)
|
||||
_ = k8sClient.Delete(ctx, srv)
|
||||
}
|
||||
})
|
||||
|
||||
createRule := func(ctx context.Context, teamRef terdutv1alpha1.TerdutTeamRef, levels []terdutv1alpha1.EscalationLevel) {
|
||||
rule := &terdutv1alpha1.TerdutEscalationRule{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: ruleName, Namespace: operatorNamespace},
|
||||
Spec: terdutv1alpha1.TerdutEscalationRuleSpec{
|
||||
TeamRef: teamRef,
|
||||
Levels: levels,
|
||||
},
|
||||
}
|
||||
Expect(k8sClient.Create(ctx, rule)).To(Succeed())
|
||||
}
|
||||
|
||||
reconcileOnce := func(ctx context.Context) {
|
||||
_, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: ruleKey})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
}
|
||||
|
||||
readyCondition := func(ctx context.Context) metav1.Condition {
|
||||
rule := &terdutv1alpha1.TerdutEscalationRule{}
|
||||
Expect(k8sClient.Get(ctx, ruleKey, rule)).To(Succeed())
|
||||
c := meta.FindStatusCondition(rule.Status.Conditions, terdutv1alpha1.ConditionReady)
|
||||
Expect(c).NotTo(BeNil())
|
||||
return *c
|
||||
}
|
||||
|
||||
sameTeamRef := func() terdutv1alpha1.TerdutTeamRef {
|
||||
return terdutv1alpha1.TerdutTeamRef{Name: team.Name}
|
||||
}
|
||||
|
||||
Describe("the happy path", func() {
|
||||
It("resolves usernames and applies the escalation policy", func(ctx SpecContext) {
|
||||
fake.seedUser("alice")
|
||||
createRule(ctx, sameTeamRef(), []terdutv1alpha1.EscalationLevel{
|
||||
{Timeout: "5m", Targets: []terdutv1alpha1.EscalationTarget{
|
||||
{Kind: terdutv1alpha1.EscalationTargetUser, Username: "alice"},
|
||||
}},
|
||||
{Timeout: "10m", Targets: []terdutv1alpha1.EscalationTarget{
|
||||
{Kind: terdutv1alpha1.EscalationTargetOncall},
|
||||
}},
|
||||
})
|
||||
reconcileOnce(ctx) // finalizer
|
||||
reconcileOnce(ctx) // resolve + apply
|
||||
|
||||
cond := readyCondition(ctx)
|
||||
Expect(cond.Status).To(Equal(metav1.ConditionTrue))
|
||||
Expect(cond.Reason).To(Equal(terdutv1alpha1.ReasonChildAdopted))
|
||||
|
||||
applied, ok := fake.escalation[team.Status.TeamID]
|
||||
Expect(ok).To(BeTrue())
|
||||
Expect(applied.Levels).To(HaveLen(2))
|
||||
Expect(applied.Levels[0].Targets[0].Kind).To(Equal("user"))
|
||||
Expect(applied.Levels[0].Targets[0].UserID).NotTo(BeNil())
|
||||
Expect(*applied.Levels[0].Targets[0].UserID).To(Equal(fake.users["alice"]))
|
||||
Expect(applied.Levels[1].Targets[0].Kind).To(Equal("oncall"))
|
||||
Expect(applied.Levels[1].Targets[0].UserID).To(BeNil())
|
||||
})
|
||||
})
|
||||
|
||||
Describe("an unresolvable username", func() {
|
||||
It("reports UnknownUser and never calls PUT /api/teams/{id}/escalation", func(ctx SpecContext) {
|
||||
createRule(ctx, sameTeamRef(), []terdutv1alpha1.EscalationLevel{
|
||||
{Timeout: "5m", Targets: []terdutv1alpha1.EscalationTarget{
|
||||
{Kind: terdutv1alpha1.EscalationTargetUser, Username: "ghost"},
|
||||
}},
|
||||
})
|
||||
reconcileOnce(ctx) // finalizer
|
||||
reconcileOnce(ctx)
|
||||
|
||||
cond := readyCondition(ctx)
|
||||
Expect(cond.Status).To(Equal(metav1.ConditionFalse))
|
||||
Expect(cond.Reason).To(Equal(terdutv1alpha1.ReasonUnknownUser))
|
||||
_, applied := fake.escalation[team.Status.TeamID]
|
||||
Expect(applied).To(BeFalse())
|
||||
})
|
||||
})
|
||||
|
||||
Describe("waiting on the referenced TerdutTeam", func() {
|
||||
It("reports TeamRefNotFound when the TerdutTeam doesn't exist", func(ctx SpecContext) {
|
||||
createRule(ctx, terdutv1alpha1.TerdutTeamRef{Name: testRefNotFoundName}, []terdutv1alpha1.EscalationLevel{
|
||||
{Timeout: "5m", Targets: []terdutv1alpha1.EscalationTarget{{Kind: terdutv1alpha1.EscalationTargetOncall}}},
|
||||
})
|
||||
reconcileOnce(ctx) // finalizer
|
||||
reconcileOnce(ctx)
|
||||
|
||||
Expect(readyCondition(ctx).Reason).To(Equal(terdutv1alpha1.ReasonTeamRefNotFound))
|
||||
})
|
||||
|
||||
It("reports WaitingForTeam when the TerdutTeam exists but isn't Ready yet", func(ctx SpecContext) {
|
||||
unreadyName := uniqueName("erteam-unready")
|
||||
unready := &terdutv1alpha1.TerdutTeam{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: unreadyName, Namespace: operatorNamespace},
|
||||
Spec: terdutv1alpha1.TerdutTeamSpec{
|
||||
ServerRef: terdutv1alpha1.TerdutServerRef{Name: srv.Name},
|
||||
DisplayName: testUnreadyDisplayName,
|
||||
},
|
||||
}
|
||||
Expect(k8sClient.Create(ctx, unready)).To(Succeed())
|
||||
DeferCleanup(func() { _ = k8sClient.Delete(ctx, unready) })
|
||||
|
||||
createRule(ctx, terdutv1alpha1.TerdutTeamRef{Name: unreadyName}, []terdutv1alpha1.EscalationLevel{
|
||||
{Timeout: "5m", Targets: []terdutv1alpha1.EscalationTarget{{Kind: terdutv1alpha1.EscalationTargetOncall}}},
|
||||
})
|
||||
reconcileOnce(ctx) // finalizer
|
||||
reconcileOnce(ctx)
|
||||
|
||||
Expect(readyCondition(ctx).Reason).To(Equal(terdutv1alpha1.ReasonWaitingForTeam))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("deletion", func() {
|
||||
It("PUTs an empty policy (this resource's only available undo) and removes the finalizer", func(ctx SpecContext) {
|
||||
fake.seedUser("alice")
|
||||
createRule(ctx, sameTeamRef(), []terdutv1alpha1.EscalationLevel{
|
||||
{Timeout: "5m", Targets: []terdutv1alpha1.EscalationTarget{
|
||||
{Kind: terdutv1alpha1.EscalationTargetUser, Username: "alice"},
|
||||
}},
|
||||
})
|
||||
reconcileOnce(ctx)
|
||||
reconcileOnce(ctx)
|
||||
Expect(readyCondition(ctx).Status).To(Equal(metav1.ConditionTrue))
|
||||
Expect(fake.escalation[team.Status.TeamID].Levels).To(HaveLen(1))
|
||||
|
||||
rule := &terdutv1alpha1.TerdutEscalationRule{}
|
||||
Expect(k8sClient.Get(ctx, ruleKey, rule)).To(Succeed())
|
||||
Expect(k8sClient.Delete(ctx, rule)).To(Succeed())
|
||||
reconcileOnce(ctx) // runs the finalizer
|
||||
|
||||
Expect(fake.escalation[team.Status.TeamID].Levels).To(BeEmpty())
|
||||
err := k8sClient.Get(ctx, ruleKey, rule)
|
||||
Expect(err).To(HaveOccurred(), "the TerdutEscalationRule itself should be gone once the finalizer clears")
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -1,196 +0,0 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
terdutv1alpha1 "git.ryuvia.com/niklas/terdut-operator/api/v1alpha1"
|
||||
"git.ryuvia.com/niklas/terdut-operator/internal/tdclient"
|
||||
)
|
||||
|
||||
// bootstrapStateLostError is DESIGN.md §6's one genuinely pathological
|
||||
// case: the server's database is already bootstrapped, and no credential for
|
||||
// it survives here -- a checkpointed admin key was used and then lost before
|
||||
// the lasting credential could be persisted, or the instance credential
|
||||
// Secret was deleted (spec.credentials.deletionPolicy: Delete, or by hand).
|
||||
// Distinct from a plain error so Reconcile can route it to a Ready: False
|
||||
// condition rather than treating it as a transient reconcile failure: no
|
||||
// retry can fix it.
|
||||
type bootstrapStateLostError struct {
|
||||
detail string
|
||||
secretName string // the instance credential Secret that would have fixed it
|
||||
}
|
||||
|
||||
func (e *bootstrapStateLostError) Error() string {
|
||||
return fmt.Sprintf(
|
||||
"server reports already bootstrapped, but there is no credential for it here: %s. "+
|
||||
"The operator cannot mint one on its own, and deleting and recreating this TerdutServer does "+
|
||||
"not clear the database. Restore Secret %q in the operator's namespace if you have a copy, "+
|
||||
"or reset the server's database and recreate this TerdutServer", e.detail, e.secretName)
|
||||
}
|
||||
|
||||
// reconcileBootstrap implements DESIGN.md §6 point 1's self-registration
|
||||
// flow, checkpointed against the two real crash windows in it rather than
|
||||
// leaving them as theoretical gaps. Only called once
|
||||
// srv.Status.CredentialsSecretRef is nil and the Deployment has a ready
|
||||
// replica.
|
||||
func (r *TerdutServerReconciler) reconcileBootstrap(ctx context.Context, srv *terdutv1alpha1.TerdutServer) error {
|
||||
// A TerdutServer recreated against a database that is already
|
||||
// bootstrapped: its instance credential may still be here (the default
|
||||
// deletion policy keeps it), and then there is nothing to bootstrap.
|
||||
adopted, err := r.adoptRetainedCredentials(ctx, srv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if adopted {
|
||||
return nil
|
||||
}
|
||||
|
||||
adminKey, err := r.getOrCreateCheckpointedAdminKey(ctx, srv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
bc := r.NewClient(serviceURL(srv)).WithToken(adminKey)
|
||||
instanceKey, err := r.getOrMintInstanceServiceAccountKey(ctx, bc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
credsName := credentialsSecretName(srv)
|
||||
if err := writeOperatorSecret(ctx, r.Client, r.OperatorNamespace, credsName, instanceKey); err != nil {
|
||||
return err
|
||||
}
|
||||
srv.Status.CredentialsSecretRef = &terdutv1alpha1.SecretKeyRef{Name: credsName, Key: credentialsSecretDataKey}
|
||||
|
||||
// Best-effort: the checkpoint has done its job. Leaving it behind on a
|
||||
// delete failure here isn't a correctness problem (the next reconcile
|
||||
// finds status.CredentialsSecretRef already set and never looks at the
|
||||
// checkpoint again) — it would just be an unused Secret sitting around,
|
||||
// cleaned up for real by the finalizer on delete.
|
||||
checkpoint := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: checkpointSecretName(srv), Namespace: r.OperatorNamespace}}
|
||||
_ = r.Delete(ctx, checkpoint)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// adoptRetainedCredentials looks for the instance credential Secret an
|
||||
// earlier TerdutServer of the same name and namespace left behind
|
||||
// (spec.credentials.deletionPolicy: Retain), and adopts it if the server
|
||||
// still accepts its token. Reports whether it did.
|
||||
//
|
||||
// The token is tried before it is trusted: a Secret that outlived a database
|
||||
// reset holds a key the server has never heard of, and adopting that would
|
||||
// make every later call 401. A rejected token (401/403) is not an error here;
|
||||
// it just means there is nothing to adopt, and bootstrap proceeds as it would
|
||||
// for a first install -- which succeeds against a freshly reset database and
|
||||
// replaces the Secret. Anything else (the server unreachable, a 5xx) is
|
||||
// returned for a retry rather than guessed at.
|
||||
func (r *TerdutServerReconciler) adoptRetainedCredentials(ctx context.Context, srv *terdutv1alpha1.TerdutServer) (bool, error) {
|
||||
name := credentialsSecretName(srv)
|
||||
var sec corev1.Secret
|
||||
if err := r.Get(ctx, client.ObjectKey{Namespace: r.OperatorNamespace, Name: name}, &sec); err != nil {
|
||||
if apierrors.IsNotFound(err) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
token := string(sec.Data[credentialsSecretDataKey])
|
||||
if token == "" {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// The operator's own service account is the one thing this credential
|
||||
// is for, so asking the server for it by name checks the token and that
|
||||
// it belongs to that account in the same call.
|
||||
sa, err := r.NewClient(serviceURL(srv)).WithToken(token).GetServiceAccountByName(ctx, serviceAccountName)
|
||||
if err != nil {
|
||||
if statusErr, ok := errors.AsType[*tdclient.StatusError](err); ok &&
|
||||
(statusErr.Code == http.StatusUnauthorized || statusErr.Code == http.StatusForbidden) {
|
||||
return false, nil
|
||||
}
|
||||
return false, fmt.Errorf("checking the retained credential in Secret %q: %w", name, err)
|
||||
}
|
||||
if sa == nil {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
srv.Status.CredentialsSecretRef = &terdutv1alpha1.SecretKeyRef{Name: name, Key: credentialsSecretDataKey}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// getOrCreateCheckpointedAdminKey returns a usable admin key: from the
|
||||
// checkpoint Secret if an earlier, interrupted attempt already got one, or
|
||||
// freshly from /api/bootstrap, immediately checkpointed before it's used
|
||||
// for anything else.
|
||||
func (r *TerdutServerReconciler) getOrCreateCheckpointedAdminKey(ctx context.Context, srv *terdutv1alpha1.TerdutServer) (string, error) {
|
||||
checkpointName := checkpointSecretName(srv)
|
||||
|
||||
var checkpoint corev1.Secret
|
||||
err := r.Get(ctx, client.ObjectKey{Namespace: r.OperatorNamespace, Name: checkpointName}, &checkpoint)
|
||||
switch {
|
||||
case err == nil:
|
||||
return string(checkpoint.Data[credentialsSecretDataKey]), nil
|
||||
case !apierrors.IsNotFound(err):
|
||||
return "", err
|
||||
}
|
||||
|
||||
bc := r.NewClient(serviceURL(srv))
|
||||
result, err := bc.Bootstrap(ctx, bootstrapUsername, bootstrapEmail)
|
||||
if err != nil {
|
||||
if statusErr, ok := errors.AsType[*tdclient.StatusError](err); ok && statusErr.Code == http.StatusForbidden {
|
||||
// §1: this operator is the only thing that ever bootstraps a
|
||||
// server it created, so a 403 here (no checkpoint, no
|
||||
// status.credentialsSecretRef) means a prior reconcile already
|
||||
// won this exact race and its checkpoint was lost afterward --
|
||||
// the one case §6 doesn't try to paper over.
|
||||
return "", &bootstrapStateLostError{
|
||||
detail: "/api/bootstrap returned 403",
|
||||
secretName: credentialsSecretName(srv),
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("POST /api/bootstrap: %w", err)
|
||||
}
|
||||
|
||||
if err := writeOperatorSecret(ctx, r.Client, r.OperatorNamespace, checkpointName, result.APIKey.Key); err != nil {
|
||||
return "", fmt.Errorf("checkpointing admin key: %w", err)
|
||||
}
|
||||
return result.APIKey.Key, nil
|
||||
}
|
||||
|
||||
// getOrMintInstanceServiceAccountKey mints the operator's own instance-
|
||||
// scoped service account, or, if an earlier interrupted attempt already
|
||||
// created it (409), adopts it and mints a fresh key rather than treating
|
||||
// the conflict as an error (DESIGN.md §6 point 1, §5's general
|
||||
// adopt-on-conflict rule).
|
||||
func (r *TerdutServerReconciler) getOrMintInstanceServiceAccountKey(ctx context.Context, bc *tdclient.Client) (string, error) {
|
||||
result, err := bc.CreateInstanceServiceAccount(ctx, serviceAccountName)
|
||||
if err == nil {
|
||||
return result.Key.Key, nil
|
||||
}
|
||||
|
||||
statusErr, ok := errors.AsType[*tdclient.StatusError](err)
|
||||
if !ok || statusErr.Code != http.StatusConflict {
|
||||
return "", fmt.Errorf("POST /api/service-accounts: %w", err)
|
||||
}
|
||||
|
||||
sa, err := bc.GetServiceAccountByName(ctx, serviceAccountName)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("GET /api/service-accounts?name=%s (adopting after 409): %w", serviceAccountName, err)
|
||||
}
|
||||
if sa == nil {
|
||||
return "", fmt.Errorf("POST /api/service-accounts 409'd for %q but GET found nothing", serviceAccountName)
|
||||
}
|
||||
key, err := bc.CreateServiceAccountKey(ctx, sa.ID, "initial")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("POST /api/service-accounts/%d/keys (adopting after 409): %w", sa.ID, err)
|
||||
}
|
||||
return key.Key, nil
|
||||
}
|
||||
@@ -2,7 +2,7 @@ package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
@@ -16,12 +16,10 @@ import (
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
|
||||
logf "sigs.k8s.io/controller-runtime/pkg/log"
|
||||
"sigs.k8s.io/controller-runtime/pkg/recorder"
|
||||
|
||||
terdutv1alpha1 "git.ryuvia.com/niklas/terdut-operator/api/v1alpha1"
|
||||
"git.ryuvia.com/niklas/terdut-operator/internal/tdclient"
|
||||
)
|
||||
|
||||
// resyncInterval is the periodic requeue on a successful reconcile (DESIGN.md
|
||||
@@ -39,27 +37,6 @@ const resyncInterval = 5 * time.Minute
|
||||
// than "someone edited something out of band."
|
||||
const waitInterval = 15 * time.Second
|
||||
|
||||
// finalizerName cleans up the Secret(s) this controller generates in the
|
||||
// operator's own namespace on delete (which of them, spec.credentials.
|
||||
// deletionPolicy decides) — the Deployment and Service are owned
|
||||
// (OwnerReference, DESIGN.md §7) and need no finalizer of their own.
|
||||
const finalizerName = "terdut.ryuvia.com/terdutserver"
|
||||
|
||||
// serviceAccountName is the name the operator registers itself under
|
||||
// server-side (DESIGN.md §6) — a fixed, repo-wide constant, not a spec
|
||||
// field: it names the automation, not anything about this one TerdutServer.
|
||||
const serviceAccountName = "terdut-operator"
|
||||
|
||||
// bootstrapUsername/bootstrapEmail found the one human-shaped user every
|
||||
// fresh install needs (terdut-server's handleBootstrap requires both).
|
||||
// Nobody signs in as this user afterward — its only purpose is minting the
|
||||
// admin key the controller immediately trades for a real service-account
|
||||
// key — so these are fixed, not spec fields.
|
||||
const (
|
||||
bootstrapUsername = "terdut-operator-bootstrap"
|
||||
bootstrapEmail = "bootstrap@terdut-operator.local"
|
||||
)
|
||||
|
||||
// postgresqlGVK is the Zalando postgres-operator's CR (DESIGN.md §8).
|
||||
// Resolved via unstructured rather than vendoring Zalando's own client, to
|
||||
// keep this operator's dependency on it to "an optional CRD read" rather
|
||||
@@ -75,21 +52,9 @@ type TerdutServerReconciler struct {
|
||||
client.Client
|
||||
Scheme *runtime.Scheme
|
||||
|
||||
// OperatorNamespace is where every credentials Secret this controller
|
||||
// generates lives (DESIGN.md §6) — never the TerdutServer's own
|
||||
// namespace. Set from the POD_NAMESPACE downward-API env var in
|
||||
// production (cmd/main.go); tests set it directly.
|
||||
OperatorNamespace string
|
||||
|
||||
// Recorder emits the Kubernetes Events DESIGN.md §12 asks for on every
|
||||
// externally-visible outcome.
|
||||
Recorder recorder.EventRecorder
|
||||
|
||||
// NewClient builds the terdut-server API client for a given endpoint. A
|
||||
// field, not a direct tdclient.New call, so tests can substitute an
|
||||
// httptest.Server's client without a real network round trip. Defaults
|
||||
// to tdclient.New via SetupWithManager.
|
||||
NewClient func(endpoint string) *tdclient.Client
|
||||
}
|
||||
|
||||
// +kubebuilder:rbac:groups=terdut.ryuvia.com,resources=terdutservers,verbs=get;list;watch;create;update;patch;delete
|
||||
@@ -113,26 +78,18 @@ func (r *TerdutServerReconciler) Reconcile(ctx context.Context, req ctrl.Request
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
|
||||
if !srv.DeletionTimestamp.IsZero() {
|
||||
return r.reconcileDelete(ctx, &srv)
|
||||
}
|
||||
|
||||
if !controllerutil.ContainsFinalizer(&srv, finalizerName) {
|
||||
controllerutil.AddFinalizer(&srv, finalizerName)
|
||||
if err := r.Update(ctx, &srv); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
// The Update above re-triggers a reconcile via the watch; nothing
|
||||
// further to do on this pass.
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
|
||||
dbEnv, dbErr := r.resolveDatabaseEnv(ctx, &srv)
|
||||
if dbErr != nil {
|
||||
return r.setNotReady(ctx, &srv, dbErr.reason, dbErr.message, waitInterval)
|
||||
}
|
||||
|
||||
deploy, err := r.reconcileDeployment(ctx, &srv, dbEnv)
|
||||
operatorKey, err := r.reconcileOperatorKey(ctx, &srv)
|
||||
if err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
srv.Status.CredentialsSecretRef = operatorKeyRef(&srv)
|
||||
|
||||
deploy, err := r.reconcileDeployment(ctx, &srv, dbEnv, operatorKeyHash(operatorKey))
|
||||
if err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
@@ -143,13 +100,6 @@ func (r *TerdutServerReconciler) Reconcile(ctx context.Context, req ctrl.Request
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
|
||||
meta.SetStatusCondition(&srv.Status.Conditions, metav1.Condition{
|
||||
Type: terdutv1alpha1.ConditionDatabaseReady,
|
||||
Status: metav1.ConditionTrue,
|
||||
Reason: terdutv1alpha1.ReasonAdopted,
|
||||
Message: "database resolved",
|
||||
})
|
||||
|
||||
if deploy.Status.ReadyReplicas < 1 {
|
||||
return r.setNotReady(ctx, &srv,
|
||||
terdutv1alpha1.ReasonWaitingForDeployment,
|
||||
@@ -157,28 +107,12 @@ func (r *TerdutServerReconciler) Reconcile(ctx context.Context, req ctrl.Request
|
||||
waitInterval)
|
||||
}
|
||||
|
||||
if srv.Status.CredentialsSecretRef == nil {
|
||||
if err := r.reconcileBootstrap(ctx, &srv); err != nil {
|
||||
if pending, ok := errors.AsType[*bootstrapStateLostError](err); ok {
|
||||
return r.setNotReady(ctx, &srv, terdutv1alpha1.ReasonBootstrapStateLost, pending.Error(), waitInterval)
|
||||
}
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
}
|
||||
|
||||
meta.SetStatusCondition(&srv.Status.Conditions, metav1.Condition{
|
||||
Type: terdutv1alpha1.ConditionBootstrapped,
|
||||
Status: metav1.ConditionTrue,
|
||||
Reason: terdutv1alpha1.ReasonAdopted,
|
||||
Message: fmt.Sprintf("credentials in Secret %q", srv.Status.CredentialsSecretRef.Name),
|
||||
})
|
||||
meta.SetStatusCondition(&srv.Status.Conditions, metav1.Condition{
|
||||
Type: terdutv1alpha1.ConditionReady,
|
||||
Status: metav1.ConditionTrue,
|
||||
Reason: terdutv1alpha1.ReasonAdopted,
|
||||
Message: "deployment ready, database resolved, credentials bootstrapped",
|
||||
Message: "deployment ready, database resolved, operator key seeded",
|
||||
})
|
||||
srv.Status.ServiceName = srv.Name
|
||||
srv.Status.ObservedGeneration = srv.Generation
|
||||
if err := r.Status().Update(ctx, &srv); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
@@ -215,48 +149,8 @@ func (r *TerdutServerReconciler) setNotReady(
|
||||
return ctrl.Result{RequeueAfter: d}, nil
|
||||
}
|
||||
|
||||
// reconcileDelete cleans up the Secrets this controller generated in the
|
||||
// operator's own namespace. The Deployment and Service are owned
|
||||
// (OwnerReference, DESIGN.md §7) and need no attention here — normal GC
|
||||
// handles them. There is no server-side "delete this install" call to
|
||||
// make: bootstrap created a user and a service account, and terdut-server's
|
||||
// API has no way to delete either (only to revoke individual keys), so
|
||||
// there is nothing meaningful to undo there either, and the database is
|
||||
// never touched.
|
||||
//
|
||||
// That is why the instance credential is kept by default
|
||||
// (spec.credentials.deletionPolicy: Retain). Everything it logs in to
|
||||
// outlives the TerdutServer, so a recreated one finds a bootstrapped server
|
||||
// it has no key for -- unless the key is still here to be adopted (see
|
||||
// adoptRetainedCredentials). The bootstrap checkpoint is always removed: it
|
||||
// is a short-lived admin key, and the instance credential is all that is
|
||||
// needed afterwards.
|
||||
func (r *TerdutServerReconciler) reconcileDelete(ctx context.Context, srv *terdutv1alpha1.TerdutServer) (ctrl.Result, error) {
|
||||
if !controllerutil.ContainsFinalizer(srv, finalizerName) {
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
names := []string{checkpointSecretName(srv)}
|
||||
if srv.Spec.Credentials.DeletionPolicy == terdutv1alpha1.CredentialsDelete {
|
||||
names = append(names, credentialsSecretName(srv))
|
||||
}
|
||||
for _, name := range names {
|
||||
sec := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: r.OperatorNamespace}}
|
||||
if err := r.Delete(ctx, sec); err != nil && !apierrors.IsNotFound(err) {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
}
|
||||
controllerutil.RemoveFinalizer(srv, finalizerName)
|
||||
if err := r.Update(ctx, srv); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
|
||||
// SetupWithManager sets up the controller with the Manager.
|
||||
func (r *TerdutServerReconciler) SetupWithManager(mgr ctrl.Manager) error {
|
||||
if r.NewClient == nil {
|
||||
r.NewClient = tdclient.New
|
||||
}
|
||||
if r.Recorder == nil {
|
||||
r.Recorder = mgr.GetEventRecorder("terdutserver-controller")
|
||||
}
|
||||
@@ -265,20 +159,13 @@ func (r *TerdutServerReconciler) SetupWithManager(mgr ctrl.Manager) error {
|
||||
Owns(&appsv1.Deployment{}).
|
||||
Owns(&corev1.Service{}).
|
||||
Owns(&policyv1.PodDisruptionBudget{}).
|
||||
Owns(&corev1.Secret{}).
|
||||
Named("terdutserver").
|
||||
Complete(r)
|
||||
}
|
||||
|
||||
// --- naming ---
|
||||
|
||||
func checkpointSecretName(srv *terdutv1alpha1.TerdutServer) string {
|
||||
return fmt.Sprintf("%s.%s-bootstrap-admin", srv.Namespace, srv.Name)
|
||||
}
|
||||
|
||||
func credentialsSecretName(srv *terdutv1alpha1.TerdutServer) string {
|
||||
return fmt.Sprintf("%s.%s-instance-credentials", srv.Namespace, srv.Name)
|
||||
}
|
||||
|
||||
func serviceURL(srv *terdutv1alpha1.TerdutServer) string {
|
||||
port := srv.Spec.Networking.ServicePort
|
||||
if port == 0 {
|
||||
|
||||
@@ -2,14 +2,7 @@ package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
@@ -27,522 +20,8 @@ import (
|
||||
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
||||
|
||||
terdutv1alpha1 "git.ryuvia.com/niklas/terdut-operator/api/v1alpha1"
|
||||
"git.ryuvia.com/niklas/terdut-operator/internal/tdclient"
|
||||
)
|
||||
|
||||
// fakeVersionString/errJSONKey are shared by every response fakeTerdutServer
|
||||
// writes -- goconst would otherwise flag "test" and "error" as repeated
|
||||
// literals across its handlers.
|
||||
const (
|
||||
fakeVersionString = "test"
|
||||
errJSONKey = "error"
|
||||
// deadmanSwitchesPath is the literal path (not Printf'd like the others
|
||||
// below) shared by the exact-match collection route and the dispatcher
|
||||
// that routes into it -- goconst flags three occurrences of the same
|
||||
// string, so this is that string, once.
|
||||
deadmanSwitchesPath = "/deadman/switches"
|
||||
)
|
||||
|
||||
// fakeTerdutServer reproduces the exact stateful semantics of
|
||||
// /api/bootstrap, /api/service-accounts and /api/version that the
|
||||
// bootstrap flow depends on (DESIGN.md §6, §11: "terdut-server's REST API
|
||||
// is faked with a small httptest.Server per controller test ... matching
|
||||
// the real handlers' request/response shapes"), including the 403-after-
|
||||
// first-success and 409-on-name-conflict behavior the adopt-on-conflict
|
||||
// recovery path exists for.
|
||||
type fakeTerdutServer struct {
|
||||
mu sync.Mutex
|
||||
bootstrapped bool
|
||||
bootstrap403 bool // force every /api/bootstrap call to 403, even the first
|
||||
// rejectedTokens: bearer tokens the fake answers 401 on GET
|
||||
// /api/service-accounts, the way a server that never issued the key
|
||||
// (a database reset since) would.
|
||||
rejectedTokens map[string]bool
|
||||
nextID int64
|
||||
accounts map[string]int64 // name -> id
|
||||
keyMints map[int64]int // id -> number of keys minted so far
|
||||
|
||||
nextTeamID int64
|
||||
teams map[string]int64 // name -> id
|
||||
teamNames map[int64]string // id -> current name (renames update this)
|
||||
teamOIDC map[int64][2]string
|
||||
teamDelete map[int64]bool // id -> true once DELETEd, for 404-on-redelete
|
||||
|
||||
// users backs GET /api/users for TerdutEscalationRule's username
|
||||
// resolution (DESIGN.md §4.3) -- a fixed, pre-seeded directory, since
|
||||
// nothing in this controller's own flow ever creates a user.
|
||||
users map[string]int64 // username -> id
|
||||
|
||||
// escalation backs PUT /api/teams/{id}/escalation -- an upsert
|
||||
// server-side (confirmed against source), so this is just "the last
|
||||
// body PUT for this team", keyed by teamID, with no separate create
|
||||
// step to model.
|
||||
escalation map[int64]tdclient.SetEscalationRequest
|
||||
|
||||
// switches/nextSwitchID/switchDelete back the dead man's switch
|
||||
// endpoints -- no unique-name constraint server-side (DESIGN.md §4.4),
|
||||
// so switches is keyed by id, not name, same as the real API's own
|
||||
// GET-list-and-match-by-name idempotent-create shape requires.
|
||||
nextSwitchID int64
|
||||
switches map[int64]map[int64]tdclient.DeadmanSwitch // teamID -> switchID -> switch
|
||||
switchDelete map[int64]bool // switchID -> true once DELETEd, for 404-on-redelete
|
||||
|
||||
// integrations/nextIntegrationID/integrationDelete back the alert
|
||||
// source endpoints -- no unique-name constraint server-side either
|
||||
// (DESIGN.md §4.5), same shape as switches, keyed by id.
|
||||
nextIntegrationID int64
|
||||
integrations map[int64]map[int64]tdclient.Integration // teamID -> integrationID -> integration
|
||||
integrationDelete map[int64]bool // integrationID -> true once DELETEd, for 404-on-redelete
|
||||
|
||||
// invites/nextInviteID/inviteDelete back TerdutTeam's own invite-minting
|
||||
// feature -- no unique constraint on an invite server-side either (every
|
||||
// POST mints a brand new row, confirmed against source), same keyed-by-id
|
||||
// shape as switches/integrations.
|
||||
nextInviteID int64
|
||||
invites map[int64]map[int64]tdclient.Invite // teamID -> inviteID -> invite
|
||||
inviteDelete map[int64]bool // inviteID -> true once DELETEd, for 404-on-redelete
|
||||
}
|
||||
|
||||
func newFakeTerdutServer() (*fakeTerdutServer, *httptest.Server) {
|
||||
f := &fakeTerdutServer{
|
||||
accounts: map[string]int64{},
|
||||
keyMints: map[int64]int{},
|
||||
teams: map[string]int64{},
|
||||
teamNames: map[int64]string{},
|
||||
teamOIDC: map[int64][2]string{},
|
||||
teamDelete: map[int64]bool{},
|
||||
users: map[string]int64{},
|
||||
escalation: map[int64]tdclient.SetEscalationRequest{},
|
||||
switches: map[int64]map[int64]tdclient.DeadmanSwitch{},
|
||||
switchDelete: map[int64]bool{},
|
||||
|
||||
integrations: map[int64]map[int64]tdclient.Integration{},
|
||||
integrationDelete: map[int64]bool{},
|
||||
|
||||
invites: map[int64]map[int64]tdclient.Invite{},
|
||||
inviteDelete: map[int64]bool{},
|
||||
}
|
||||
return f, httptest.NewServer(f)
|
||||
}
|
||||
|
||||
// seedUser registers a username the fake GET /api/users will return --
|
||||
// called from test setup, before the controller under test ever runs.
|
||||
// Callers read the assigned id back from f.users themselves, so this has
|
||||
// nothing left to return.
|
||||
func (f *fakeTerdutServer) seedUser(username string) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.nextID++
|
||||
f.users[username] = f.nextID
|
||||
}
|
||||
|
||||
func (f *fakeTerdutServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
|
||||
switch {
|
||||
case r.URL.Path == "/api/version":
|
||||
writeJSON(w, http.StatusOK, map[string]string{"version": fakeVersionString})
|
||||
|
||||
case r.URL.Path == "/api/bootstrap" && r.Method == http.MethodPost:
|
||||
if f.bootstrapped || f.bootstrap403 {
|
||||
writeJSON(w, http.StatusForbidden, map[string]string{errJSONKey: "bootstrap already completed"})
|
||||
return
|
||||
}
|
||||
f.bootstrapped = true
|
||||
writeJSON(w, http.StatusCreated, tdclient.BootstrapResult{
|
||||
APIKey: tdclient.APIKey{ID: 1, Name: "bootstrap", Key: "admin-key-raw"},
|
||||
})
|
||||
|
||||
case r.URL.Path == "/api/service-accounts" && r.Method == http.MethodPost:
|
||||
var req struct {
|
||||
Name string `json:"name"`
|
||||
Scope string `json:"scope"`
|
||||
}
|
||||
_ = json.NewDecoder(r.Body).Decode(&req)
|
||||
if _, exists := f.accounts[req.Name]; exists {
|
||||
writeJSON(w, http.StatusConflict, map[string]string{errJSONKey: "a service account with that name already exists"})
|
||||
return
|
||||
}
|
||||
f.nextID++
|
||||
id := f.nextID
|
||||
f.accounts[req.Name] = id
|
||||
f.keyMints[id] = 1
|
||||
writeJSON(w, http.StatusCreated, tdclient.CreateServiceAccountResult{
|
||||
ServiceAccount: tdclient.ServiceAccount{ID: id, Name: req.Name, Scope: req.Scope},
|
||||
Key: tdclient.APIKey{ID: 1, Name: "initial", Key: fmt.Sprintf("instance-key-%d-initial", id)},
|
||||
})
|
||||
|
||||
case r.URL.Path == "/api/service-accounts" && r.Method == http.MethodGet:
|
||||
if f.rejectedTokens[strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")] {
|
||||
writeJSON(w, http.StatusUnauthorized, map[string]string{errJSONKey: "invalid or expired API key"})
|
||||
return
|
||||
}
|
||||
name := r.URL.Query().Get("name")
|
||||
id, exists := f.accounts[name]
|
||||
if !exists {
|
||||
writeJSON(w, http.StatusOK, []tdclient.ServiceAccount{})
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, []tdclient.ServiceAccount{{ID: id, Name: name, Scope: "instance"}})
|
||||
|
||||
case r.URL.Path == "/api/teams" && r.Method == http.MethodPost:
|
||||
var req struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
_ = json.NewDecoder(r.Body).Decode(&req)
|
||||
if _, exists := f.teams[req.Name]; exists {
|
||||
writeJSON(w, http.StatusConflict, map[string]string{errJSONKey: "a team with that name already exists"})
|
||||
return
|
||||
}
|
||||
f.nextTeamID++
|
||||
id := f.nextTeamID
|
||||
f.teams[req.Name] = id
|
||||
f.teamNames[id] = req.Name
|
||||
writeJSON(w, http.StatusCreated, tdclient.Team{ID: id, Name: req.Name})
|
||||
|
||||
case r.URL.Path == "/api/teams" && r.Method == http.MethodGet:
|
||||
// The controller never calls this without ?name= (TEAM-LOOKUP.md's
|
||||
// own lookup shape) -- the fake only needs to answer that form.
|
||||
name := r.URL.Query().Get("name")
|
||||
id, exists := f.teams[name]
|
||||
if !exists {
|
||||
writeJSON(w, http.StatusOK, []tdclient.Team{})
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, []tdclient.Team{{ID: id, Name: f.teamNames[id]}})
|
||||
|
||||
case r.URL.Path == "/api/users" && r.Method == http.MethodGet:
|
||||
// No query filter -- GetUserByUsername fetches the whole list and
|
||||
// matches client-side (confirmed against source: no server-side
|
||||
// filter either), so the fake does the same.
|
||||
users := make([]tdclient.User, 0, len(f.users))
|
||||
for name, id := range f.users {
|
||||
users = append(users, tdclient.User{ID: id, Username: name})
|
||||
}
|
||||
writeJSON(w, http.StatusOK, users)
|
||||
|
||||
default:
|
||||
if id, name, ok := parseKeysPath(r.URL.Path); ok && r.Method == http.MethodPost {
|
||||
f.keyMints[id]++
|
||||
writeJSON(w, http.StatusCreated, tdclient.APIKey{
|
||||
ID: int64(f.keyMints[id]), Name: name,
|
||||
Key: fmt.Sprintf("instance-key-%d-mint%d", id, f.keyMints[id]),
|
||||
})
|
||||
return
|
||||
}
|
||||
if id, rest, ok := parseTeamSubPath(r.URL.Path); ok {
|
||||
f.handleTeamSubPath(w, r, id, rest)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}
|
||||
|
||||
// handleTeamSubPath answers everything under /api/teams/{id}: PUT (rename),
|
||||
// DELETE, PUT .../oidc-groups, PUT .../escalation, and the dead man's
|
||||
// switch collection/item endpoints. rest is whatever parseTeamSubPath found
|
||||
// after "/api/teams/{id}" -- "" for the bare resource.
|
||||
func (f *fakeTerdutServer) handleTeamSubPath(w http.ResponseWriter, r *http.Request, id int64, rest string) {
|
||||
switch {
|
||||
case rest == "" && r.Method == http.MethodPut:
|
||||
var req struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
_ = json.NewDecoder(r.Body).Decode(&req)
|
||||
oldName, exists := f.teamNames[id]
|
||||
if !exists {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
delete(f.teams, oldName)
|
||||
f.teamNames[id] = req.Name
|
||||
f.teams[req.Name] = id
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
|
||||
case rest == "" && r.Method == http.MethodDelete:
|
||||
name, exists := f.teamNames[id]
|
||||
if !exists {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
delete(f.teams, name)
|
||||
delete(f.teamNames, id)
|
||||
f.teamDelete[id] = true
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
|
||||
case rest == "/oidc-groups" && r.Method == http.MethodPut:
|
||||
var req struct {
|
||||
MemberGroup string `json:"member_group"`
|
||||
OwnerGroup string `json:"owner_group"`
|
||||
}
|
||||
_ = json.NewDecoder(r.Body).Decode(&req)
|
||||
if _, exists := f.teamNames[id]; !exists {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
f.teamOIDC[id] = [2]string{req.MemberGroup, req.OwnerGroup}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
|
||||
case rest == "/escalation" && r.Method == http.MethodPut:
|
||||
var req tdclient.SetEscalationRequest
|
||||
_ = json.NewDecoder(r.Body).Decode(&req)
|
||||
f.escalation[id] = req
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
|
||||
case rest == deadmanSwitchesPath || strings.HasPrefix(rest, deadmanSwitchesPath+"/"):
|
||||
f.handleDeadmanSubPath(w, r, id, rest)
|
||||
|
||||
case rest == "/integrations" || strings.HasPrefix(rest, "/integrations/"):
|
||||
f.handleIntegrationSubPath(w, r, id, rest)
|
||||
|
||||
case rest == "/invites" || strings.HasPrefix(rest, "/invites/"):
|
||||
f.handleInviteSubPath(w, r, id, rest)
|
||||
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}
|
||||
|
||||
// handleDeadmanSubPath answers GET/POST /api/teams/{id}/deadman/switches and
|
||||
// PUT/DELETE .../deadman/switches/{switchID} -- split out of
|
||||
// handleTeamSubPath for the same gocyclo reason as handleIntegrationSubPath.
|
||||
func (f *fakeTerdutServer) handleDeadmanSubPath(w http.ResponseWriter, r *http.Request, id int64, rest string) {
|
||||
switch {
|
||||
case rest == deadmanSwitchesPath && r.Method == http.MethodGet:
|
||||
existing := f.switches[id]
|
||||
out := make([]tdclient.DeadmanSwitch, 0, len(existing))
|
||||
for _, s := range existing {
|
||||
out = append(out, s)
|
||||
}
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
|
||||
case rest == deadmanSwitchesPath && r.Method == http.MethodPost:
|
||||
var req deadmanSwitchFakeRequest
|
||||
_ = json.NewDecoder(r.Body).Decode(&req)
|
||||
f.nextSwitchID++
|
||||
switchID := f.nextSwitchID
|
||||
name := req.Name
|
||||
if name == "" {
|
||||
name = "derived-" + req.Matcher
|
||||
}
|
||||
sw := tdclient.DeadmanSwitch{
|
||||
ID: switchID, Name: name, Matcher: req.Matcher,
|
||||
TimeoutSeconds: req.TimeoutSeconds, Severity: req.Severity,
|
||||
}
|
||||
if f.switches[id] == nil {
|
||||
f.switches[id] = map[int64]tdclient.DeadmanSwitch{}
|
||||
}
|
||||
f.switches[id][switchID] = sw
|
||||
writeJSON(w, http.StatusCreated, sw)
|
||||
|
||||
case strings.HasPrefix(rest, "/deadman/switches/") && r.Method == http.MethodPut:
|
||||
switchID, ok := parseTrailingID(rest, "/deadman/switches/")
|
||||
if !ok {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
if _, exists := f.switches[id][switchID]; !exists {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
var req deadmanSwitchFakeRequest
|
||||
_ = json.NewDecoder(r.Body).Decode(&req)
|
||||
name := req.Name
|
||||
if name == "" {
|
||||
name = f.switches[id][switchID].Name
|
||||
}
|
||||
f.switches[id][switchID] = tdclient.DeadmanSwitch{
|
||||
ID: switchID, Name: name, Matcher: req.Matcher,
|
||||
TimeoutSeconds: req.TimeoutSeconds, Severity: req.Severity,
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
|
||||
case strings.HasPrefix(rest, "/deadman/switches/") && r.Method == http.MethodDelete:
|
||||
switchID, ok := parseTrailingID(rest, "/deadman/switches/")
|
||||
if !ok {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
if _, exists := f.switches[id][switchID]; !exists {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
delete(f.switches[id], switchID)
|
||||
f.switchDelete[switchID] = true
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}
|
||||
|
||||
// handleInviteSubPath answers POST /api/teams/{id}/invites and
|
||||
// DELETE .../invites/{inviteID} -- split out for the same gocyclo reason as
|
||||
// handleIntegrationSubPath.
|
||||
func (f *fakeTerdutServer) handleInviteSubPath(w http.ResponseWriter, r *http.Request, id int64, rest string) {
|
||||
switch {
|
||||
case rest == "/invites" && r.Method == http.MethodPost:
|
||||
var req struct {
|
||||
Role string `json:"role"`
|
||||
MaxUses int64 `json:"max_uses"`
|
||||
}
|
||||
_ = json.NewDecoder(r.Body).Decode(&req)
|
||||
f.nextInviteID++
|
||||
inviteID := f.nextInviteID
|
||||
inv := tdclient.Invite{
|
||||
ID: inviteID, TeamID: id, Role: req.Role, MaxUses: req.MaxUses,
|
||||
ExpiresAt: time.Now().Add(7 * 24 * time.Hour),
|
||||
URL: fmt.Sprintf("https://terdut.example.invalid/signup?invite=invite-token-%d", inviteID),
|
||||
}
|
||||
if f.invites[id] == nil {
|
||||
f.invites[id] = map[int64]tdclient.Invite{}
|
||||
}
|
||||
f.invites[id][inviteID] = inv
|
||||
writeJSON(w, http.StatusCreated, inv)
|
||||
|
||||
case strings.HasPrefix(rest, "/invites/") && r.Method == http.MethodDelete:
|
||||
inviteID, ok := parseTrailingID(rest, "/invites/")
|
||||
if !ok {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
if _, exists := f.invites[id][inviteID]; !exists {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
delete(f.invites[id], inviteID)
|
||||
f.inviteDelete[inviteID] = true
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}
|
||||
|
||||
// handleIntegrationSubPath answers POST /api/teams/{id}/integrations,
|
||||
// PATCH .../integrations/{integrationID} and DELETE .../integrations/{integrationID}
|
||||
// -- split out of handleTeamSubPath so that switch's own cyclomatic
|
||||
// complexity stays under golangci-lint's gocyclo threshold.
|
||||
func (f *fakeTerdutServer) handleIntegrationSubPath(w http.ResponseWriter, r *http.Request, id int64, rest string) {
|
||||
switch {
|
||||
case rest == "/integrations" && r.Method == http.MethodPost:
|
||||
var req struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
}
|
||||
_ = json.NewDecoder(r.Body).Decode(&req)
|
||||
f.nextIntegrationID++
|
||||
integID := f.nextIntegrationID
|
||||
integ := tdclient.Integration{
|
||||
ID: integID, TeamID: id, Kind: req.Kind, Name: req.Name,
|
||||
// Key/URL are only ever in *this* response -- never again,
|
||||
// matching terdut-server's own one-time-show semantics
|
||||
// (DESIGN.md §4.5) -- so what's stored for later GET/PATCH
|
||||
// calls in this fake deliberately omits them too.
|
||||
Key: fmt.Sprintf("webhook-key-%d", integID),
|
||||
URL: fmt.Sprintf("https://terdut.example.invalid/api/integrations/webhook-key-%d/%s", integID, req.Kind),
|
||||
}
|
||||
if f.integrations[id] == nil {
|
||||
f.integrations[id] = map[int64]tdclient.Integration{}
|
||||
}
|
||||
f.integrations[id][integID] = tdclient.Integration{ID: integID, TeamID: id, Kind: req.Kind, Name: req.Name}
|
||||
writeJSON(w, http.StatusCreated, integ)
|
||||
|
||||
case strings.HasPrefix(rest, "/integrations/") && r.Method == http.MethodPatch:
|
||||
integID, ok := parseTrailingID(rest, "/integrations/")
|
||||
if !ok {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
existing, exists := f.integrations[id][integID]
|
||||
if !exists {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
var req struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
_ = json.NewDecoder(r.Body).Decode(&req)
|
||||
existing.Name = req.Name
|
||||
f.integrations[id][integID] = existing
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
|
||||
case strings.HasPrefix(rest, "/integrations/") && r.Method == http.MethodDelete:
|
||||
integID, ok := parseTrailingID(rest, "/integrations/")
|
||||
if !ok {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
if _, exists := f.integrations[id][integID]; !exists {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
delete(f.integrations[id], integID)
|
||||
f.integrationDelete[integID] = true
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}
|
||||
|
||||
// deadmanSwitchFakeRequest mirrors tdclient's own (unexported)
|
||||
// deadmanSwitchRequest -- the fake needs its own copy to decode the same
|
||||
// wire shape without reaching across package boundaries for an internal type.
|
||||
type deadmanSwitchFakeRequest struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
Matcher string `json:"matcher"`
|
||||
TimeoutSeconds int64 `json:"timeout_seconds"`
|
||||
Severity string `json:"severity"`
|
||||
}
|
||||
|
||||
// parseTeamSubPath splits "/api/teams/{id}" from anything after it --
|
||||
// "" for an exact match, "/oidc-groups", "/escalation", "/deadman/switches"
|
||||
// or "/deadman/switches/{switchID}" otherwise. Doesn't itself validate the
|
||||
// suffix; handleTeamSubPath's own switch does that.
|
||||
func parseTeamSubPath(path string) (id int64, rest string, ok bool) {
|
||||
const prefix = "/api/teams/"
|
||||
if !strings.HasPrefix(path, prefix) {
|
||||
return 0, "", false
|
||||
}
|
||||
trimmed := path[len(prefix):]
|
||||
parts := strings.SplitN(trimmed, "/", 2)
|
||||
parsedID, err := strconv.ParseInt(parts[0], 10, 64)
|
||||
if err != nil {
|
||||
return 0, "", false
|
||||
}
|
||||
if len(parts) == 1 {
|
||||
return parsedID, "", true
|
||||
}
|
||||
return parsedID, "/" + parts[1], true
|
||||
}
|
||||
|
||||
// parseTrailingID parses the numeric id after prefix within rest, e.g.
|
||||
// parseTrailingID("/deadman/switches/7", "/deadman/switches/") -> 7, true.
|
||||
func parseTrailingID(rest, prefix string) (id int64, ok bool) {
|
||||
parsedID, err := strconv.ParseInt(strings.TrimPrefix(rest, prefix), 10, 64)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
return parsedID, true
|
||||
}
|
||||
|
||||
func parseKeysPath(path string) (id int64, mintName string, ok bool) {
|
||||
var parsedID int64
|
||||
n, err := fmt.Sscanf(path, "/api/service-accounts/%d/keys", &parsedID)
|
||||
if err != nil || n != 1 {
|
||||
return 0, "", false
|
||||
}
|
||||
return parsedID, "minted", true
|
||||
}
|
||||
|
||||
func writeJSON(w http.ResponseWriter, status int, v any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(v)
|
||||
}
|
||||
|
||||
var _ = Describe("TerdutServer Controller", func() {
|
||||
const operatorNamespace = "default"
|
||||
|
||||
@@ -554,9 +33,8 @@ var _ = Describe("TerdutServer Controller", func() {
|
||||
|
||||
BeforeEach(func() {
|
||||
reconciler = &TerdutServerReconciler{
|
||||
Client: k8sClient,
|
||||
Scheme: k8sClient.Scheme(),
|
||||
OperatorNamespace: operatorNamespace,
|
||||
Client: k8sClient,
|
||||
Scheme: k8sClient.Scheme(),
|
||||
}
|
||||
name = fmt.Sprintf("test-server-%d-%d", GinkgoRandomSeed(), GinkgoParallelProcess())
|
||||
objKey = types.NamespacedName{Name: name, Namespace: operatorNamespace}
|
||||
@@ -569,9 +47,7 @@ var _ = Describe("TerdutServer Controller", func() {
|
||||
_ = k8sClient.Update(ctx, srv)
|
||||
_ = k8sClient.Delete(ctx, srv)
|
||||
}
|
||||
for _, n := range []string{checkpointSecretNameFor(name), credentialsSecretNameFor(name)} {
|
||||
_ = k8sClient.Delete(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: n, Namespace: operatorNamespace}})
|
||||
}
|
||||
_ = k8sClient.Delete(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name + "-operator-key", Namespace: operatorNamespace}})
|
||||
})
|
||||
|
||||
dsnSpec := func() terdutv1alpha1.TerdutServerSpec {
|
||||
@@ -608,127 +84,117 @@ var _ = Describe("TerdutServer Controller", func() {
|
||||
srv := &terdutv1alpha1.TerdutServer{}
|
||||
Expect(k8sClient.Get(ctx, objKey, srv)).To(Succeed())
|
||||
c := meta.FindStatusCondition(srv.Status.Conditions, terdutv1alpha1.ConditionReady)
|
||||
Expect(c).NotTo(BeNil(), "Ready condition should always be set after a reconcile past the finalizer-add pass")
|
||||
Expect(c).NotTo(BeNil(), "Ready condition should always be set after a reconcile")
|
||||
return *c
|
||||
}
|
||||
|
||||
Describe("bring-your-own DSN path", func() {
|
||||
It("reaches Ready through the full lifecycle: finalizer, Deployment/Service, wait, bootstrap", func(ctx SpecContext) {
|
||||
fake, fakeSrv := newFakeTerdutServer()
|
||||
DeferCleanup(fakeSrv.Close)
|
||||
reconciler.NewClient = func(string) *tdclient.Client { return tdclient.New(fakeSrv.URL) }
|
||||
envOf := func(ctx context.Context) map[string]corev1.EnvVar {
|
||||
var deploy appsv1.Deployment
|
||||
Expect(k8sClient.Get(ctx, objKey, &deploy)).To(Succeed())
|
||||
out := map[string]corev1.EnvVar{}
|
||||
for _, e := range deploy.Spec.Template.Spec.Containers[0].Env {
|
||||
out[e.Name] = e
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
Describe("bring-your-own DSN path", func() {
|
||||
It("creates Deployment, Service and operator key, then reaches Ready once a replica is up", func(ctx SpecContext) {
|
||||
createServer(ctx, dsnSpec())
|
||||
|
||||
// Pass 1: adds the finalizer and returns early.
|
||||
reconcileOnce(ctx)
|
||||
srv := &terdutv1alpha1.TerdutServer{}
|
||||
Expect(k8sClient.Get(ctx, objKey, srv)).To(Succeed())
|
||||
Expect(srv.Finalizers).To(ContainElement(finalizerName))
|
||||
|
||||
// Pass 2: creates Deployment + Service, waits for readiness.
|
||||
reconcileOnce(ctx)
|
||||
Expect(readyCondition(ctx).Reason).To(Equal(terdutv1alpha1.ReasonWaitingForDeployment))
|
||||
|
||||
var deploy appsv1.Deployment
|
||||
Expect(k8sClient.Get(ctx, objKey, &deploy)).To(Succeed())
|
||||
Expect(deploy.Spec.Template.Spec.Containers[0].Image).To(Equal("example.invalid/terdut-server:test"))
|
||||
envNames := map[string]string{}
|
||||
for _, e := range deploy.Spec.Template.Spec.Containers[0].Env {
|
||||
envNames[e.Name] = e.Value
|
||||
}
|
||||
Expect(envNames).To(HaveKeyWithValue("TERDUT_DB_DSN", testDSN))
|
||||
Expect(envNames).To(HaveKeyWithValue("TERDUT_OPERATOR_MODE", "true"))
|
||||
env := envOf(ctx)
|
||||
Expect(env["TERDUT_DB_DSN"].Value).To(Equal(testDSN))
|
||||
Expect(env["TERDUT_OPERATOR_MODE"].Value).To(Equal("true"))
|
||||
Expect(env).NotTo(HaveKey("TERDUT_DEADMAN_MATCHERS"), "dead man's switches are per team now")
|
||||
|
||||
var svc corev1.Service
|
||||
Expect(k8sClient.Get(ctx, objKey, &svc)).To(Succeed())
|
||||
Expect(svc.Spec.Ports[0].Port).To(Equal(int32(8080)))
|
||||
|
||||
// Simulate the Deployment becoming ready (envtest has no
|
||||
// kubelet/deployment-controller to do this for real).
|
||||
markDeploymentReady(ctx)
|
||||
|
||||
// Pass 3: bootstraps for real against the fake server.
|
||||
reconcileOnce(ctx)
|
||||
|
||||
Expect(fake.bootstrapped).To(BeTrue())
|
||||
|
||||
Expect(k8sClient.Get(ctx, objKey, srv)).To(Succeed())
|
||||
ready := meta.FindStatusCondition(srv.Status.Conditions, terdutv1alpha1.ConditionReady)
|
||||
Expect(ready.Status).To(Equal(metav1.ConditionTrue))
|
||||
Expect(ready.Reason).To(Equal(terdutv1alpha1.ReasonAdopted))
|
||||
Expect(srv.Status.CredentialsSecretRef).NotTo(BeNil())
|
||||
Expect(srv.Status.CredentialsSecretRef.Key).To(Equal(credentialsSecretDataKey))
|
||||
|
||||
var credsSecret corev1.Secret
|
||||
Expect(k8sClient.Get(ctx, types.NamespacedName{
|
||||
Name: srv.Status.CredentialsSecretRef.Name, Namespace: operatorNamespace,
|
||||
}, &credsSecret)).To(Succeed())
|
||||
Expect(string(credsSecret.Data[credentialsSecretDataKey])).To(Equal("instance-key-1-initial"))
|
||||
|
||||
// The checkpoint is cleaned up once the lasting credential is
|
||||
// written (DESIGN.md §6 point 2).
|
||||
var checkpoint corev1.Secret
|
||||
err := k8sClient.Get(ctx, types.NamespacedName{Name: checkpointSecretName(srv), Namespace: operatorNamespace}, &checkpoint)
|
||||
Expect(err).To(HaveOccurred())
|
||||
})
|
||||
})
|
||||
|
||||
Describe("the adopt-on-409 recovery path", func() {
|
||||
It("mints a fresh key instead of erroring when the service account already exists", func(ctx SpecContext) {
|
||||
fake, fakeSrv := newFakeTerdutServer()
|
||||
DeferCleanup(fakeSrv.Close)
|
||||
fake.bootstrapped = true // an earlier attempt already bootstrapped...
|
||||
fake.nextID = 1
|
||||
fake.accounts[serviceAccountName] = 1 // ...and already created the service account.
|
||||
fake.keyMints[1] = 1
|
||||
reconciler.NewClient = func(string) *tdclient.Client { return tdclient.New(fakeSrv.URL) }
|
||||
|
||||
createServer(ctx, dsnSpec())
|
||||
reconcileOnce(ctx) // finalizer
|
||||
reconcileOnce(ctx) // Deployment/Service, WaitingForDeployment
|
||||
markDeploymentReady(ctx)
|
||||
|
||||
// The earlier attempt's checkpoint survived (that's how this
|
||||
// reconcile can authenticate at all to recover).
|
||||
Expect(writeOperatorSecret(ctx, k8sClient, operatorNamespace, checkpointSecretName(&terdutv1alpha1.TerdutServer{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: operatorNamespace},
|
||||
}), "admin-key-raw")).To(Succeed())
|
||||
|
||||
reconcileOnce(ctx)
|
||||
|
||||
srv := &terdutv1alpha1.TerdutServer{}
|
||||
Expect(k8sClient.Get(ctx, objKey, srv)).To(Succeed())
|
||||
Expect(meta.FindStatusCondition(srv.Status.Conditions, terdutv1alpha1.ConditionReady).Status).To(Equal(metav1.ConditionTrue))
|
||||
|
||||
var credsSecret corev1.Secret
|
||||
Expect(k8sClient.Get(ctx, types.NamespacedName{
|
||||
Name: srv.Status.CredentialsSecretRef.Name, Namespace: operatorNamespace,
|
||||
}, &credsSecret)).To(Succeed())
|
||||
// Minted fresh, not the (never-seen-by-this-reconcile) "initial"
|
||||
// key from the account's original creation.
|
||||
Expect(string(credsSecret.Data[credentialsSecretDataKey])).To(Equal("instance-key-1-mint2"))
|
||||
ready := meta.FindStatusCondition(srv.Status.Conditions, terdutv1alpha1.ConditionReady)
|
||||
Expect(ready.Status).To(Equal(metav1.ConditionTrue))
|
||||
Expect(ready.Reason).To(Equal(terdutv1alpha1.ReasonAdopted))
|
||||
Expect(srv.Status.CredentialsSecretRef).To(Equal(&terdutv1alpha1.SecretKeyRef{
|
||||
Name: name + "-operator-key", Key: operatorKeyDataKey,
|
||||
}))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("the BootstrapStateLost path", func() {
|
||||
It("fails closed when the server reports already-bootstrapped with no checkpoint to recover from", func(ctx SpecContext) {
|
||||
fake, fakeSrv := newFakeTerdutServer()
|
||||
_ = fake
|
||||
DeferCleanup(fakeSrv.Close)
|
||||
fake.bootstrap403 = true
|
||||
reconciler.NewClient = func(string) *tdclient.Client { return tdclient.New(fakeSrv.URL) }
|
||||
Describe("the operator key", func() {
|
||||
keySecret := func(ctx context.Context) corev1.Secret {
|
||||
var s corev1.Secret
|
||||
Expect(k8sClient.Get(ctx, types.NamespacedName{Name: name + "-operator-key", Namespace: operatorNamespace}, &s)).To(Succeed())
|
||||
return s
|
||||
}
|
||||
|
||||
It("is generated once, owned by the TerdutServer, and wired into the pod", func(ctx SpecContext) {
|
||||
createServer(ctx, dsnSpec())
|
||||
reconcileOnce(ctx) // finalizer
|
||||
reconcileOnce(ctx) // Deployment/Service
|
||||
markDeploymentReady(ctx)
|
||||
|
||||
reconcileOnce(ctx)
|
||||
|
||||
cond := readyCondition(ctx)
|
||||
Expect(cond.Status).To(Equal(metav1.ConditionFalse))
|
||||
Expect(cond.Reason).To(Equal(terdutv1alpha1.ReasonBootstrapStateLost))
|
||||
s := keySecret(ctx)
|
||||
value := string(s.Data[operatorKeyDataKey])
|
||||
Expect(len(value)).To(BeNumerically(">=", 32), "terdut-server refuses a shorter TERDUT_OPERATOR_KEY")
|
||||
Expect(value).To(HavePrefix(operatorKeyPrefix))
|
||||
Expect(s.OwnerReferences).To(ContainElement(HaveField("Name", name)))
|
||||
|
||||
env := envOf(ctx)
|
||||
Expect(env["TERDUT_OPERATOR_KEY"].ValueFrom).NotTo(BeNil())
|
||||
Expect(env["TERDUT_OPERATOR_KEY"].ValueFrom.SecretKeyRef.Name).To(Equal(name + "-operator-key"))
|
||||
Expect(env["TERDUT_OPERATOR_KEY"].Value).To(BeEmpty(), "the key itself must never appear in the pod spec")
|
||||
|
||||
var deploy appsv1.Deployment
|
||||
Expect(k8sClient.Get(ctx, objKey, &deploy)).To(Succeed())
|
||||
Expect(deploy.Spec.Template.Annotations).To(HaveKeyWithValue(operatorKeyHashAnnotation, operatorKeyHash(value)))
|
||||
|
||||
// A later reconcile keeps the same key: the running server was seeded with it.
|
||||
reconcileOnce(ctx)
|
||||
Expect(string(keySecret(ctx).Data[operatorKeyDataKey])).To(Equal(value))
|
||||
})
|
||||
|
||||
It("rolls the Deployment when the Secret is replaced", func(ctx SpecContext) {
|
||||
createServer(ctx, dsnSpec())
|
||||
reconcileOnce(ctx)
|
||||
first := string(keySecret(ctx).Data[operatorKeyDataKey])
|
||||
|
||||
s := keySecret(ctx)
|
||||
Expect(k8sClient.Delete(ctx, &s)).To(Succeed())
|
||||
reconcileOnce(ctx)
|
||||
|
||||
second := string(keySecret(ctx).Data[operatorKeyDataKey])
|
||||
Expect(second).NotTo(Equal(first))
|
||||
var deploy appsv1.Deployment
|
||||
Expect(k8sClient.Get(ctx, objKey, &deploy)).To(Succeed())
|
||||
Expect(deploy.Spec.Template.Annotations).To(HaveKeyWithValue(operatorKeyHashAnnotation, operatorKeyHash(second)))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("spec.oidc claims", func() {
|
||||
It("passes the claim names and trustEmail through", func(ctx SpecContext) {
|
||||
spec := dsnSpec()
|
||||
spec.OIDC = terdutv1alpha1.OIDCSpec{
|
||||
Enabled: true, Issuer: "https://sso.example.invalid", ClientID: "terdut",
|
||||
UsernameClaim: "upn", EmailClaim: "mail", GroupsClaim: "roles", TrustEmail: true,
|
||||
SessionMaxAge: "12h",
|
||||
}
|
||||
createServer(ctx, spec)
|
||||
reconcileOnce(ctx)
|
||||
|
||||
env := envOf(ctx)
|
||||
Expect(env["TERDUT_OIDC_USERNAME_CLAIM"].Value).To(Equal("upn"))
|
||||
Expect(env["TERDUT_OIDC_EMAIL_CLAIM"].Value).To(Equal("mail"))
|
||||
Expect(env["TERDUT_OIDC_GROUPS_CLAIM"].Value).To(Equal("roles"))
|
||||
Expect(env["TERDUT_OIDC_TRUST_EMAIL"].Value).To(Equal("true"))
|
||||
})
|
||||
})
|
||||
|
||||
@@ -743,7 +209,6 @@ var _ = Describe("TerdutServer Controller", func() {
|
||||
|
||||
It("waits with reason PostgresClusterNotFound when the postgresql CR doesn't exist yet", func(ctx SpecContext) {
|
||||
createServer(ctx, zalandoSpec("missing-cluster"))
|
||||
reconcileOnce(ctx) // finalizer
|
||||
reconcileOnce(ctx)
|
||||
|
||||
Expect(readyCondition(ctx).Reason).To(Equal(terdutv1alpha1.ReasonPostgresClusterNotFound))
|
||||
@@ -767,13 +232,7 @@ var _ = Describe("TerdutServer Controller", func() {
|
||||
Expect(k8sClient.Create(ctx, zalandoSecret)).To(Succeed())
|
||||
DeferCleanup(func() { _ = k8sClient.Delete(ctx, zalandoSecret) })
|
||||
|
||||
fake, fakeSrv := newFakeTerdutServer()
|
||||
_ = fake
|
||||
DeferCleanup(fakeSrv.Close)
|
||||
reconciler.NewClient = func(string) *tdclient.Client { return tdclient.New(fakeSrv.URL) }
|
||||
|
||||
createServer(ctx, zalandoSpec(clusterName))
|
||||
reconcileOnce(ctx) // finalizer
|
||||
reconcileOnce(ctx) // Deployment/Service created with resolved DB env
|
||||
|
||||
var deploy appsv1.Deployment
|
||||
@@ -795,11 +254,6 @@ var _ = Describe("TerdutServer Controller", func() {
|
||||
|
||||
Describe("spec.pod", func() {
|
||||
It("wires pod-level customization onto the right spot on the Deployment", func(ctx SpecContext) {
|
||||
fake, fakeSrv := newFakeTerdutServer()
|
||||
_ = fake
|
||||
DeferCleanup(fakeSrv.Close)
|
||||
reconciler.NewClient = func(string) *tdclient.Client { return tdclient.New(fakeSrv.URL) }
|
||||
|
||||
spec := dsnSpec()
|
||||
qty := resource.MustParse("250m")
|
||||
spec.Pod = terdutv1alpha1.PodSpec{
|
||||
@@ -811,8 +265,7 @@ var _ = Describe("TerdutServer Controller", func() {
|
||||
ExtraVolumeMounts: []corev1.VolumeMount{{Name: "extra-ca", MountPath: "/etc/extra-ca"}},
|
||||
}
|
||||
createServer(ctx, spec)
|
||||
reconcileOnce(ctx) // finalizer
|
||||
reconcileOnce(ctx) // Deployment/Service/PDB
|
||||
reconcileOnce(ctx)
|
||||
|
||||
var deploy appsv1.Deployment
|
||||
Expect(k8sClient.Get(ctx, objKey, &deploy)).To(Succeed())
|
||||
@@ -836,17 +289,11 @@ var _ = Describe("TerdutServer Controller", func() {
|
||||
|
||||
Describe("spec.pod.disruptionBudget", func() {
|
||||
It("creates an owned PodDisruptionBudget when set, and deletes it once cleared", func(ctx SpecContext) {
|
||||
fake, fakeSrv := newFakeTerdutServer()
|
||||
_ = fake
|
||||
DeferCleanup(fakeSrv.Close)
|
||||
reconciler.NewClient = func(string) *tdclient.Client { return tdclient.New(fakeSrv.URL) }
|
||||
|
||||
spec := dsnSpec()
|
||||
minAvail := intstr.FromInt32(1)
|
||||
spec.Pod.DisruptionBudget = &terdutv1alpha1.PodDisruptionBudgetSpec{MinAvailable: &minAvail}
|
||||
createServer(ctx, spec)
|
||||
reconcileOnce(ctx) // finalizer
|
||||
reconcileOnce(ctx) // Deployment/Service/PDB
|
||||
reconcileOnce(ctx)
|
||||
|
||||
var pdb policyv1.PodDisruptionBudget
|
||||
Expect(k8sClient.Get(ctx, objKey, &pdb)).To(Succeed())
|
||||
@@ -882,133 +329,6 @@ var _ = Describe("TerdutServer Controller", func() {
|
||||
})
|
||||
})
|
||||
|
||||
Describe("deletion", func() {
|
||||
// runToReady brings a server to Ready against a fresh fake and
|
||||
// returns the name of the instance credential Secret it minted.
|
||||
runToReady := func(ctx SpecContext, spec terdutv1alpha1.TerdutServerSpec) string {
|
||||
_, fakeSrv := newFakeTerdutServer()
|
||||
DeferCleanup(fakeSrv.Close)
|
||||
reconciler.NewClient = func(string) *tdclient.Client { return tdclient.New(fakeSrv.URL) }
|
||||
|
||||
createServer(ctx, spec)
|
||||
reconcileOnce(ctx)
|
||||
reconcileOnce(ctx)
|
||||
markDeploymentReady(ctx)
|
||||
reconcileOnce(ctx)
|
||||
|
||||
srv := &terdutv1alpha1.TerdutServer{}
|
||||
Expect(k8sClient.Get(ctx, objKey, srv)).To(Succeed())
|
||||
return srv.Status.CredentialsSecretRef.Name
|
||||
}
|
||||
deleteAndFinalize := func(ctx SpecContext) {
|
||||
srv := &terdutv1alpha1.TerdutServer{}
|
||||
Expect(k8sClient.Get(ctx, objKey, srv)).To(Succeed())
|
||||
Expect(k8sClient.Delete(ctx, srv)).To(Succeed())
|
||||
reconcileOnce(ctx) // runs the finalizer
|
||||
Expect(k8sClient.Get(ctx, objKey, srv)).NotTo(Succeed(),
|
||||
"the TerdutServer itself should be gone once the finalizer clears")
|
||||
}
|
||||
secretExists := func(ctx SpecContext, secretName string) bool {
|
||||
var s corev1.Secret
|
||||
err := k8sClient.Get(ctx, types.NamespacedName{Name: secretName, Namespace: operatorNamespace}, &s)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
It("keeps the instance credential by default and removes the checkpoint", func(ctx SpecContext) {
|
||||
credsName := runToReady(ctx, dsnSpec())
|
||||
// A checkpoint left behind, as if the best-effort delete had failed.
|
||||
Expect(k8sClient.Create(ctx, &corev1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: checkpointSecretNameFor(name), Namespace: operatorNamespace},
|
||||
Data: map[string][]byte{credentialsSecretDataKey: []byte("admin-key-raw")},
|
||||
})).To(Succeed())
|
||||
|
||||
deleteAndFinalize(ctx)
|
||||
|
||||
Expect(secretExists(ctx, credsName)).To(BeTrue(),
|
||||
"the credential is kept so a recreated TerdutServer can adopt it")
|
||||
Expect(secretExists(ctx, checkpointSecretNameFor(name))).To(BeFalse(),
|
||||
"the checkpoint is a short-lived admin key and is always removed")
|
||||
})
|
||||
|
||||
It("removes the credentials and checkpoint Secrets and the finalizer when the policy is Delete", func(ctx SpecContext) {
|
||||
spec := dsnSpec()
|
||||
spec.Credentials.DeletionPolicy = terdutv1alpha1.CredentialsDelete
|
||||
credsName := runToReady(ctx, spec)
|
||||
|
||||
deleteAndFinalize(ctx)
|
||||
|
||||
Expect(secretExists(ctx, credsName)).To(BeFalse(),
|
||||
"the credentials Secret should have been cleaned up by the finalizer")
|
||||
})
|
||||
})
|
||||
|
||||
Describe("recreating a TerdutServer against an already-bootstrapped server", func() {
|
||||
// retainedSecret stands in for what the previous TerdutServer left.
|
||||
retainedSecret := func(ctx SpecContext, token string) {
|
||||
Expect(k8sClient.Create(ctx, &corev1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: credentialsSecretNameFor(name), Namespace: operatorNamespace},
|
||||
Data: map[string][]byte{credentialsSecretDataKey: []byte(token)},
|
||||
})).To(Succeed())
|
||||
}
|
||||
bringUp := func(ctx SpecContext, fakeSrv *httptest.Server) {
|
||||
DeferCleanup(fakeSrv.Close)
|
||||
reconciler.NewClient = func(string) *tdclient.Client { return tdclient.New(fakeSrv.URL) }
|
||||
createServer(ctx, dsnSpec())
|
||||
reconcileOnce(ctx) // finalizer
|
||||
reconcileOnce(ctx) // Deployment/Service
|
||||
markDeploymentReady(ctx)
|
||||
reconcileOnce(ctx)
|
||||
}
|
||||
credsToken := func(ctx SpecContext) string {
|
||||
var s corev1.Secret
|
||||
Expect(k8sClient.Get(ctx, types.NamespacedName{Name: credentialsSecretNameFor(name), Namespace: operatorNamespace}, &s)).To(Succeed())
|
||||
return string(s.Data[credentialsSecretDataKey])
|
||||
}
|
||||
|
||||
It("adopts the retained credential when the server still accepts it, without bootstrapping", func(ctx SpecContext) {
|
||||
fake, fakeSrv := newFakeTerdutServer()
|
||||
fake.bootstrapped = true // every /api/bootstrap call 403s
|
||||
fake.accounts[serviceAccountName] = 7
|
||||
retainedSecret(ctx, "retained-key")
|
||||
|
||||
bringUp(ctx, fakeSrv)
|
||||
|
||||
Expect(readyCondition(ctx).Status).To(Equal(metav1.ConditionTrue))
|
||||
srv := &terdutv1alpha1.TerdutServer{}
|
||||
Expect(k8sClient.Get(ctx, objKey, srv)).To(Succeed())
|
||||
Expect(srv.Status.CredentialsSecretRef).NotTo(BeNil())
|
||||
Expect(srv.Status.CredentialsSecretRef.Name).To(Equal(credentialsSecretNameFor(name)))
|
||||
Expect(credsToken(ctx)).To(Equal("retained-key"), "the retained key is reused, not replaced")
|
||||
})
|
||||
|
||||
It("ignores a retained credential the server rejects and bootstraps afresh after a database reset", func(ctx SpecContext) {
|
||||
fake, fakeSrv := newFakeTerdutServer() // a fresh database: bootstrap succeeds
|
||||
fake.rejectedTokens = map[string]bool{"stale-key": true}
|
||||
retainedSecret(ctx, "stale-key")
|
||||
|
||||
bringUp(ctx, fakeSrv)
|
||||
|
||||
Expect(readyCondition(ctx).Status).To(Equal(metav1.ConditionTrue))
|
||||
Expect(credsToken(ctx)).NotTo(Equal("stale-key"), "the stale credential is replaced by a freshly minted one")
|
||||
})
|
||||
|
||||
It("fails closed, naming the Secret to restore, when the server is bootstrapped and the retained credential is rejected", func(ctx SpecContext) {
|
||||
fake, fakeSrv := newFakeTerdutServer()
|
||||
fake.bootstrapped = true
|
||||
fake.rejectedTokens = map[string]bool{"stale-key": true}
|
||||
retainedSecret(ctx, "stale-key")
|
||||
|
||||
bringUp(ctx, fakeSrv)
|
||||
|
||||
cond := readyCondition(ctx)
|
||||
Expect(cond.Status).To(Equal(metav1.ConditionFalse))
|
||||
Expect(cond.Reason).To(Equal(terdutv1alpha1.ReasonBootstrapStateLost))
|
||||
Expect(cond.Message).To(ContainSubstring(credentialsSecretNameFor(name)),
|
||||
"the message names the Secret that would restore it")
|
||||
Expect(cond.Message).To(ContainSubstring("does not clear the database"))
|
||||
})
|
||||
})
|
||||
|
||||
When("the TerdutServer object no longer exists", func() {
|
||||
It("returns no error (deleted between enqueue and reconcile)", func(ctx SpecContext) {
|
||||
_, err := reconciler.Reconcile(ctx, reconcile.Request{
|
||||
@@ -1018,13 +338,3 @@ var _ = Describe("TerdutServer Controller", func() {
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
// checkpointSecretNameFor/credentialsSecretNameFor let AfterEach clean up
|
||||
// without needing a live TerdutServer object (it may already be gone by
|
||||
// then in the deletion test).
|
||||
func checkpointSecretNameFor(name string) string {
|
||||
return fmt.Sprintf("default.%s-bootstrap-admin", name)
|
||||
}
|
||||
func credentialsSecretNameFor(name string) string {
|
||||
return fmt.Sprintf("default.%s-instance-credentials", name)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
|
||||
|
||||
terdutv1alpha1 "git.ryuvia.com/niklas/terdut-operator/api/v1alpha1"
|
||||
)
|
||||
|
||||
// operatorKeyDataKey is the data key of the operator key Secret.
|
||||
const operatorKeyDataKey = "token"
|
||||
|
||||
// operatorKeyPrefix marks the key as a service-account credential in logs, the
|
||||
// way terdut-server's own generated ones are.
|
||||
const operatorKeyPrefix = "tdsa_"
|
||||
|
||||
// operatorKeySecretName is the Secret holding a TerdutServer's operator key. It
|
||||
// lives beside the Deployment because the pod mounts it (TERDUT_OPERATOR_KEY)
|
||||
// and a pod can only reference Secrets of its own namespace; it is owned by the
|
||||
// TerdutServer, so deleting that deletes the key and a recreated one starts
|
||||
// with a fresh one the server re-seeds on its next start.
|
||||
func operatorKeySecretName(srv *terdutv1alpha1.TerdutServer) string {
|
||||
return srv.Name + "-operator-key"
|
||||
}
|
||||
|
||||
// reconcileOperatorKey returns this server's operator key, generating the Secret
|
||||
// on first use. An existing Secret is never overwritten: the running server was
|
||||
// seeded with that value, and replacing it would lock the operator out until
|
||||
// every pod restarted.
|
||||
func (r *TerdutServerReconciler) reconcileOperatorKey(ctx context.Context, srv *terdutv1alpha1.TerdutServer) (string, error) {
|
||||
var secret corev1.Secret
|
||||
key := client.ObjectKey{Namespace: srv.Namespace, Name: operatorKeySecretName(srv)}
|
||||
err := r.Get(ctx, key, &secret)
|
||||
if err == nil {
|
||||
if v := string(secret.Data[operatorKeyDataKey]); v != "" {
|
||||
return v, nil
|
||||
}
|
||||
// Present but empty or foreign: treat as ours to fill rather than
|
||||
// failing every reconcile on it.
|
||||
} else if !apierrors.IsNotFound(err) {
|
||||
return "", err
|
||||
}
|
||||
|
||||
raw := make([]byte, 24)
|
||||
if _, err := rand.Read(raw); err != nil {
|
||||
return "", fmt.Errorf("generating operator key: %w", err)
|
||||
}
|
||||
value := operatorKeyPrefix + hex.EncodeToString(raw)
|
||||
|
||||
secret = corev1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: key.Name, Namespace: key.Namespace},
|
||||
Data: map[string][]byte{operatorKeyDataKey: []byte(value)},
|
||||
}
|
||||
if err := controllerutil.SetControllerReference(srv, &secret, r.Scheme); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := r.Create(ctx, &secret); err != nil {
|
||||
if apierrors.IsAlreadyExists(err) {
|
||||
// Lost a race with a stale cache read: the next reconcile finds it.
|
||||
return "", fmt.Errorf("operator key Secret %s appeared during creation; retrying", key.Name)
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
// operatorKeyHash is a short digest of the key, stamped on the pod template so
|
||||
// a replaced Secret rolls the Deployment: the server only reads the env var at
|
||||
// start.
|
||||
func operatorKeyHash(value string) string {
|
||||
sum := sha256.Sum256([]byte(value))
|
||||
return hex.EncodeToString(sum[:8])
|
||||
}
|
||||
|
||||
// operatorKeyRef is the reference status.credentialsSecretRef reports.
|
||||
func operatorKeyRef(srv *terdutv1alpha1.TerdutServer) *terdutv1alpha1.SecretKeyRef {
|
||||
return &terdutv1alpha1.SecretKeyRef{Name: operatorKeySecretName(srv), Key: operatorKeyDataKey}
|
||||
}
|
||||
|
||||
// readOperatorKey reads a TerdutServer's key from its own namespace, for the
|
||||
// controllers that call the server's API.
|
||||
func readOperatorKey(ctx context.Context, c client.Client, srv *terdutv1alpha1.TerdutServer) (string, error) {
|
||||
var secret corev1.Secret
|
||||
if err := c.Get(ctx, client.ObjectKey{Namespace: srv.Namespace, Name: operatorKeySecretName(srv)}, &secret); err != nil {
|
||||
return "", err
|
||||
}
|
||||
v := string(secret.Data[operatorKeyDataKey])
|
||||
if v == "" {
|
||||
return "", fmt.Errorf("operator key Secret %s/%s is empty", srv.Namespace, secret.Name)
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package controller
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"maps"
|
||||
"strings"
|
||||
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
@@ -31,7 +32,7 @@ func labelsFor(srv *terdutv1alpha1.TerdutServer) map[string]string {
|
||||
// object (not just the desired one) so the caller can check
|
||||
// status.readyReplicas.
|
||||
func (r *TerdutServerReconciler) reconcileDeployment(
|
||||
ctx context.Context, srv *terdutv1alpha1.TerdutServer, dbEnv []corev1.EnvVar,
|
||||
ctx context.Context, srv *terdutv1alpha1.TerdutServer, dbEnv []corev1.EnvVar, keyHash string,
|
||||
) (*appsv1.Deployment, error) {
|
||||
deploy := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: srv.Name, Namespace: srv.Namespace}}
|
||||
_, err := controllerutil.CreateOrUpdate(ctx, r.Client, deploy, func() error {
|
||||
@@ -60,13 +61,8 @@ func (r *TerdutServerReconciler) reconcileDeployment(
|
||||
deploy.Spec.Strategy = appsv1.DeploymentStrategy{Type: appsv1.RollingUpdateDeploymentStrategyType}
|
||||
pod := srv.Spec.Pod
|
||||
deploy.Spec.Template = corev1.PodTemplateSpec{
|
||||
// pod.Annotations is assigned directly, not merged -- nothing
|
||||
// else sets pod-template annotations today. If a future change
|
||||
// needs the controller to set one of its own (e.g. a
|
||||
// Prometheus-scrape annotation), this needs to become a real
|
||||
// map merge with a stated precedence rather than silently
|
||||
// clobbering one side.
|
||||
ObjectMeta: metav1.ObjectMeta{Labels: labels, Annotations: pod.Annotations},
|
||||
// The controller's own annotation wins over a same-named user one.
|
||||
ObjectMeta: metav1.ObjectMeta{Labels: labels, Annotations: podAnnotations(pod.Annotations, keyHash)},
|
||||
Spec: corev1.PodSpec{
|
||||
EnableServiceLinks: new(false),
|
||||
NodeSelector: pod.NodeSelector,
|
||||
@@ -127,6 +123,17 @@ func (r *TerdutServerReconciler) reconcileService(ctx context.Context, srv *terd
|
||||
return err
|
||||
}
|
||||
|
||||
// operatorKeyHashAnnotation records which operator key the pods were started
|
||||
// with, so replacing the key's Secret rolls them.
|
||||
const operatorKeyHashAnnotation = "terdut.ryuvia.com/operator-key-hash"
|
||||
|
||||
func podAnnotations(user map[string]string, keyHash string) map[string]string {
|
||||
out := make(map[string]string, len(user)+1)
|
||||
maps.Copy(out, user)
|
||||
out[operatorKeyHashAnnotation] = keyHash
|
||||
return out
|
||||
}
|
||||
|
||||
func servicePort(srv *terdutv1alpha1.TerdutServer) int32 {
|
||||
if srv.Spec.Networking.ServicePort == 0 {
|
||||
return 8080
|
||||
@@ -194,13 +201,6 @@ func buildEnv(srv *terdutv1alpha1.TerdutServer, dbEnv []corev1.EnvVar) []corev1.
|
||||
corev1.EnvVar{Name: "TERDUT_ARCHIVE_AFTER", Value: sweeper.ArchiveAfter},
|
||||
)
|
||||
|
||||
deadman := srv.Spec.Deadman
|
||||
env = append(env,
|
||||
corev1.EnvVar{Name: "TERDUT_DEADMAN_MATCHERS", Value: deadman.Matchers},
|
||||
corev1.EnvVar{Name: "TERDUT_DEADMAN_TIMEOUT", Value: deadman.Timeout},
|
||||
corev1.EnvVar{Name: "TERDUT_DEADMAN_SEVERITY", Value: deadman.Severity},
|
||||
)
|
||||
|
||||
notify := srv.Spec.Notify
|
||||
if notify.NtfyURL != "" {
|
||||
env = append(env,
|
||||
@@ -230,6 +230,9 @@ func buildEnv(srv *terdutv1alpha1.TerdutServer, dbEnv []corev1.EnvVar) []corev1.
|
||||
// here on" -- which is unconditionally true for anything this
|
||||
// operator creates).
|
||||
corev1.EnvVar{Name: "TERDUT_OPERATOR_MODE", Value: "true"},
|
||||
// The credential this operator calls the API with; the server
|
||||
// creates or re-keys its instance-scoped account from it at start.
|
||||
corev1.EnvVar{Name: "TERDUT_OPERATOR_KEY", ValueFrom: secretEnvSource(operatorKeyRef(srv))},
|
||||
)
|
||||
|
||||
if oidc := srv.Spec.OIDC; oidc.Enabled {
|
||||
@@ -238,14 +241,10 @@ func buildEnv(srv *terdutv1alpha1.TerdutServer, dbEnv []corev1.EnvVar) []corev1.
|
||||
corev1.EnvVar{Name: "TERDUT_OIDC_CLIENT_ID", Value: oidc.ClientID},
|
||||
corev1.EnvVar{Name: "TERDUT_OIDC_NAME", Value: oidc.Name},
|
||||
corev1.EnvVar{Name: "TERDUT_OIDC_SCOPES", Value: oidc.Scopes},
|
||||
// terdut-server's own defaults for the claims/trust-email knobs
|
||||
// the chart exposes but DESIGN.md's spec doesn't (§4.1's doc
|
||||
// comment on OIDCSpec) -- not configurable here, not an
|
||||
// oversight.
|
||||
corev1.EnvVar{Name: "TERDUT_OIDC_USERNAME_CLAIM", Value: "preferred_username"},
|
||||
corev1.EnvVar{Name: "TERDUT_OIDC_EMAIL_CLAIM", Value: "email"},
|
||||
corev1.EnvVar{Name: "TERDUT_OIDC_GROUPS_CLAIM", Value: "groups"},
|
||||
corev1.EnvVar{Name: "TERDUT_OIDC_TRUST_EMAIL", Value: "false"},
|
||||
corev1.EnvVar{Name: "TERDUT_OIDC_USERNAME_CLAIM", Value: oidc.UsernameClaim},
|
||||
corev1.EnvVar{Name: "TERDUT_OIDC_EMAIL_CLAIM", Value: oidc.EmailClaim},
|
||||
corev1.EnvVar{Name: "TERDUT_OIDC_GROUPS_CLAIM", Value: oidc.GroupsClaim},
|
||||
corev1.EnvVar{Name: "TERDUT_OIDC_TRUST_EMAIL", Value: boolString(oidc.TrustEmail)},
|
||||
corev1.EnvVar{Name: "TERDUT_OIDC_SESSION_MAX_AGE", Value: oidc.SessionMaxAge},
|
||||
)
|
||||
if oidc.ClientSecretRef != nil {
|
||||
|
||||
@@ -1,84 +0,0 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
terdutv1alpha1 "git.ryuvia.com/niklas/terdut-operator/api/v1alpha1"
|
||||
"git.ryuvia.com/niklas/terdut-operator/internal/tdclient"
|
||||
)
|
||||
|
||||
// createOrAdoptTeam calls POST /api/teams with the TerdutServer's
|
||||
// instance-scoped credential, or, if an earlier interrupted attempt
|
||||
// already created this name (409), adopts it via GET /api/teams?name=
|
||||
// (TEAM-LOOKUP.md) rather than treating the conflict as an error --
|
||||
// DESIGN.md §5's general adopt-on-conflict rule, the same shape
|
||||
// TerdutServer's own bootstrap flow uses for minting its instance account.
|
||||
func (r *TerdutTeamReconciler) createOrAdoptTeam(ctx context.Context, team *terdutv1alpha1.TerdutTeam, instanceClient *tdclient.Client) error {
|
||||
created, err := instanceClient.CreateTeam(ctx, team.Spec.DisplayName)
|
||||
if err == nil {
|
||||
team.Status.TeamID = created.ID
|
||||
return nil
|
||||
}
|
||||
|
||||
statusErr, ok := errors.AsType[*tdclient.StatusError](err)
|
||||
if !ok || statusErr.Code != http.StatusConflict {
|
||||
return fmt.Errorf("POST /api/teams: %w", err)
|
||||
}
|
||||
|
||||
found, err := instanceClient.GetTeamByName(ctx, team.Spec.DisplayName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("GET /api/teams?name=%s (adopting after 409): %w", team.Spec.DisplayName, err)
|
||||
}
|
||||
if found == nil {
|
||||
// Genuinely pathological, not just a narrow crash window: the name
|
||||
// was taken a moment ago and isn't now. Surfaced as a plain error
|
||||
// (standard requeue-with-backoff) rather than a dedicated
|
||||
// condition -- there's no documented recovery to point at that
|
||||
// differs from "try again".
|
||||
return fmt.Errorf("POST /api/teams 409'd for %q but GET found nothing", team.Spec.DisplayName)
|
||||
}
|
||||
team.Status.TeamID = found.ID
|
||||
return nil
|
||||
}
|
||||
|
||||
// mintTeamCredential mints this team's own team-scoped service account,
|
||||
// using the TerdutServer's instance-scoped credential (DESIGN.md §6 point
|
||||
// 3: an instance-scoped caller may do this against any team). Adopts via
|
||||
// GET+mint-new-key on a 409, the same pattern TerdutServer's own bootstrap
|
||||
// flow uses.
|
||||
func (r *TerdutTeamReconciler) mintTeamCredential(ctx context.Context, team *terdutv1alpha1.TerdutTeam, instanceClient *tdclient.Client) error {
|
||||
saName := teamServiceAccountName(team)
|
||||
|
||||
result, err := instanceClient.CreateTeamServiceAccount(ctx, saName, team.Status.TeamID)
|
||||
var key string
|
||||
if err == nil {
|
||||
key = result.Key.Key
|
||||
} else {
|
||||
statusErr, ok := errors.AsType[*tdclient.StatusError](err)
|
||||
if !ok || statusErr.Code != http.StatusConflict {
|
||||
return fmt.Errorf("POST /api/service-accounts (team scope): %w", err)
|
||||
}
|
||||
sa, err := instanceClient.GetServiceAccountByName(ctx, saName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("GET /api/service-accounts?name=%s (adopting after 409): %w", saName, err)
|
||||
}
|
||||
if sa == nil {
|
||||
return fmt.Errorf("POST /api/service-accounts 409'd for %q but GET found nothing", saName)
|
||||
}
|
||||
minted, err := instanceClient.CreateServiceAccountKey(ctx, sa.ID, "initial")
|
||||
if err != nil {
|
||||
return fmt.Errorf("POST /api/service-accounts/%d/keys (adopting after 409): %w", sa.ID, err)
|
||||
}
|
||||
key = minted.Key
|
||||
}
|
||||
|
||||
credsName := teamCredentialsSecretName(team)
|
||||
if err := writeOperatorSecret(ctx, r.Client, r.OperatorNamespace, credsName, key); err != nil {
|
||||
return err
|
||||
}
|
||||
team.Status.CredentialsSecretRef = &terdutv1alpha1.SecretKeyRef{Name: credsName, Key: credentialsSecretDataKey}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
terdutv1alpha1 "git.ryuvia.com/niklas/terdut-operator/api/v1alpha1"
|
||||
"git.ryuvia.com/niklas/terdut-operator/internal/tdclient"
|
||||
)
|
||||
|
||||
// reconcileEscalation applies spec.escalation as the team's whole ladder, or
|
||||
// clears it when the field is absent (an empty ladder is the server's "none").
|
||||
// The second return is an expected, reportable condition (a bad duration, an
|
||||
// unknown user), the third a failure to retry.
|
||||
func (r *TerdutTeamReconciler) reconcileEscalation(
|
||||
ctx context.Context, tc *tdclient.Client, team *terdutv1alpha1.TerdutTeam,
|
||||
) (*teamError, error) {
|
||||
body, err := buildEscalationRequest(team.Spec.Escalation)
|
||||
if err != nil {
|
||||
return &teamError{reason: terdutv1alpha1.ReasonInvalidSpec, message: err.Error()}, nil
|
||||
}
|
||||
if err := tc.SetEscalation(ctx, team.Status.TeamID, body); err != nil {
|
||||
// The server resolves usernames; one it does not know is a state to
|
||||
// wait out (the person may be created later), not a failure.
|
||||
if se, ok := errors.AsType[*tdclient.StatusError](err); ok &&
|
||||
se.Code == http.StatusBadRequest && strings.HasPrefix(se.Message, "unknown user") {
|
||||
return &teamError{reason: terdutv1alpha1.ReasonUnknownUser, message: se.Message}, nil
|
||||
}
|
||||
return nil, fmt.Errorf("PUT /api/teams/%d/escalation: %w", team.Status.TeamID, err)
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func buildEscalationRequest(spec *terdutv1alpha1.EscalationSpec) (tdclient.SetEscalationRequest, error) {
|
||||
if spec == nil {
|
||||
return tdclient.SetEscalationRequest{Levels: []tdclient.EscalationLevelRequest{}}, nil
|
||||
}
|
||||
levels := make([]tdclient.EscalationLevelRequest, len(spec.Levels))
|
||||
for i, lvl := range spec.Levels {
|
||||
timeout, err := time.ParseDuration(lvl.Timeout)
|
||||
if err != nil || timeout <= 0 {
|
||||
return tdclient.SetEscalationRequest{}, fmt.Errorf("spec.escalation.levels[%d].timeout %q is not a positive duration", i, lvl.Timeout)
|
||||
}
|
||||
targets := make([]tdclient.EscalationTargetRequest, len(lvl.Targets))
|
||||
for j, t := range lvl.Targets {
|
||||
targets[j] = tdclient.EscalationTargetRequest{Kind: string(t.Kind), Username: t.Username}
|
||||
}
|
||||
levels[i] = tdclient.EscalationLevelRequest{
|
||||
Position: int64(i + 1),
|
||||
TimeoutSeconds: int64(timeout.Seconds()),
|
||||
Targets: targets,
|
||||
}
|
||||
}
|
||||
return tdclient.SetEscalationRequest{
|
||||
RepeatCount: spec.RepeatCount,
|
||||
FallbackTopic: spec.FallbackTopic,
|
||||
Levels: levels,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// reconcileDeadmanSwitches makes the team's switches on the server exactly
|
||||
// spec.deadmanSwitches, matched by name (unique per team server-side): create
|
||||
// what is missing, update what differs, delete what is not listed. In operator
|
||||
// mode nobody else can add one, so anything extra is leftover to remove.
|
||||
func (r *TerdutTeamReconciler) reconcileDeadmanSwitches(
|
||||
ctx context.Context, tc *tdclient.Client, team *terdutv1alpha1.TerdutTeam,
|
||||
) (*teamError, error) {
|
||||
type want struct {
|
||||
spec terdutv1alpha1.DeadmanSwitchSpec
|
||||
seconds int64
|
||||
}
|
||||
desired := make(map[string]want, len(team.Spec.DeadmanSwitches))
|
||||
for _, sw := range team.Spec.DeadmanSwitches {
|
||||
timeout, err := time.ParseDuration(sw.Timeout)
|
||||
if err != nil || timeout <= 0 {
|
||||
return &teamError{
|
||||
reason: terdutv1alpha1.ReasonInvalidSpec,
|
||||
message: fmt.Sprintf("spec.deadmanSwitches[%q].timeout %q is not a positive duration", sw.Name, sw.Timeout),
|
||||
}, nil
|
||||
}
|
||||
desired[sw.Name] = want{spec: sw, seconds: int64(timeout.Seconds())}
|
||||
}
|
||||
|
||||
existing, err := tc.ListDeadmanSwitches(ctx, team.Status.TeamID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("GET /api/teams/%d/deadman/switches: %w", team.Status.TeamID, err)
|
||||
}
|
||||
seen := make(map[string]bool, len(existing))
|
||||
for _, have := range existing {
|
||||
w, keep := desired[have.Name]
|
||||
if !keep {
|
||||
if err := tc.DeleteDeadmanSwitch(ctx, team.Status.TeamID, have.ID); err != nil {
|
||||
return nil, fmt.Errorf("DELETE deadman switch %q: %w", have.Name, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
seen[have.Name] = true
|
||||
severity := severityOrDefault(w.spec.Severity)
|
||||
if have.Matcher == w.spec.Matcher && have.TimeoutSeconds == w.seconds && have.Severity == severity {
|
||||
continue
|
||||
}
|
||||
if err := tc.UpdateDeadmanSwitch(ctx, team.Status.TeamID, have.ID, w.spec.Name, w.spec.Matcher, w.seconds, severity); err != nil {
|
||||
return nil, fmt.Errorf("PUT deadman switch %q: %w", have.Name, err)
|
||||
}
|
||||
}
|
||||
for _, sw := range team.Spec.DeadmanSwitches {
|
||||
if seen[sw.Name] {
|
||||
continue
|
||||
}
|
||||
w := desired[sw.Name]
|
||||
if _, err := tc.CreateDeadmanSwitch(ctx, team.Status.TeamID, sw.Name, sw.Matcher, w.seconds, severityOrDefault(sw.Severity)); err != nil {
|
||||
return nil, fmt.Errorf("POST deadman switch %q: %w", sw.Name, err)
|
||||
}
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func severityOrDefault(s string) string {
|
||||
if s == "" {
|
||||
return "critical"
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
@@ -15,33 +14,25 @@ import (
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
|
||||
"sigs.k8s.io/controller-runtime/pkg/handler"
|
||||
logf "sigs.k8s.io/controller-runtime/pkg/log"
|
||||
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
||||
"sigs.k8s.io/controller-runtime/pkg/recorder"
|
||||
|
||||
terdutv1alpha1 "git.ryuvia.com/niklas/terdut-operator/api/v1alpha1"
|
||||
"git.ryuvia.com/niklas/terdut-operator/internal/tdclient"
|
||||
)
|
||||
|
||||
// teamFinalizerName cleans up the team-scoped credentials Secret this
|
||||
// controller generates in the operator's own namespace on delete.
|
||||
// teamFinalizerName deletes the team on the server before the CR goes.
|
||||
const teamFinalizerName = "terdut.ryuvia.com/terdutteam"
|
||||
|
||||
// TerdutTeamReconciler reconciles a TerdutTeam object.
|
||||
//
|
||||
// Every child resolves its own teamRef/serverRef independently and never
|
||||
// chains up through another controller (DESIGN.md §5) — this one talks
|
||||
// directly to the TerdutServer it references and to terdut-server's API,
|
||||
// never to TerdutServerReconciler.
|
||||
// TerdutTeamReconciler reconciles a TerdutTeam: the team itself plus its
|
||||
// escalation ladder and dead man's switches, all through the referenced
|
||||
// TerdutServer's operator key.
|
||||
type TerdutTeamReconciler struct {
|
||||
client.Client
|
||||
Scheme *runtime.Scheme
|
||||
|
||||
// OperatorNamespace is where every credentials Secret this controller
|
||||
// reads (the referenced TerdutServer's) or writes (this team's own)
|
||||
// lives (DESIGN.md §6) — never a TerdutTeam's or TerdutServer's own
|
||||
// namespace.
|
||||
OperatorNamespace string
|
||||
|
||||
Recorder recorder.EventRecorder
|
||||
NewClient func(endpoint string) *tdclient.Client
|
||||
}
|
||||
@@ -50,10 +41,16 @@ type TerdutTeamReconciler struct {
|
||||
// +kubebuilder:rbac:groups=terdut.ryuvia.com,resources=terdutteams/status,verbs=get;update;patch
|
||||
// +kubebuilder:rbac:groups=terdut.ryuvia.com,resources=terdutteams/finalizers,verbs=update
|
||||
// +kubebuilder:rbac:groups=terdut.ryuvia.com,resources=terdutservers,verbs=get;list;watch
|
||||
// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch;delete
|
||||
// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch
|
||||
// +kubebuilder:rbac:groups="",resources=namespaces,verbs=get;list;watch
|
||||
// +kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=create;patch
|
||||
|
||||
// teamExternalID is the identity this CR gives its team on the server, so the
|
||||
// team is found by who owns it and not by its (editable, global) display name.
|
||||
func teamExternalID(team *terdutv1alpha1.TerdutTeam) string {
|
||||
return team.Namespace + "/" + team.Name
|
||||
}
|
||||
|
||||
func (r *TerdutTeamReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
|
||||
log := logf.FromContext(ctx)
|
||||
|
||||
@@ -79,67 +76,56 @@ func (r *TerdutTeamReconciler) Reconcile(ctx context.Context, req ctrl.Request)
|
||||
|
||||
srv, resolveErr := r.resolveServerRef(ctx, &team)
|
||||
if resolveErr != nil {
|
||||
return r.setTeamNotReady(ctx, &team, resolveErr.reason, resolveErr.message, waitInterval)
|
||||
return r.setTeamNotReady(ctx, &team, resolveErr.reason, resolveErr.message)
|
||||
}
|
||||
if srv.Status.CredentialsSecretRef == nil {
|
||||
if !meta.IsStatusConditionTrue(srv.Status.Conditions, terdutv1alpha1.ConditionReady) {
|
||||
return r.setTeamNotReady(ctx, &team, terdutv1alpha1.ReasonWaitingForServer,
|
||||
fmt.Sprintf("TerdutServer %q is not Bootstrapped yet", srv.Name), waitInterval)
|
||||
fmt.Sprintf("TerdutServer %q is not Ready yet", srv.Name))
|
||||
}
|
||||
|
||||
newClient := r.NewClient
|
||||
if newClient == nil {
|
||||
newClient = tdclient.New
|
||||
}
|
||||
|
||||
instanceKey, err := readOperatorSecret(ctx, r.Client, r.OperatorNamespace, srv.Status.CredentialsSecretRef)
|
||||
tc, err := r.serverClient(ctx, srv)
|
||||
if err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
endpoint := serviceURL(srv)
|
||||
team.Status.ServerEndpoint = endpoint
|
||||
instanceClient := newClient(endpoint).WithToken(instanceKey)
|
||||
|
||||
if team.Status.TeamID == 0 {
|
||||
if err := r.createOrAdoptTeam(ctx, &team, instanceClient); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
}
|
||||
|
||||
if team.Status.CredentialsSecretRef == nil {
|
||||
if err := r.mintTeamCredential(ctx, &team, instanceClient); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
}
|
||||
|
||||
teamKey, err := readOperatorSecret(ctx, r.Client, r.OperatorNamespace, team.Status.CredentialsSecretRef)
|
||||
// Idempotent on the external id: the same team comes back whether this is
|
||||
// the first reconcile, a retry after a lost status write, or a repair after
|
||||
// somebody deleted the team behind our back.
|
||||
created, err := tc.CreateTeam(ctx, team.Spec.DisplayName, teamExternalID(&team))
|
||||
if err != nil {
|
||||
return ctrl.Result{}, err
|
||||
if se, ok := errors.AsType[*tdclient.StatusError](err); ok && se.Code == http.StatusConflict {
|
||||
return r.setTeamNotReady(ctx, &team, terdutv1alpha1.ReasonTeamNameTaken, nameTakenMessage(&team))
|
||||
}
|
||||
return ctrl.Result{}, fmt.Errorf("POST /api/teams: %w", err)
|
||||
}
|
||||
teamClient := newClient(endpoint).WithToken(teamKey)
|
||||
team.Status.TeamID = created.ID
|
||||
|
||||
// Owner-gated on terdut-server, so this always runs with the
|
||||
// team-scoped credential just minted above, never the instance-scoped
|
||||
// one used to create the team (internal/api/teams.go's requireTeamOwner
|
||||
// has no branch for an instance-scoped service account, confirmed
|
||||
// against source). Applied unconditionally rather than diffed against a
|
||||
// stored "last-applied" value: both calls are idempotent PUTs of the
|
||||
// whole resource, the same "cheap because it's small" reasoning §5
|
||||
// already applies to the escalation policy's whole-policy PUT.
|
||||
if err := teamClient.RenameTeam(ctx, team.Status.TeamID, team.Spec.DisplayName); err != nil {
|
||||
return ctrl.Result{}, fmt.Errorf("PUT /api/teams/%d: %w", team.Status.TeamID, err)
|
||||
if err := tc.RenameTeam(ctx, created.ID, team.Spec.DisplayName); err != nil {
|
||||
if se, ok := errors.AsType[*tdclient.StatusError](err); ok && se.Code == http.StatusConflict {
|
||||
return r.setTeamNotReady(ctx, &team, terdutv1alpha1.ReasonTeamNameTaken, nameTakenMessage(&team))
|
||||
}
|
||||
return ctrl.Result{}, fmt.Errorf("PUT /api/teams/%d: %w", created.ID, err)
|
||||
}
|
||||
if err := teamClient.SetTeamOIDCGroups(ctx, team.Status.TeamID, team.Spec.OIDC.MemberGroup, team.Spec.OIDC.OwnerGroup); err != nil {
|
||||
return ctrl.Result{}, fmt.Errorf("PUT /api/teams/%d/oidc-groups: %w", team.Status.TeamID, err)
|
||||
if err := tc.SetTeamOIDCGroups(ctx, created.ID, team.Spec.OIDC.MemberGroup, team.Spec.OIDC.OwnerGroup); err != nil {
|
||||
return ctrl.Result{}, fmt.Errorf("PUT /api/teams/%d/oidc-groups: %w", created.ID, err)
|
||||
}
|
||||
if err := r.reconcileInvite(ctx, &team, teamClient); err != nil {
|
||||
|
||||
if condErr, err := r.reconcileEscalation(ctx, tc, &team); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
} else if condErr != nil {
|
||||
return r.setTeamNotReady(ctx, &team, condErr.reason, condErr.message)
|
||||
}
|
||||
if condErr, err := r.reconcileDeadmanSwitches(ctx, tc, &team); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
} else if condErr != nil {
|
||||
return r.setTeamNotReady(ctx, &team, condErr.reason, condErr.message)
|
||||
}
|
||||
|
||||
meta.SetStatusCondition(&team.Status.Conditions, metav1.Condition{
|
||||
Type: terdutv1alpha1.ConditionReady,
|
||||
Status: metav1.ConditionTrue,
|
||||
Reason: terdutv1alpha1.ReasonTeamAdopted,
|
||||
Message: fmt.Sprintf("team %d ready, credentials in Secret %q", team.Status.TeamID, team.Status.CredentialsSecretRef.Name),
|
||||
Message: fmt.Sprintf("team %d applied", team.Status.TeamID),
|
||||
})
|
||||
team.Status.ObservedGeneration = team.Generation
|
||||
if err := r.Status().Update(ctx, &team); err != nil {
|
||||
@@ -147,13 +133,31 @@ func (r *TerdutTeamReconciler) Reconcile(ctx context.Context, req ctrl.Request)
|
||||
}
|
||||
if r.Recorder != nil {
|
||||
r.Recorder.Eventf(&team, nil, corev1.EventTypeNormal, terdutv1alpha1.ReasonTeamAdopted, terdutv1alpha1.ReasonTeamAdopted,
|
||||
"team ready")
|
||||
"team applied")
|
||||
}
|
||||
log.Info("TerdutTeam ready", "name", team.Name, "teamID", team.Status.TeamID)
|
||||
|
||||
return ctrl.Result{RequeueAfter: resyncInterval}, nil
|
||||
}
|
||||
|
||||
func nameTakenMessage(team *terdutv1alpha1.TerdutTeam) string {
|
||||
return fmt.Sprintf("the server already has a different team named %q; choose another spec.displayName, "+
|
||||
"or remove that team", team.Spec.DisplayName)
|
||||
}
|
||||
|
||||
// serverClient builds a client for srv, authenticated with its operator key.
|
||||
func (r *TerdutTeamReconciler) serverClient(ctx context.Context, srv *terdutv1alpha1.TerdutServer) (*tdclient.Client, error) {
|
||||
key, err := readOperatorKey(ctx, r.Client, srv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
newClient := r.NewClient
|
||||
if newClient == nil {
|
||||
newClient = tdclient.New
|
||||
}
|
||||
return newClient(serviceURL(srv)).WithToken(key), nil
|
||||
}
|
||||
|
||||
// teamError carries a condition reason/message, the same role databaseError
|
||||
// plays for TerdutServer: an expected, requeue-and-retry outcome, not a
|
||||
// reconcile failure.
|
||||
@@ -168,6 +172,21 @@ func (e *teamError) Error() string { return e.message }
|
||||
// consent check (DESIGN.md §4.6) when serverRef.namespace differs from this
|
||||
// TerdutTeam's own.
|
||||
func (r *TerdutTeamReconciler) resolveServerRef(ctx context.Context, team *terdutv1alpha1.TerdutTeam) (*terdutv1alpha1.TerdutServer, *teamError) {
|
||||
srv, ns, getErr := r.getServerRef(ctx, team)
|
||||
if getErr != nil {
|
||||
return nil, getErr
|
||||
}
|
||||
if ns == team.Namespace {
|
||||
return srv, nil
|
||||
}
|
||||
return r.checkConsent(ctx, team, srv, ns)
|
||||
}
|
||||
|
||||
// getServerRef fetches the referenced TerdutServer and the namespace it was
|
||||
// looked up in, without the consent check. Deleting a team uses it directly:
|
||||
// consent gates what the operator will start acting on, not whether it may
|
||||
// clean up after itself once the server owner has narrowed allowedTeams.
|
||||
func (r *TerdutTeamReconciler) getServerRef(ctx context.Context, team *terdutv1alpha1.TerdutTeam) (*terdutv1alpha1.TerdutServer, string, *teamError) {
|
||||
ns := team.Spec.ServerRef.Namespace
|
||||
if ns == "" {
|
||||
ns = team.Namespace
|
||||
@@ -176,18 +195,19 @@ func (r *TerdutTeamReconciler) resolveServerRef(ctx context.Context, team *terdu
|
||||
var srv terdutv1alpha1.TerdutServer
|
||||
if err := r.Get(ctx, client.ObjectKey{Namespace: ns, Name: team.Spec.ServerRef.Name}, &srv); err != nil {
|
||||
if apierrors.IsNotFound(err) {
|
||||
return nil, &teamError{
|
||||
return nil, ns, &teamError{
|
||||
reason: terdutv1alpha1.ReasonServerRefNotFound,
|
||||
message: fmt.Sprintf("TerdutServer %q not found in namespace %q", team.Spec.ServerRef.Name, ns),
|
||||
}
|
||||
}
|
||||
return nil, &teamError{reason: terdutv1alpha1.ReasonServerRefNotFound, message: err.Error()}
|
||||
}
|
||||
|
||||
if ns == team.Namespace {
|
||||
return &srv, nil
|
||||
return nil, ns, &teamError{reason: terdutv1alpha1.ReasonServerRefNotFound, message: err.Error()}
|
||||
}
|
||||
return &srv, ns, nil
|
||||
}
|
||||
|
||||
// checkConsent applies DESIGN.md §4.6's allowedTeams gate to a cross-namespace
|
||||
// reference.
|
||||
func (r *TerdutTeamReconciler) checkConsent(ctx context.Context, team *terdutv1alpha1.TerdutTeam, srv *terdutv1alpha1.TerdutServer, ns string) (*terdutv1alpha1.TerdutServer, *teamError) {
|
||||
var ownNamespace corev1.Namespace
|
||||
if err := r.Get(ctx, client.ObjectKey{Name: team.Namespace}, &ownNamespace); err != nil {
|
||||
return nil, &teamError{
|
||||
@@ -206,11 +226,11 @@ func (r *TerdutTeamReconciler) resolveServerRef(ctx context.Context, team *terdu
|
||||
ns, team.Spec.ServerRef.Name, team.Namespace),
|
||||
}
|
||||
}
|
||||
return &srv, nil
|
||||
return srv, nil
|
||||
}
|
||||
|
||||
func (r *TerdutTeamReconciler) setTeamNotReady(
|
||||
ctx context.Context, team *terdutv1alpha1.TerdutTeam, reason, message string, d time.Duration,
|
||||
ctx context.Context, team *terdutv1alpha1.TerdutTeam, reason, message string,
|
||||
) (ctrl.Result, error) {
|
||||
meta.SetStatusCondition(&team.Status.Conditions, metav1.Condition{
|
||||
Type: terdutv1alpha1.ConditionReady,
|
||||
@@ -225,42 +245,18 @@ func (r *TerdutTeamReconciler) setTeamNotReady(
|
||||
if r.Recorder != nil {
|
||||
r.Recorder.Eventf(team, nil, corev1.EventTypeWarning, reason, reason, message)
|
||||
}
|
||||
return ctrl.Result{RequeueAfter: d}, nil
|
||||
return ctrl.Result{RequeueAfter: waitInterval}, nil
|
||||
}
|
||||
|
||||
// teamCredentialsSecretName/teamServiceAccountName follow the same
|
||||
// <namespace>.<name>-suffix convention TerdutServer's secrets use
|
||||
// (DESIGN.md §6 point 3), keyed on the TerdutTeam CR's own identity rather
|
||||
// than its (mutable) displayName -- a service account's own name is
|
||||
// globally unique across the whole install (internal/db/migrations/
|
||||
// 014_service_accounts.sql's UNIQUE constraint, confirmed against source),
|
||||
// so this has to be collision-safe the same way the credentials Secret
|
||||
// names already are.
|
||||
func teamCredentialsSecretName(team *terdutv1alpha1.TerdutTeam) string {
|
||||
return fmt.Sprintf("%s.%s-team-credentials", team.Namespace, team.Name)
|
||||
}
|
||||
|
||||
func teamServiceAccountName(team *terdutv1alpha1.TerdutTeam) string {
|
||||
return fmt.Sprintf("terdut-team.%s.%s", team.Namespace, team.Name)
|
||||
}
|
||||
|
||||
// reconcileTeamDelete cleans up the team-scoped credentials Secret and, if
|
||||
// one was ever minted, deletes the team server-side first. If
|
||||
// status.teamID or status.credentialsSecretRef was never set (the CR was
|
||||
// deleted before reconciliation ever got that far), there is deliberately
|
||||
// no attempt to clean up server-side: terdut-server's DELETE /api/teams/
|
||||
// {teamID} is owner-gated (requireTeamOwner), and an instance-scoped
|
||||
// credential -- the only one this controller would otherwise hold -- does
|
||||
// not satisfy that check (confirmed against source, same finding as
|
||||
// TEAM-LOOKUP.md's). A team created but never fully reconciled to Ready is
|
||||
// left orphaned server-side for a human with real owner/admin access to
|
||||
// clean up -- a known, documented limitation, not a silent gap.
|
||||
// reconcileTeamDelete removes the team on the server, then the finalizer. The
|
||||
// server refuses while the team has open incidents (409), which surfaces as a
|
||||
// retried error: tidying up must not be how a live page disappears.
|
||||
func (r *TerdutTeamReconciler) reconcileTeamDelete(ctx context.Context, team *terdutv1alpha1.TerdutTeam) (ctrl.Result, error) {
|
||||
if !controllerutil.ContainsFinalizer(team, teamFinalizerName) {
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
|
||||
if team.Status.TeamID != 0 && team.Status.CredentialsSecretRef != nil {
|
||||
if team.Status.TeamID != 0 {
|
||||
if err := r.deleteTeamServerSide(ctx, team); err != nil {
|
||||
if r.Recorder != nil {
|
||||
r.Recorder.Eventf(team, nil, corev1.EventTypeWarning, "DeleteFailed", "DeleteFailed", err.Error())
|
||||
@@ -269,40 +265,33 @@ func (r *TerdutTeamReconciler) reconcileTeamDelete(ctx context.Context, team *te
|
||||
}
|
||||
}
|
||||
|
||||
secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: teamCredentialsSecretName(team), Namespace: r.OperatorNamespace}}
|
||||
if err := r.Delete(ctx, secret); err != nil && !apierrors.IsNotFound(err) {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
|
||||
controllerutil.RemoveFinalizer(team, teamFinalizerName)
|
||||
return ctrl.Result{}, r.Update(ctx, team)
|
||||
}
|
||||
|
||||
func (r *TerdutTeamReconciler) deleteTeamServerSide(ctx context.Context, team *terdutv1alpha1.TerdutTeam) error {
|
||||
srv, resolveErr := r.resolveServerRef(ctx, team)
|
||||
if resolveErr != nil {
|
||||
// The TerdutServer (or the namespace consent for it) is gone too --
|
||||
// most likely the whole install is being torn down together.
|
||||
// Nothing to delete against; proceed rather than block forever on
|
||||
// a parent that no longer exists.
|
||||
return nil
|
||||
}
|
||||
teamKey, err := readOperatorSecret(ctx, r.Client, r.OperatorNamespace, team.Status.CredentialsSecretRef)
|
||||
if err != nil {
|
||||
// Consent is deliberately not checked here: it gates what the operator
|
||||
// starts acting on, not whether it may clean up after itself once the
|
||||
// server owner has narrowed allowedTeams. Only a TerdutServer that is
|
||||
// really gone lets the delete proceed (there is nothing to delete against);
|
||||
// any other failure is retried rather than orphaning the team.
|
||||
srv, ns, getErr := r.getServerRef(ctx, team)
|
||||
if getErr != nil {
|
||||
var probe terdutv1alpha1.TerdutServer
|
||||
err := r.Get(ctx, client.ObjectKey{Namespace: ns, Name: team.Spec.ServerRef.Name}, &probe)
|
||||
if apierrors.IsNotFound(err) {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("reading TerdutServer %q/%q: %w", ns, team.Spec.ServerRef.Name, getErr)
|
||||
}
|
||||
tc, err := r.serverClient(ctx, srv)
|
||||
if err != nil {
|
||||
if apierrors.IsNotFound(err) {
|
||||
return nil // the key Secret went with its TerdutServer
|
||||
}
|
||||
return err
|
||||
}
|
||||
newClient := r.NewClient
|
||||
if newClient == nil {
|
||||
newClient = tdclient.New
|
||||
}
|
||||
err = newClient(serviceURL(srv)).WithToken(teamKey).DeleteTeam(ctx, team.Status.TeamID)
|
||||
if statusErr, ok := errors.AsType[*tdclient.StatusError](err); ok && statusErr.Code == http.StatusNotFound {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
return tc.DeleteTeam(ctx, team.Status.TeamID)
|
||||
}
|
||||
|
||||
// SetupWithManager sets up the controller with the Manager.
|
||||
@@ -315,6 +304,28 @@ func (r *TerdutTeamReconciler) SetupWithManager(mgr ctrl.Manager) error {
|
||||
}
|
||||
return ctrl.NewControllerManagedBy(mgr).
|
||||
For(&terdutv1alpha1.TerdutTeam{}).
|
||||
Watches(&terdutv1alpha1.TerdutServer{}, handler.EnqueueRequestsFromMapFunc(r.teamsForServer)).
|
||||
Named("terdutteam").
|
||||
Complete(r)
|
||||
}
|
||||
|
||||
// teamsForServer re-reconciles every TerdutTeam that references a TerdutServer
|
||||
// when it changes (becomes Ready, say), instead of polling for it.
|
||||
func (r *TerdutTeamReconciler) teamsForServer(ctx context.Context, obj client.Object) []reconcile.Request {
|
||||
var teams terdutv1alpha1.TerdutTeamList
|
||||
if err := r.List(ctx, &teams); err != nil {
|
||||
return nil
|
||||
}
|
||||
var out []reconcile.Request
|
||||
for i := range teams.Items {
|
||||
t := &teams.Items[i]
|
||||
ns := t.Spec.ServerRef.Namespace
|
||||
if ns == "" {
|
||||
ns = t.Namespace
|
||||
}
|
||||
if t.Spec.ServerRef.Name == obj.GetName() && ns == obj.GetNamespace() {
|
||||
out = append(out, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(t)})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -2,9 +2,7 @@ package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http/httptest"
|
||||
"time"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
@@ -18,6 +16,12 @@ import (
|
||||
"git.ryuvia.com/niklas/terdut-operator/internal/tdclient"
|
||||
)
|
||||
|
||||
// platformName is the display name most specs give their team.
|
||||
const platformName = "platform"
|
||||
|
||||
// allNamespaces is allowedTeams.namespaces.from admitting every namespace.
|
||||
const allNamespaces = "All"
|
||||
|
||||
var _ = Describe("TerdutTeam Controller", func() {
|
||||
const operatorNamespace = "default"
|
||||
|
||||
@@ -34,13 +38,12 @@ var _ = Describe("TerdutTeam Controller", func() {
|
||||
fake, fakeSrv = newFakeTerdutServer()
|
||||
DeferCleanup(fakeSrv.Close)
|
||||
|
||||
srv = bootstrapReadyTerdutServer(ctx, uniqueName("ttserver"), fakeSrv.URL)
|
||||
srv = readyTerdutServer(ctx, uniqueName("ttserver"))
|
||||
|
||||
reconciler = &TerdutTeamReconciler{
|
||||
Client: k8sClient,
|
||||
Scheme: k8sClient.Scheme(),
|
||||
OperatorNamespace: operatorNamespace,
|
||||
NewClient: func(string) *tdclient.Client { return tdclient.New(fakeSrv.URL) },
|
||||
Client: k8sClient,
|
||||
Scheme: k8sClient.Scheme(),
|
||||
NewClient: func(string) *tdclient.Client { return tdclient.New(fakeSrv.URL) },
|
||||
}
|
||||
teamName = uniqueName("team")
|
||||
teamKey = types.NamespacedName{Name: teamName, Namespace: operatorNamespace}
|
||||
@@ -53,13 +56,7 @@ var _ = Describe("TerdutTeam Controller", func() {
|
||||
_ = k8sClient.Update(ctx, team)
|
||||
_ = k8sClient.Delete(ctx, team)
|
||||
}
|
||||
_ = k8sClient.Delete(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{
|
||||
Name: fmt.Sprintf("%s.%s-team-credentials", operatorNamespace, teamName), Namespace: operatorNamespace,
|
||||
}})
|
||||
|
||||
// The TerdutServer bootstrapReadyTerdutServer created in BeforeEach
|
||||
// carries its own finalizer; clear it directly the same way, rather
|
||||
// than relying on TerdutServerReconciler to ever run again here.
|
||||
srvKey := types.NamespacedName{Name: srv.Name, Namespace: operatorNamespace}
|
||||
if err := k8sClient.Get(ctx, srvKey, srv); err == nil {
|
||||
srv.Finalizers = nil
|
||||
@@ -67,7 +64,7 @@ var _ = Describe("TerdutTeam Controller", func() {
|
||||
_ = k8sClient.Delete(ctx, srv)
|
||||
}
|
||||
_ = k8sClient.Delete(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{
|
||||
Name: fmt.Sprintf("%s.%s-instance-credentials", operatorNamespace, srv.Name), Namespace: operatorNamespace,
|
||||
Name: srv.Name + "-operator-key", Namespace: operatorNamespace,
|
||||
}})
|
||||
})
|
||||
|
||||
@@ -96,39 +93,56 @@ var _ = Describe("TerdutTeam Controller", func() {
|
||||
return terdutv1alpha1.TerdutServerRef{Name: srv.Name}
|
||||
}
|
||||
|
||||
getTeam := func(ctx context.Context) *terdutv1alpha1.TerdutTeam {
|
||||
team := &terdutv1alpha1.TerdutTeam{}
|
||||
Expect(k8sClient.Get(ctx, teamKey, team)).To(Succeed())
|
||||
return team
|
||||
}
|
||||
|
||||
// reconcileToReady drives a freshly created team through the finalizer pass
|
||||
// and one apply pass.
|
||||
reconcileToReady := func(ctx context.Context) {
|
||||
reconcileOnce(ctx) // finalizer
|
||||
reconcileOnce(ctx) // apply
|
||||
}
|
||||
|
||||
Describe("the happy path", func() {
|
||||
It("creates the team, mints its credential, and applies rename/oidc-groups", func(ctx SpecContext) {
|
||||
createTeam(ctx, "platform", sameNSRef())
|
||||
reconcileOnce(ctx) // finalizer
|
||||
reconcileOnce(ctx) // full create+mint+apply
|
||||
It("creates the team under its CR identity with the operator key, and applies oidc groups", func(ctx SpecContext) {
|
||||
// The fake only answers a request carrying the server's operator key.
|
||||
var keySecret corev1.Secret
|
||||
Expect(k8sClient.Get(ctx, types.NamespacedName{Name: srv.Name + "-operator-key", Namespace: operatorNamespace}, &keySecret)).To(Succeed())
|
||||
fake.expectKey = string(keySecret.Data[operatorKeyDataKey])
|
||||
|
||||
team := &terdutv1alpha1.TerdutTeam{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: teamName, Namespace: operatorNamespace},
|
||||
Spec: terdutv1alpha1.TerdutTeamSpec{
|
||||
ServerRef: sameNSRef(), DisplayName: platformName,
|
||||
OIDC: terdutv1alpha1.TerdutTeamOIDC{MemberGroup: "members", OwnerGroup: "owners"},
|
||||
},
|
||||
}
|
||||
Expect(k8sClient.Create(ctx, team)).To(Succeed())
|
||||
reconcileToReady(ctx)
|
||||
|
||||
cond := readyCondition(ctx)
|
||||
Expect(cond.Status).To(Equal(metav1.ConditionTrue))
|
||||
Expect(cond.Reason).To(Equal(terdutv1alpha1.ReasonTeamAdopted))
|
||||
|
||||
team := &terdutv1alpha1.TerdutTeam{}
|
||||
Expect(k8sClient.Get(ctx, teamKey, team)).To(Succeed())
|
||||
Expect(team.Status.TeamID).To(Equal(fake.teams["platform"]))
|
||||
Expect(team.Status.CredentialsSecretRef).NotTo(BeNil())
|
||||
|
||||
var credsSecret corev1.Secret
|
||||
Expect(k8sClient.Get(ctx, types.NamespacedName{
|
||||
Name: team.Status.CredentialsSecretRef.Name, Namespace: operatorNamespace,
|
||||
}, &credsSecret)).To(Succeed())
|
||||
Expect(credsSecret.Data[credentialsSecretDataKey]).NotTo(BeEmpty())
|
||||
team = getTeam(ctx)
|
||||
Expect(team.Status.TeamID).To(Equal(fake.teams[platformName]))
|
||||
Expect(fake.teamExt).To(HaveKeyWithValue(operatorNamespace+"/"+teamName, team.Status.TeamID))
|
||||
Expect(fake.teamOIDC[team.Status.TeamID]).To(Equal([2]string{"members", "owners"}))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("waiting on the referenced TerdutServer", func() {
|
||||
It("reports ServerRefNotFound when the TerdutServer doesn't exist", func(ctx SpecContext) {
|
||||
createTeam(ctx, "orphan", terdutv1alpha1.TerdutServerRef{Name: testRefNotFoundName})
|
||||
reconcileOnce(ctx) // finalizer
|
||||
reconcileOnce(ctx)
|
||||
reconcileToReady(ctx)
|
||||
|
||||
Expect(readyCondition(ctx).Reason).To(Equal(terdutv1alpha1.ReasonServerRefNotFound))
|
||||
})
|
||||
|
||||
It("reports WaitingForServer when the TerdutServer exists but isn't Bootstrapped yet", func(ctx SpecContext) {
|
||||
It("reports WaitingForServer when the TerdutServer exists but isn't Ready yet", func(ctx SpecContext) {
|
||||
unreadyName := uniqueName("ttserver-unready")
|
||||
unready := &terdutv1alpha1.TerdutServer{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: unreadyName, Namespace: operatorNamespace},
|
||||
@@ -142,8 +156,7 @@ var _ = Describe("TerdutTeam Controller", func() {
|
||||
DeferCleanup(func() { _ = k8sClient.Delete(ctx, unready) })
|
||||
|
||||
createTeam(ctx, "waiting", terdutv1alpha1.TerdutServerRef{Name: unreadyName})
|
||||
reconcileOnce(ctx) // finalizer
|
||||
reconcileOnce(ctx)
|
||||
reconcileToReady(ctx)
|
||||
|
||||
Expect(readyCondition(ctx).Reason).To(Equal(terdutv1alpha1.ReasonWaitingForServer))
|
||||
})
|
||||
@@ -182,7 +195,7 @@ var _ = Describe("TerdutTeam Controller", func() {
|
||||
|
||||
It("permits when the TerdutServer's allowedTeams.namespaces.from is All", func(ctx SpecContext) {
|
||||
Expect(k8sClient.Get(ctx, types.NamespacedName{Name: srv.Name, Namespace: operatorNamespace}, srv)).To(Succeed())
|
||||
srv.Spec.AllowedTeams.Namespaces.From = "All"
|
||||
srv.Spec.AllowedTeams.Namespaces.From = allNamespaces
|
||||
Expect(k8sClient.Update(ctx, srv)).To(Succeed())
|
||||
|
||||
team := &terdutv1alpha1.TerdutTeam{
|
||||
@@ -209,154 +222,292 @@ var _ = Describe("TerdutTeam Controller", func() {
|
||||
})
|
||||
})
|
||||
|
||||
Describe("adopt-on-409 recovery", func() {
|
||||
It("adopts an already-created team instead of erroring", func(ctx SpecContext) {
|
||||
Describe("team identity", func() {
|
||||
It("refuses a name that belongs to a team this CR did not create", func(ctx SpecContext) {
|
||||
fake.nextTeamID = 1
|
||||
fake.teams["platform"] = 1
|
||||
fake.teamNames[1] = "platform"
|
||||
fake.teams[platformName] = 1
|
||||
fake.teamNames[1] = platformName
|
||||
|
||||
createTeam(ctx, "platform", sameNSRef())
|
||||
reconcileOnce(ctx) // finalizer
|
||||
reconcileOnce(ctx)
|
||||
createTeam(ctx, platformName, sameNSRef())
|
||||
reconcileToReady(ctx)
|
||||
|
||||
team := &terdutv1alpha1.TerdutTeam{}
|
||||
Expect(k8sClient.Get(ctx, teamKey, team)).To(Succeed())
|
||||
Expect(team.Status.TeamID).To(Equal(int64(1)))
|
||||
Expect(readyCondition(ctx).Status).To(Equal(metav1.ConditionTrue))
|
||||
cond := readyCondition(ctx)
|
||||
Expect(cond.Status).To(Equal(metav1.ConditionFalse))
|
||||
Expect(cond.Reason).To(Equal(terdutv1alpha1.ReasonTeamNameTaken))
|
||||
Expect(getTeam(ctx).Status.TeamID).To(BeZero())
|
||||
Expect(fake.teamOIDC).To(BeEmpty(), "nothing is configured on a team that is not ours")
|
||||
})
|
||||
|
||||
It("adopts an already-created team-scoped service account instead of erroring", func(ctx SpecContext) {
|
||||
createTeam(ctx, "platform", sameNSRef())
|
||||
reconcileOnce(ctx) // finalizer
|
||||
It("keeps the same team when spec.displayName changes", func(ctx SpecContext) {
|
||||
createTeam(ctx, platformName, sameNSRef())
|
||||
reconcileToReady(ctx)
|
||||
id := getTeam(ctx).Status.TeamID
|
||||
|
||||
// Pre-seed the service account the mint step is about to try to
|
||||
// create, simulating an attempt that got this far before being
|
||||
// interrupted.
|
||||
fake.nextID = 1
|
||||
saName := fmt.Sprintf("terdut-team.%s.%s", operatorNamespace, teamName)
|
||||
fake.accounts[saName] = 1
|
||||
fake.keyMints[1] = 1
|
||||
team := getTeam(ctx)
|
||||
team.Spec.DisplayName = "platform-renamed"
|
||||
Expect(k8sClient.Update(ctx, team)).To(Succeed())
|
||||
reconcileOnce(ctx)
|
||||
|
||||
Expect(getTeam(ctx).Status.TeamID).To(Equal(id))
|
||||
Expect(fake.teamNames[id]).To(Equal("platform-renamed"))
|
||||
Expect(fake.nextTeamID).To(Equal(id), "renamed in place, not recreated")
|
||||
})
|
||||
|
||||
It("reports TeamNameTaken when a rename collides with another team", func(ctx SpecContext) {
|
||||
createTeam(ctx, platformName, sameNSRef())
|
||||
reconcileToReady(ctx)
|
||||
fake.nextTeamID++
|
||||
fake.teams["other"] = fake.nextTeamID
|
||||
fake.teamNames[fake.nextTeamID] = "other"
|
||||
|
||||
team := getTeam(ctx)
|
||||
team.Spec.DisplayName = "other"
|
||||
Expect(k8sClient.Update(ctx, team)).To(Succeed())
|
||||
reconcileOnce(ctx)
|
||||
|
||||
Expect(readyCondition(ctx).Reason).To(Equal(terdutv1alpha1.ReasonTeamNameTaken))
|
||||
})
|
||||
|
||||
It("finds its team again when the status was lost", func(ctx SpecContext) {
|
||||
createTeam(ctx, platformName, sameNSRef())
|
||||
reconcileToReady(ctx)
|
||||
id := getTeam(ctx).Status.TeamID
|
||||
|
||||
team := getTeam(ctx)
|
||||
team.Status.TeamID = 0
|
||||
Expect(k8sClient.Status().Update(ctx, team)).To(Succeed())
|
||||
reconcileOnce(ctx)
|
||||
|
||||
Expect(getTeam(ctx).Status.TeamID).To(Equal(id))
|
||||
Expect(fake.nextTeamID).To(Equal(id), "no second team was created")
|
||||
})
|
||||
|
||||
It("recreates a team deleted behind its back", func(ctx SpecContext) {
|
||||
createTeam(ctx, platformName, sameNSRef())
|
||||
reconcileToReady(ctx)
|
||||
id := getTeam(ctx).Status.TeamID
|
||||
delete(fake.teams, platformName)
|
||||
delete(fake.teamNames, id)
|
||||
delete(fake.teamExt, operatorNamespace+"/"+teamName)
|
||||
|
||||
reconcileOnce(ctx)
|
||||
|
||||
Expect(getTeam(ctx).Status.TeamID).NotTo(Equal(id))
|
||||
Expect(readyCondition(ctx).Status).To(Equal(metav1.ConditionTrue))
|
||||
team := &terdutv1alpha1.TerdutTeam{}
|
||||
Expect(k8sClient.Get(ctx, teamKey, team)).To(Succeed())
|
||||
var credsSecret corev1.Secret
|
||||
Expect(k8sClient.Get(ctx, types.NamespacedName{
|
||||
Name: team.Status.CredentialsSecretRef.Name, Namespace: operatorNamespace,
|
||||
}, &credsSecret)).To(Succeed())
|
||||
// Minted fresh (mint2), not the pre-seeded account's original
|
||||
// (never-issued-to-this-reconcile) key.
|
||||
Expect(string(credsSecret.Data[credentialsSecretDataKey])).To(Equal("instance-key-1-mint2"))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("spec.invite", func() {
|
||||
It("mints a link into the TerdutTeam's own namespace, not the operator's", func(ctx SpecContext) {
|
||||
createTeam(ctx, "platform", sameNSRef())
|
||||
reconcileOnce(ctx) // finalizer
|
||||
reconcileOnce(ctx) // create+mint+apply
|
||||
Describe("spec.escalation", func() {
|
||||
ladder := func() *terdutv1alpha1.EscalationSpec {
|
||||
return &terdutv1alpha1.EscalationSpec{
|
||||
RepeatCount: 2, FallbackTopic: "oncall",
|
||||
Levels: []terdutv1alpha1.EscalationLevel{
|
||||
{Timeout: "5m", Targets: []terdutv1alpha1.EscalationTarget{{Kind: terdutv1alpha1.EscalationTargetOncall}}},
|
||||
{Timeout: "15m", Targets: []terdutv1alpha1.EscalationTarget{{Kind: terdutv1alpha1.EscalationTargetUser, Username: "alice"}}},
|
||||
},
|
||||
}
|
||||
}
|
||||
createWithLadder := func(ctx context.Context, e *terdutv1alpha1.EscalationSpec) {
|
||||
Expect(k8sClient.Create(ctx, &terdutv1alpha1.TerdutTeam{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: teamName, Namespace: operatorNamespace},
|
||||
Spec: terdutv1alpha1.TerdutTeamSpec{ServerRef: sameNSRef(), DisplayName: platformName, Escalation: e},
|
||||
})).To(Succeed())
|
||||
}
|
||||
|
||||
team := &terdutv1alpha1.TerdutTeam{}
|
||||
Expect(k8sClient.Get(ctx, teamKey, team)).To(Succeed())
|
||||
team.Spec.Invite.Enabled = true
|
||||
Expect(k8sClient.Update(ctx, team)).To(Succeed())
|
||||
reconcileOnce(ctx)
|
||||
It("PUTs the ladder with usernames for the server to resolve", func(ctx SpecContext) {
|
||||
createWithLadder(ctx, ladder())
|
||||
reconcileToReady(ctx)
|
||||
|
||||
Expect(k8sClient.Get(ctx, teamKey, team)).To(Succeed())
|
||||
Expect(team.Status.InviteSecretRef).NotTo(BeNil())
|
||||
|
||||
var secret corev1.Secret
|
||||
Expect(k8sClient.Get(ctx, types.NamespacedName{
|
||||
Name: team.Status.InviteSecretRef.Name, Namespace: team.Namespace,
|
||||
}, &secret)).To(Succeed())
|
||||
Expect(string(secret.Data[inviteSecretURLKey])).To(ContainSubstring("invite="))
|
||||
Expect(string(secret.Data[inviteSecretInviteIDKey])).To(Equal("1"))
|
||||
|
||||
inv := fake.invites[team.Status.TeamID][1]
|
||||
Expect(inv.Role).To(Equal("member"), "default role")
|
||||
Expect(inv.MaxUses).To(Equal(int64(1)), "default max uses")
|
||||
got := fake.escalation[getTeam(ctx).Status.TeamID]
|
||||
Expect(got.RepeatCount).To(Equal(int64(2)))
|
||||
Expect(got.FallbackTopic).To(Equal("oncall"))
|
||||
Expect(got.Levels).To(HaveLen(2))
|
||||
Expect(got.Levels[0].TimeoutSeconds).To(Equal(int64(300)))
|
||||
Expect(got.Levels[1].Targets[0]).To(Equal(tdclient.EscalationTargetRequest{Kind: "user", Username: "alice"}))
|
||||
})
|
||||
|
||||
It("refreshes a link that's within a day of terdut-server's 7-day TTL", func(ctx SpecContext) {
|
||||
createTeam(ctx, "platform", sameNSRef())
|
||||
It("waits with UnknownUser while the server does not know a username", func(ctx SpecContext) {
|
||||
fake.unknownUsers["alice"] = true
|
||||
createWithLadder(ctx, ladder())
|
||||
reconcileToReady(ctx)
|
||||
|
||||
cond := readyCondition(ctx)
|
||||
Expect(cond.Reason).To(Equal(terdutv1alpha1.ReasonUnknownUser))
|
||||
Expect(cond.Message).To(ContainSubstring("alice"))
|
||||
|
||||
delete(fake.unknownUsers, "alice")
|
||||
reconcileOnce(ctx)
|
||||
reconcileOnce(ctx)
|
||||
|
||||
team := &terdutv1alpha1.TerdutTeam{}
|
||||
Expect(k8sClient.Get(ctx, teamKey, team)).To(Succeed())
|
||||
team.Spec.Invite.Enabled = true
|
||||
Expect(k8sClient.Update(ctx, team)).To(Succeed())
|
||||
reconcileOnce(ctx)
|
||||
|
||||
Expect(k8sClient.Get(ctx, teamKey, team)).To(Succeed())
|
||||
firstSecretName := team.Status.InviteSecretRef.Name
|
||||
var secret corev1.Secret
|
||||
Expect(k8sClient.Get(ctx, types.NamespacedName{Name: firstSecretName, Namespace: team.Namespace}, &secret)).To(Succeed())
|
||||
|
||||
// Simulate the stored link being within the refresh window of
|
||||
// expiry, the way it genuinely would be six days from now,
|
||||
// without the test waiting six days.
|
||||
secret.Data[inviteSecretExpiresAtKey] = []byte(time.Now().Add(12 * time.Hour).Format(time.RFC3339)) // inside inviteRefreshWindow
|
||||
Expect(k8sClient.Update(ctx, &secret)).To(Succeed())
|
||||
|
||||
reconcileOnce(ctx)
|
||||
|
||||
Expect(fake.inviteDelete[1]).To(BeTrue(), "the stale invite should have been revoked")
|
||||
Expect(fake.invites[team.Status.TeamID]).To(HaveKey(int64(2)), "a replacement should have been minted")
|
||||
Expect(readyCondition(ctx).Status).To(Equal(metav1.ConditionTrue))
|
||||
})
|
||||
|
||||
It("revokes the invite when spec.invite.enabled flips back to false", func(ctx SpecContext) {
|
||||
createTeam(ctx, "platform", sameNSRef())
|
||||
reconcileOnce(ctx)
|
||||
reconcileOnce(ctx)
|
||||
It("clears the ladder when spec.escalation is removed", func(ctx SpecContext) {
|
||||
createWithLadder(ctx, ladder())
|
||||
reconcileToReady(ctx)
|
||||
|
||||
team := &terdutv1alpha1.TerdutTeam{}
|
||||
Expect(k8sClient.Get(ctx, teamKey, team)).To(Succeed())
|
||||
team.Spec.Invite.Enabled = true
|
||||
team := getTeam(ctx)
|
||||
team.Spec.Escalation = nil
|
||||
Expect(k8sClient.Update(ctx, team)).To(Succeed())
|
||||
reconcileOnce(ctx)
|
||||
|
||||
Expect(k8sClient.Get(ctx, teamKey, team)).To(Succeed())
|
||||
secretName := team.Status.InviteSecretRef.Name
|
||||
team.Spec.Invite.Enabled = false
|
||||
Expect(fake.escalation[team.Status.TeamID].Levels).To(BeEmpty())
|
||||
})
|
||||
|
||||
It("reports InvalidSpec for a duration that does not parse", func(ctx SpecContext) {
|
||||
bad := ladder()
|
||||
bad.Levels[0].Timeout = "5 minutes"
|
||||
// The CRD pattern rejects it at admission when the API server enforces it;
|
||||
// envtest does, so write it past the pattern by creating a valid one first.
|
||||
bad.Levels[0].Timeout = "5m"
|
||||
createWithLadder(ctx, bad)
|
||||
reconcileToReady(ctx)
|
||||
team := getTeam(ctx)
|
||||
team.Spec.Escalation.Levels[0].Timeout = "0s"
|
||||
Expect(k8sClient.Update(ctx, team)).To(Succeed())
|
||||
reconcileOnce(ctx)
|
||||
|
||||
Expect(fake.inviteDelete[1]).To(BeTrue())
|
||||
Expect(k8sClient.Get(ctx, teamKey, team)).To(Succeed())
|
||||
Expect(team.Status.InviteSecretRef).To(BeNil())
|
||||
Expect(readyCondition(ctx).Reason).To(Equal(terdutv1alpha1.ReasonInvalidSpec))
|
||||
})
|
||||
})
|
||||
|
||||
var secret corev1.Secret
|
||||
err := k8sClient.Get(ctx, types.NamespacedName{Name: secretName, Namespace: team.Namespace}, &secret)
|
||||
Expect(err).To(HaveOccurred(), "the invite Secret should have been deleted")
|
||||
Describe("spec.deadmanSwitches", func() {
|
||||
sw := func(name, matcher, timeout string) terdutv1alpha1.DeadmanSwitchSpec {
|
||||
return terdutv1alpha1.DeadmanSwitchSpec{Name: name, Matcher: matcher, Timeout: timeout, Severity: "critical"}
|
||||
}
|
||||
setSwitches := func(ctx context.Context, switches ...terdutv1alpha1.DeadmanSwitchSpec) {
|
||||
team := getTeam(ctx)
|
||||
team.Spec.DeadmanSwitches = switches
|
||||
Expect(k8sClient.Update(ctx, team)).To(Succeed())
|
||||
reconcileOnce(ctx)
|
||||
}
|
||||
names := func(teamID int64) []string {
|
||||
out := make([]string, 0, len(fake.switches[teamID]))
|
||||
for _, s := range fake.switches[teamID] {
|
||||
out = append(out, s.Name)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
It("creates, updates and prunes switches by name", func(ctx SpecContext) {
|
||||
createTeam(ctx, platformName, sameNSRef())
|
||||
reconcileToReady(ctx)
|
||||
id := getTeam(ctx).Status.TeamID
|
||||
|
||||
setSwitches(ctx, sw("watchdog", "alertname=Watchdog", "15m"), sw("edge", "alertname=Edge", "5m"))
|
||||
Expect(names(id)).To(ConsistOf("watchdog", "edge"))
|
||||
|
||||
// Same name, new timeout: updated in place, same id.
|
||||
var watchdogID int64
|
||||
for _, s := range fake.switches[id] {
|
||||
if s.Name == "watchdog" {
|
||||
watchdogID = s.ID
|
||||
}
|
||||
}
|
||||
setSwitches(ctx, sw("watchdog", "alertname=Watchdog", "30m"))
|
||||
Expect(names(id)).To(ConsistOf("watchdog"))
|
||||
Expect(fake.switches[id][watchdogID].TimeoutSeconds).To(Equal(int64(1800)))
|
||||
Expect(readyCondition(ctx).Status).To(Equal(metav1.ConditionTrue))
|
||||
})
|
||||
|
||||
It("removes a switch somebody added on the server", func(ctx SpecContext) {
|
||||
createTeam(ctx, platformName, sameNSRef())
|
||||
reconcileToReady(ctx)
|
||||
id := getTeam(ctx).Status.TeamID
|
||||
fake.nextSwitchID++
|
||||
fake.switches[id] = map[int64]tdclient.DeadmanSwitch{
|
||||
fake.nextSwitchID: {ID: fake.nextSwitchID, Name: "stray", Matcher: "alertname=X", TimeoutSeconds: 60, Severity: "critical"},
|
||||
}
|
||||
|
||||
reconcileOnce(ctx)
|
||||
|
||||
Expect(fake.switches[id]).To(BeEmpty())
|
||||
})
|
||||
|
||||
It("recreates a switch deleted on the server", func(ctx SpecContext) {
|
||||
createTeam(ctx, platformName, sameNSRef())
|
||||
reconcileToReady(ctx)
|
||||
id := getTeam(ctx).Status.TeamID
|
||||
setSwitches(ctx, sw("watchdog", "alertname=Watchdog", "15m"))
|
||||
fake.switches[id] = map[int64]tdclient.DeadmanSwitch{}
|
||||
|
||||
reconcileOnce(ctx)
|
||||
|
||||
Expect(names(id)).To(ConsistOf("watchdog"))
|
||||
})
|
||||
|
||||
It("reports InvalidSpec for a timeout that does not parse", func(ctx SpecContext) {
|
||||
createTeam(ctx, platformName, sameNSRef())
|
||||
reconcileToReady(ctx)
|
||||
setSwitches(ctx, sw("watchdog", "alertname=Watchdog", "0s"))
|
||||
|
||||
Expect(readyCondition(ctx).Reason).To(Equal(terdutv1alpha1.ReasonInvalidSpec))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("deletion", func() {
|
||||
It("deletes the team server-side and removes the credentials Secret", func(ctx SpecContext) {
|
||||
It("deletes the team on the server and clears the finalizer", func(ctx SpecContext) {
|
||||
createTeam(ctx, "to-delete", sameNSRef())
|
||||
reconcileOnce(ctx)
|
||||
reconcileOnce(ctx)
|
||||
Expect(readyCondition(ctx).Status).To(Equal(metav1.ConditionTrue))
|
||||
reconcileToReady(ctx)
|
||||
teamID := getTeam(ctx).Status.TeamID
|
||||
|
||||
team := &terdutv1alpha1.TerdutTeam{}
|
||||
Expect(k8sClient.Get(ctx, teamKey, team)).To(Succeed())
|
||||
teamID := team.Status.TeamID
|
||||
credsName := team.Status.CredentialsSecretRef.Name
|
||||
|
||||
Expect(k8sClient.Delete(ctx, team)).To(Succeed())
|
||||
Expect(k8sClient.Delete(ctx, getTeam(ctx))).To(Succeed())
|
||||
reconcileOnce(ctx) // runs the finalizer
|
||||
|
||||
Expect(fake.teamDelete[teamID]).To(BeTrue())
|
||||
Expect(k8sClient.Get(ctx, teamKey, &terdutv1alpha1.TerdutTeam{})).NotTo(Succeed(),
|
||||
"the TerdutTeam should be gone once the finalizer clears")
|
||||
})
|
||||
|
||||
err := k8sClient.Get(ctx, teamKey, team)
|
||||
Expect(err).To(HaveOccurred(), "the TerdutTeam itself should be gone once the finalizer clears")
|
||||
It("still deletes the team after the server's allowedTeams consent is revoked", func(ctx SpecContext) {
|
||||
otherNS := uniqueName("ns")
|
||||
Expect(k8sClient.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: otherNS}})).To(Succeed())
|
||||
DeferCleanup(func() { _ = k8sClient.Delete(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: otherNS}}) })
|
||||
|
||||
var leftover corev1.Secret
|
||||
err = k8sClient.Get(ctx, types.NamespacedName{Name: credsName, Namespace: operatorNamespace}, &leftover)
|
||||
Expect(err).To(HaveOccurred(), "the team-credentials Secret should have been cleaned up")
|
||||
Expect(k8sClient.Get(ctx, types.NamespacedName{Name: srv.Name, Namespace: operatorNamespace}, srv)).To(Succeed())
|
||||
srv.Spec.AllowedTeams.Namespaces.From = allNamespaces
|
||||
Expect(k8sClient.Update(ctx, srv)).To(Succeed())
|
||||
|
||||
crossKey := types.NamespacedName{Name: teamName, Namespace: otherNS}
|
||||
Expect(k8sClient.Create(ctx, &terdutv1alpha1.TerdutTeam{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: teamName, Namespace: otherNS},
|
||||
Spec: terdutv1alpha1.TerdutTeamSpec{
|
||||
ServerRef: terdutv1alpha1.TerdutServerRef{Name: srv.Name, Namespace: operatorNamespace},
|
||||
DisplayName: "revoked",
|
||||
},
|
||||
})).To(Succeed())
|
||||
for range 2 {
|
||||
_, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: crossKey})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
}
|
||||
cross := &terdutv1alpha1.TerdutTeam{}
|
||||
Expect(k8sClient.Get(ctx, crossKey, cross)).To(Succeed())
|
||||
teamID := cross.Status.TeamID
|
||||
Expect(teamID).NotTo(BeZero())
|
||||
|
||||
Expect(k8sClient.Get(ctx, types.NamespacedName{Name: srv.Name, Namespace: operatorNamespace}, srv)).To(Succeed())
|
||||
srv.Spec.AllowedTeams.Namespaces.From = "None"
|
||||
Expect(k8sClient.Update(ctx, srv)).To(Succeed())
|
||||
|
||||
Expect(k8sClient.Delete(ctx, cross)).To(Succeed())
|
||||
_, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: crossKey})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
Expect(fake.teamDelete[teamID]).To(BeTrue(), "revoking consent must not orphan the team")
|
||||
})
|
||||
|
||||
It("lets the CR go when its TerdutServer is already gone", func(ctx SpecContext) {
|
||||
createTeam(ctx, "to-delete", sameNSRef())
|
||||
reconcileToReady(ctx)
|
||||
|
||||
serverKey := types.NamespacedName{Name: srv.Name, Namespace: operatorNamespace}
|
||||
Expect(k8sClient.Get(ctx, serverKey, srv)).To(Succeed())
|
||||
srv.Finalizers = nil
|
||||
Expect(k8sClient.Update(ctx, srv)).To(Succeed())
|
||||
Expect(k8sClient.Delete(ctx, srv)).To(Succeed())
|
||||
|
||||
Expect(k8sClient.Delete(ctx, getTeam(ctx))).To(Succeed())
|
||||
reconcileOnce(ctx)
|
||||
|
||||
Expect(k8sClient.Get(ctx, teamKey, &terdutv1alpha1.TerdutTeam{})).NotTo(Succeed())
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,149 +0,0 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
|
||||
|
||||
terdutv1alpha1 "git.ryuvia.com/niklas/terdut-operator/api/v1alpha1"
|
||||
"git.ryuvia.com/niklas/terdut-operator/internal/tdclient"
|
||||
)
|
||||
|
||||
// Data keys inside the generated invite Secret, following the same naming
|
||||
// shape as TerdutAlertSource's webhookSecret*Key constants.
|
||||
const (
|
||||
inviteSecretURLKey = "url"
|
||||
inviteSecretInviteIDKey = "inviteID"
|
||||
inviteSecretExpiresAtKey = "expiresAt"
|
||||
)
|
||||
|
||||
// inviteRefreshWindow is how far ahead of expiry this controller mints a
|
||||
// replacement link, so a human reading status.inviteSecretRef never finds a
|
||||
// dead link mid-use. terdut-server's invite TTL is a fixed, unconfigurable
|
||||
// 7 days (internal/api/signup.go's inviteTTL) -- refreshing a full day
|
||||
// ahead of that leaves comfortable margin against this controller's own
|
||||
// 5-minute resync interval ever being delayed.
|
||||
const inviteRefreshWindow = 24 * time.Hour
|
||||
|
||||
func inviteSecretName(team *terdutv1alpha1.TerdutTeam) string {
|
||||
return team.Name + "-terdut-invite"
|
||||
}
|
||||
|
||||
// reconcileInvite applies spec.invite against teamClient -- this team's own
|
||||
// team-scoped credential, already owner-equivalent for every /invites route
|
||||
// (terdut-server's SERVICE-ACCOUNTS.md, ratified not accidental). Mints,
|
||||
// refreshes ahead of expiry, or revokes, entirely independent of this
|
||||
// team's own Ready condition: an invite is a convenience for onboarding a
|
||||
// human, never something anything else in this reconcile waits on.
|
||||
func (r *TerdutTeamReconciler) reconcileInvite(ctx context.Context, team *terdutv1alpha1.TerdutTeam, teamClient *tdclient.Client) error {
|
||||
if !team.Spec.Invite.Enabled {
|
||||
return r.revokeInvite(ctx, team, teamClient)
|
||||
}
|
||||
|
||||
secretName := inviteSecretName(team)
|
||||
var secret corev1.Secret
|
||||
err := r.Get(ctx, client.ObjectKey{Namespace: team.Namespace, Name: secretName}, &secret)
|
||||
switch {
|
||||
case err == nil:
|
||||
expiresAt, parseErr := time.Parse(time.RFC3339, string(secret.Data[inviteSecretExpiresAtKey]))
|
||||
if parseErr == nil && time.Until(expiresAt) > inviteRefreshWindow {
|
||||
team.Status.InviteSecretRef = &terdutv1alpha1.LocalSecretRef{Name: secretName}
|
||||
return nil // still fresh, nothing to do this reconcile
|
||||
}
|
||||
// Expired, about to expire, or unreadable: mint a replacement.
|
||||
// Revoke the old row by id first (best-effort) so a leaked old link
|
||||
// stops working immediately rather than lingering unrevoked until
|
||||
// its own TTL -- failure here is not fatal, since the replacement
|
||||
// below is what actually matters.
|
||||
if oldID, idErr := strconv.ParseInt(string(secret.Data[inviteSecretInviteIDKey]), 10, 64); idErr == nil {
|
||||
_ = teamClient.RevokeInvite(ctx, team.Status.TeamID, oldID)
|
||||
}
|
||||
return r.mintInvite(ctx, team, teamClient, secretName)
|
||||
case apierrors.IsNotFound(err):
|
||||
// Low stakes, unlike TerdutAlertSource's webhook URL: nothing
|
||||
// external holds a durable dependency on one specific invite link
|
||||
// staying stable the way an Alertmanager config depends on a
|
||||
// webhook URL -- it's read once by one human and handed out. So
|
||||
// this silently re-mints rather than failing closed the way
|
||||
// TerdutAlertSource's ReasonWebhookSecretLost does for its Secret.
|
||||
return r.mintInvite(ctx, team, teamClient, secretName)
|
||||
default:
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
func (r *TerdutTeamReconciler) mintInvite(
|
||||
ctx context.Context, team *terdutv1alpha1.TerdutTeam, teamClient *tdclient.Client, secretName string,
|
||||
) error {
|
||||
role := team.Spec.Invite.Role
|
||||
if role == "" {
|
||||
role = "member"
|
||||
}
|
||||
maxUses := team.Spec.Invite.MaxUses
|
||||
if maxUses == 0 {
|
||||
maxUses = 1
|
||||
}
|
||||
inv, err := teamClient.CreateInvite(ctx, team.Status.TeamID, role, maxUses)
|
||||
if err != nil {
|
||||
return fmt.Errorf("POST /api/teams/%d/invites: %w", team.Status.TeamID, err)
|
||||
}
|
||||
|
||||
secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: secretName, Namespace: team.Namespace}}
|
||||
if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, secret, func() error {
|
||||
secret.Data = map[string][]byte{
|
||||
inviteSecretURLKey: []byte(inv.URL),
|
||||
inviteSecretInviteIDKey: []byte(strconv.FormatInt(inv.ID, 10)),
|
||||
inviteSecretExpiresAtKey: []byte(inv.ExpiresAt.Format(time.RFC3339)),
|
||||
}
|
||||
return controllerutil.SetControllerReference(team, secret, r.Scheme)
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
team.Status.InviteSecretRef = &terdutv1alpha1.LocalSecretRef{Name: secretName}
|
||||
if r.Recorder != nil {
|
||||
r.Recorder.Eventf(team, nil, corev1.EventTypeNormal, terdutv1alpha1.ReasonInviteMinted, terdutv1alpha1.ReasonInviteMinted,
|
||||
"invite link minted into Secret %q", secretName)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// revokeInvite tears down spec.invite's Secret and server-side row when
|
||||
// spec.invite.enabled is false (or was never set). Same-namespace and
|
||||
// OwnerReference'd, so deleting the TerdutTeam itself already garbage-
|
||||
// collects this Secret -- this path exists for the narrower case of
|
||||
// flipping enabled back to false on an otherwise-live TerdutTeam.
|
||||
func (r *TerdutTeamReconciler) revokeInvite(ctx context.Context, team *terdutv1alpha1.TerdutTeam, teamClient *tdclient.Client) error {
|
||||
if team.Status.InviteSecretRef == nil {
|
||||
return nil
|
||||
}
|
||||
name := team.Status.InviteSecretRef.Name
|
||||
var secret corev1.Secret
|
||||
err := r.Get(ctx, client.ObjectKey{Namespace: team.Namespace, Name: name}, &secret)
|
||||
switch {
|
||||
case err == nil:
|
||||
if id, idErr := strconv.ParseInt(string(secret.Data[inviteSecretInviteIDKey]), 10, 64); idErr == nil {
|
||||
if err := teamClient.RevokeInvite(ctx, team.Status.TeamID, id); err != nil {
|
||||
return fmt.Errorf("DELETE /api/teams/%d/invites/%d: %w", team.Status.TeamID, id, err)
|
||||
}
|
||||
}
|
||||
if err := r.Delete(ctx, &secret); err != nil && !apierrors.IsNotFound(err) {
|
||||
return err
|
||||
}
|
||||
case !apierrors.IsNotFound(err):
|
||||
return err
|
||||
}
|
||||
|
||||
team.Status.InviteSecretRef = nil
|
||||
if r.Recorder != nil {
|
||||
r.Recorder.Eventf(team, nil, corev1.EventTypeNormal, terdutv1alpha1.ReasonInviteRevoked, terdutv1alpha1.ReasonInviteRevoked,
|
||||
"invite link revoked")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
"k8s.io/apimachinery/pkg/api/meta"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
||||
@@ -47,21 +48,15 @@ const (
|
||||
testOperatorNamespace = "default"
|
||||
)
|
||||
|
||||
// bootstrapReadyTerdutServer creates a TerdutServer with a bring-your-own
|
||||
// DSN and drives it to Ready against fakeURL, the same three-pass sequence
|
||||
// terdutserver_controller_test.go's own happy-path test exercises directly
|
||||
// -- shared here so TerdutTeam's tests (which need a real, Ready
|
||||
// TerdutServer to resolve against) don't duplicate it.
|
||||
func bootstrapReadyTerdutServer(ctx context.Context, name, fakeURL string) *terdutv1alpha1.TerdutServer {
|
||||
// readyTerdutServer creates a TerdutServer with a bring-your-own DSN and drives
|
||||
// it to Ready the way the happy-path test does -- shared so the tests of
|
||||
// everything that references a server (TerdutTeam, TerdutAlertSource) start
|
||||
// from a real, Ready one.
|
||||
func readyTerdutServer(ctx context.Context, name string) *terdutv1alpha1.TerdutServer {
|
||||
GinkgoHelper()
|
||||
namespace := testOperatorNamespace
|
||||
|
||||
reconciler := &TerdutServerReconciler{
|
||||
Client: k8sClient,
|
||||
Scheme: k8sClient.Scheme(),
|
||||
OperatorNamespace: namespace,
|
||||
NewClient: func(string) *tdclient.Client { return tdclient.New(fakeURL) },
|
||||
}
|
||||
reconciler := &TerdutServerReconciler{Client: k8sClient, Scheme: k8sClient.Scheme()}
|
||||
objKey := types.NamespacedName{Name: name, Namespace: namespace}
|
||||
|
||||
srv := &terdutv1alpha1.TerdutServer{
|
||||
@@ -74,9 +69,7 @@ func bootstrapReadyTerdutServer(ctx context.Context, name, fakeURL string) *terd
|
||||
}
|
||||
Expect(k8sClient.Create(ctx, srv)).To(Succeed())
|
||||
|
||||
_, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: objKey}) // finalizer
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
_, err = reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: objKey}) // Deployment/Service
|
||||
_, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: objKey}) // Deployment, Service, key
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
var deploy appsv1.Deployment
|
||||
@@ -85,27 +78,25 @@ func bootstrapReadyTerdutServer(ctx context.Context, name, fakeURL string) *terd
|
||||
deploy.Status.Replicas = 1
|
||||
Expect(k8sClient.Status().Update(ctx, &deploy)).To(Succeed())
|
||||
|
||||
_, err = reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: objKey}) // bootstrap
|
||||
_, err = reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: objKey}) // Ready
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
Expect(k8sClient.Get(ctx, objKey, srv)).To(Succeed())
|
||||
Expect(srv.Status.CredentialsSecretRef).NotTo(BeNil(), "test setup: TerdutServer %s/%s did not reach Bootstrapped", namespace, name)
|
||||
Expect(meta.IsStatusConditionTrue(srv.Status.Conditions, terdutv1alpha1.ConditionReady)).To(BeTrue(),
|
||||
"test setup: TerdutServer %s/%s did not reach Ready", namespace, name)
|
||||
return srv
|
||||
}
|
||||
|
||||
// readyTerdutTeam creates a TerdutTeam under srv (an already-Ready
|
||||
// TerdutServer, e.g. from bootstrapReadyTerdutServer) and drives it to
|
||||
// Ready against fakeURL -- shared by TerdutEscalationRule's and
|
||||
// TerdutDeadmanSwitch's own tests, which both just need a resolvable
|
||||
// teamRef (DESIGN.md §5), not TerdutTeam's own behavior.
|
||||
// TerdutServer, from readyTerdutServer) and drives it to Ready against the fake
|
||||
// at fakeURL.
|
||||
func readyTerdutTeam(ctx context.Context, namespace, name string, srv *terdutv1alpha1.TerdutServer, fakeURL string) *terdutv1alpha1.TerdutTeam {
|
||||
GinkgoHelper()
|
||||
|
||||
reconciler := &TerdutTeamReconciler{
|
||||
Client: k8sClient,
|
||||
Scheme: k8sClient.Scheme(),
|
||||
OperatorNamespace: namespace,
|
||||
NewClient: func(string) *tdclient.Client { return tdclient.New(fakeURL) },
|
||||
Client: k8sClient,
|
||||
Scheme: k8sClient.Scheme(),
|
||||
NewClient: func(string) *tdclient.Client { return tdclient.New(fakeURL) },
|
||||
}
|
||||
objKey := types.NamespacedName{Name: name, Namespace: namespace}
|
||||
|
||||
@@ -120,11 +111,11 @@ func readyTerdutTeam(ctx context.Context, namespace, name string, srv *terdutv1a
|
||||
|
||||
_, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: objKey}) // finalizer
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
_, err = reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: objKey}) // create+mint+apply
|
||||
_, err = reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: objKey}) // apply
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
Expect(k8sClient.Get(ctx, objKey, team)).To(Succeed())
|
||||
Expect(team.Status.CredentialsSecretRef).NotTo(BeNil(), "test setup: TerdutTeam %s/%s did not reach Ready", namespace, name)
|
||||
Expect(team.Status.TeamID).NotTo(BeZero(), "test setup: TerdutTeam %s/%s did not reach Ready", namespace, name)
|
||||
return team
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user