package api import ( "database/sql" "encoding/json" "log" "net/http" "time" ) // Values for alerts.resolution_source, recording why an alert left the firing // state: a real Alertmanager notification, or inference by the sweeper. const ( resolutionAlertmanager = "alertmanager" resolutionExpiry = "expiry" ) // amPayload mirrors the Alertmanager webhook v4 payload. type amPayload struct { Version string `json:"version"` Status string `json:"status"` Alerts []amAlert `json:"alerts"` } type amAlert struct { Status string `json:"status"` Labels map[string]string `json:"labels"` Annotations map[string]string `json:"annotations"` StartsAt time.Time `json:"startsAt"` EndsAt time.Time `json:"endsAt"` GeneratorURL string `json:"generatorURL"` Fingerprint string `json:"fingerprint"` } func handleAlertmanagerWebhook(db *sql.DB) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { var payload amPayload if err := decodeJSON(r, &payload); err != nil { respond(w, http.StatusBadRequest, errResp("invalid payload")) return } now := time.Now().Unix() for _, a := range payload.Alerts { name := a.Labels["alertname"] labelsJSON, _ := json.Marshal(a.Labels) annotationsJSON, _ := json.Marshal(a.Annotations) // Zero time ("0001-01-01T00:00:00Z") means "no end known" — that is the // convention of Alertmanager's ingest API. Outgoing notifications // normally carry a real future endsAt instead, which is the watermark // the sweeper uses to expire alerts that stop being refreshed. var endsAtUnix *int64 if a.EndsAt.Year() > 1 { t := a.EndsAt.Unix() endsAtUnix = &t } var resolutionSource *string if a.Status == "resolved" { s := resolutionAlertmanager resolutionSource = &s } // The WHERE clause discards payloads that describe an alert instance // older than the stored one. Alertmanager retries failed notifications, // so a stale firing retry can arrive after the resolved one; it carries // the same startsAt, whereas a genuine re-fire carries a newer one. // Within a single instance, resolution is terminal. _, err := db.ExecContext(r.Context(), ` INSERT INTO alerts (fingerprint, name, status, labels, annotations, starts_at, ends_at, generator_url, received_at, resolution_source) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(fingerprint) DO UPDATE SET status = excluded.status, labels = excluded.labels, annotations = excluded.annotations, starts_at = excluded.starts_at, ends_at = excluded.ends_at, generator_url = excluded.generator_url, -- Load-bearing: advancing received_at on every accepted -- payload, re-sends included, is the documented liveness -- heartbeat clients and the sweeper both read. Removing it -- is a breaking API change — see models.Alert.ReceivedAt. received_at = excluded.received_at, resolution_source = excluded.resolution_source, -- A re-fire makes the alert current again, so it leaves the archive. archived_at = CASE WHEN excluded.status = 'firing' THEN NULL ELSE alerts.archived_at END WHERE excluded.starts_at > alerts.starts_at OR (excluded.starts_at = alerts.starts_at AND NOT (alerts.status = 'resolved' AND excluded.status = 'firing'))`, a.Fingerprint, name, a.Status, string(labelsJSON), string(annotationsJSON), a.StartsAt.Unix(), endsAtUnix, a.GeneratorURL, now, resolutionSource, ) if err != nil { log.Printf("upsert alert %s: %v", a.Fingerprint, err) } } w.WriteHeader(http.StatusOK) } }