Niklas Ye
a602ff3efc
The API reference listed endpoints but never the alert object's fields, so
two of them were load-bearing for clients while being described nowhere.
received_at appeared only in passing, as a stats filter; resolution_source
only inside the stale-expiry prose.
Both carry meaning a client cannot derive on its own. starts_at comes from
Prometheus and never changes for an alert instance, so received_at is the
only signal that a firing alert is still being refreshed — it advances on
every accepted webhook, including the unchanged notifications Alertmanager
re-sends every repeat_interval. resolution_source then says how much to
trust ends_at: under 'alertmanager' it is an end time somebody reported,
but under 'expiry' nothing ever reported one, so it is either a stale
watermark or the sweep timestamp, and only an upper bound.
README gains an alert object field table plus a contract section for each,
including the nullability rules and the advice to tolerate unrecognised
resolution_source values. The field comments in models.Alert now say these
are public API rather than ingest details, and the upsert carries a note at
the received_at line, which is where a regression would be introduced.
Three tests lock the newly documented behaviour, none of which was covered
before — the whole suite passed with the received_at bump deleted from the
upsert, because the expiry tests only ever set that column via SQL:
- a re-send advances received_at and leaves starts_at alone
- a discarded out-of-order retry does not count as a heartbeat
- an expiry resolve preserves a reported ends_at watermark and stamps
sweep time only when none was known
107 lines
3.7 KiB
Go
107 lines
3.7 KiB
Go
package api
|
|
|
|
import (
|
|
"database/sql"
|
|
"encoding/json"
|
|
"log"
|
|
"net/http"
|
|
"time"
|
|
)
|
|
|
|
// Values for alerts.resolution_source, recording why an alert left the firing
|
|
// state: a real Alertmanager notification, or inference by the sweeper.
|
|
const (
|
|
resolutionAlertmanager = "alertmanager"
|
|
resolutionExpiry = "expiry"
|
|
)
|
|
|
|
// amPayload mirrors the Alertmanager webhook v4 payload.
|
|
type amPayload struct {
|
|
Version string `json:"version"`
|
|
Status string `json:"status"`
|
|
Alerts []amAlert `json:"alerts"`
|
|
}
|
|
|
|
type amAlert struct {
|
|
Status string `json:"status"`
|
|
Labels map[string]string `json:"labels"`
|
|
Annotations map[string]string `json:"annotations"`
|
|
StartsAt time.Time `json:"startsAt"`
|
|
EndsAt time.Time `json:"endsAt"`
|
|
GeneratorURL string `json:"generatorURL"`
|
|
Fingerprint string `json:"fingerprint"`
|
|
}
|
|
|
|
func handleAlertmanagerWebhook(db *sql.DB) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
var payload amPayload
|
|
if err := decodeJSON(r, &payload); err != nil {
|
|
respond(w, http.StatusBadRequest, errResp("invalid payload"))
|
|
return
|
|
}
|
|
|
|
now := time.Now().Unix()
|
|
for _, a := range payload.Alerts {
|
|
name := a.Labels["alertname"]
|
|
labelsJSON, _ := json.Marshal(a.Labels)
|
|
annotationsJSON, _ := json.Marshal(a.Annotations)
|
|
|
|
// Zero time ("0001-01-01T00:00:00Z") means "no end known" — that is the
|
|
// convention of Alertmanager's ingest API. Outgoing notifications
|
|
// normally carry a real future endsAt instead, which is the watermark
|
|
// the sweeper uses to expire alerts that stop being refreshed.
|
|
var endsAtUnix *int64
|
|
if a.EndsAt.Year() > 1 {
|
|
t := a.EndsAt.Unix()
|
|
endsAtUnix = &t
|
|
}
|
|
|
|
var resolutionSource *string
|
|
if a.Status == "resolved" {
|
|
s := resolutionAlertmanager
|
|
resolutionSource = &s
|
|
}
|
|
|
|
// The WHERE clause discards payloads that describe an alert instance
|
|
// older than the stored one. Alertmanager retries failed notifications,
|
|
// so a stale firing retry can arrive after the resolved one; it carries
|
|
// the same startsAt, whereas a genuine re-fire carries a newer one.
|
|
// Within a single instance, resolution is terminal.
|
|
_, err := db.ExecContext(r.Context(), `
|
|
INSERT INTO alerts
|
|
(fingerprint, name, status, labels, annotations, starts_at, ends_at,
|
|
generator_url, received_at, resolution_source)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
ON CONFLICT(fingerprint) DO UPDATE SET
|
|
status = excluded.status,
|
|
labels = excluded.labels,
|
|
annotations = excluded.annotations,
|
|
starts_at = excluded.starts_at,
|
|
ends_at = excluded.ends_at,
|
|
generator_url = excluded.generator_url,
|
|
-- Load-bearing: advancing received_at on every accepted
|
|
-- payload, re-sends included, is the documented liveness
|
|
-- heartbeat clients and the sweeper both read. Removing it
|
|
-- is a breaking API change — see models.Alert.ReceivedAt.
|
|
received_at = excluded.received_at,
|
|
resolution_source = excluded.resolution_source,
|
|
-- A re-fire makes the alert current again, so it leaves the archive.
|
|
archived_at = CASE WHEN excluded.status = 'firing'
|
|
THEN NULL ELSE alerts.archived_at END
|
|
WHERE excluded.starts_at > alerts.starts_at
|
|
OR (excluded.starts_at = alerts.starts_at
|
|
AND NOT (alerts.status = 'resolved' AND excluded.status = 'firing'))`,
|
|
a.Fingerprint, name, a.Status,
|
|
string(labelsJSON), string(annotationsJSON),
|
|
a.StartsAt.Unix(), endsAtUnix,
|
|
a.GeneratorURL, now, resolutionSource,
|
|
)
|
|
if err != nil {
|
|
log.Printf("upsert alert %s: %v", a.Fingerprint, err)
|
|
}
|
|
}
|
|
|
|
w.WriteHeader(http.StatusOK)
|
|
}
|
|
}
|