List alert sources with their status and last arrival on Team -> Sources
Like Team -> Switches, the page is now a table: a status badge (Active
if the key posted within a day, Quiet if it has but not lately, Never
used), when it last posted a webhook, when an alert last arrived on it,
how many distinct alerts it refreshed in the last 24 hours, and when it
was created. Adding a source moved into a "New source" sheet, and owners
can rename one from its row.
"Last alert" and the count needed alerts to remember which source they
came in on, which they never did, so migration 010 adds
alerts.integration_id and every accepted payload stamps it. Last sender
wins when two sources post the same fingerprint. It is not backfilled: a
NULL says "before this was recorded" rather than guessing, and it heals
by itself as Alertmanager re-sends each alert every repeat_interval.
Revoking a source keeps its alerts, unattributed.
Last webhook and last alert are separate on purpose: a payload with
nothing usable in it stamps the first and not the second. The Quiet
threshold is a fixed day, a colour and not an alarm, since silence that
should page is what dead man's switches are for.
The counts are indexed subqueries (alerts_integration_idx) rather than a
join, which would read every alert a source ever delivered.
API: the integrations list gains status, last_alert_at and alerts_24h,
and PATCH /api/teams/{id}/integrations/{id} renames. Both are additive;
terdut-tui needs nothing.
This commit is contained in:
@@ -78,7 +78,7 @@ type ingested struct {
|
||||
// post, and which team the alerts belong to.
|
||||
func handleIntegrationWebhook(db *sql.DB, notify NotifyConfig) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
teamID, err := teamIDForKey(r.Context(), db, chi.URLParam(r, "key"))
|
||||
src, err := sourceForKey(r.Context(), db, chi.URLParam(r, "key"))
|
||||
if err != nil {
|
||||
if errors.Is(err, errUnknownIntegration) {
|
||||
// 401 and not 404: the path is real, the key is not, and a
|
||||
@@ -90,11 +90,12 @@ func handleIntegrationWebhook(db *sql.DB, notify NotifyConfig) http.HandlerFunc
|
||||
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||
return
|
||||
}
|
||||
receiveWebhook(w, r, db, notify, teamID)
|
||||
receiveWebhook(w, r, db, notify, src)
|
||||
}
|
||||
}
|
||||
|
||||
func receiveWebhook(w http.ResponseWriter, r *http.Request, db *sql.DB, notify NotifyConfig, teamID int64) {
|
||||
func receiveWebhook(w http.ResponseWriter, r *http.Request, db *sql.DB, notify NotifyConfig, src alertSource) {
|
||||
teamID := src.teamID
|
||||
var payload amPayload
|
||||
if err := decodeJSON(r, &payload); err != nil {
|
||||
respond(w, http.StatusBadRequest, errResp("invalid payload"))
|
||||
@@ -104,7 +105,7 @@ func receiveWebhook(w http.ResponseWriter, r *http.Request, db *sql.DB, notify N
|
||||
// Alertmanager retries anything that is not 2xx, and a retry of a payload
|
||||
// we failed to store is more useful than an error it cannot act on — so
|
||||
// failures are logged, not surfaced.
|
||||
if err := ingest(r.Context(), db, notify, teamID, payload); err != nil {
|
||||
if err := ingest(r.Context(), db, notify, src, payload); err != nil {
|
||||
log.Printf("webhook ingest (team %d, group %q): %v", teamID, payload.GroupKey, err)
|
||||
}
|
||||
|
||||
@@ -114,7 +115,8 @@ func receiveWebhook(w http.ResponseWriter, r *http.Request, db *sql.DB, notify N
|
||||
// ingest stores a payload's alerts and reconciles the incident for its group.
|
||||
// The whole payload is one transaction: an incident that opened but whose alerts
|
||||
// failed to link would be a work item nobody could act on.
|
||||
func ingest(ctx context.Context, db *sql.DB, notify NotifyConfig, teamID int64, payload amPayload) error {
|
||||
func ingest(ctx context.Context, db *sql.DB, notify NotifyConfig, src alertSource, payload amPayload) error {
|
||||
teamID := src.teamID
|
||||
tx, err := db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -129,7 +131,7 @@ func ingest(ctx context.Context, db *sql.DB, notify NotifyConfig, teamID int64,
|
||||
return err
|
||||
}
|
||||
|
||||
accepted, err := upsertAlerts(ctx, tx, deadman, teamID, payload.Alerts)
|
||||
accepted, err := upsertAlerts(ctx, tx, deadman, src, payload.Alerts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -186,7 +188,8 @@ func ingest(ctx context.Context, db *sql.DB, notify NotifyConfig, teamID int64,
|
||||
|
||||
// upsertAlerts stores each alert of a payload and reports what changed. Payloads
|
||||
// the ordering guard rejected are left out entirely.
|
||||
func upsertAlerts(ctx context.Context, tx *sql.Tx, deadman deadmanSet, teamID int64, alerts []amAlert) ([]ingested, error) {
|
||||
func upsertAlerts(ctx context.Context, tx *sql.Tx, deadman deadmanSet, src alertSource, alerts []amAlert) ([]ingested, error) {
|
||||
teamID := src.teamID
|
||||
now := time.Now().Unix()
|
||||
accepted := make([]ingested, 0, len(alerts))
|
||||
|
||||
@@ -242,8 +245,8 @@ func upsertAlerts(ctx context.Context, tx *sql.Tx, deadman deadmanSet, teamID in
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
INSERT INTO alerts
|
||||
(team_id, fingerprint, name, status, labels, annotations, starts_at, ends_at,
|
||||
generator_url, received_at, resolution_source)
|
||||
VALUES ($1, $2, $3, $4, $5::jsonb, $6::jsonb, $7, $8, $9, $10, $11)
|
||||
generator_url, received_at, resolution_source, integration_id)
|
||||
VALUES ($1, $2, $3, $4, $5::jsonb, $6::jsonb, $7, $8, $9, $10, $11, $12)
|
||||
ON CONFLICT (team_id, fingerprint) DO UPDATE SET
|
||||
status = excluded.status,
|
||||
labels = excluded.labels,
|
||||
@@ -257,6 +260,8 @@ func upsertAlerts(ctx context.Context, tx *sql.Tx, deadman deadmanSet, teamID in
|
||||
-- is a breaking API change — see models.Alert.ReceivedAt.
|
||||
received_at = excluded.received_at,
|
||||
resolution_source = excluded.resolution_source,
|
||||
-- Last sender wins; see migration 010.
|
||||
integration_id = excluded.integration_id,
|
||||
-- A re-fire makes the alert current again, so it leaves the archive.
|
||||
archived_at = CASE WHEN excluded.status = 'firing'
|
||||
THEN NULL ELSE alerts.archived_at END
|
||||
@@ -267,7 +272,7 @@ func upsertAlerts(ctx context.Context, tx *sql.Tx, deadman deadmanSet, teamID in
|
||||
teamID, a.Fingerprint, name, a.Status,
|
||||
string(labelsJSON), string(annotationsJSON),
|
||||
a.StartsAt.Unix(), endsAtUnix,
|
||||
a.GeneratorURL, now, resolutionSource,
|
||||
a.GeneratorURL, now, resolutionSource, src.integrationID,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user