feat!: incidents as the primary object
Release / build (amd64, darwin) (push) Failing after 9s
Release / build (arm64, darwin) (push) Failing after 10s
Release / build (amd64, linux) (push) Failing after 10s
Release / release (push) Has been skipped
Release / build (arm64, linux) (push) Failing after 11s

terdut-server v0.4.0 splits the alerts row into two objects, and the
endpoints this client drove for acknowledgement, comments and archiving
are gone. Pointing the same screens at the new paths would have missed
the point of the split: alerts are now Alertmanager's record, read-only
and carrying no human state, while the incident is the thing anyone
actually works on.

Incidents lead the section list and are what the client opens on. The
queue shows severity, status, assignee and age, and the detail view adds
what only exists server-side now: the group labels Alertmanager
correlated on, the member alerts, and an append-only timeline where
system events and notes are interleaved. That timeline is the whole
history the server keeps — alert rows are still mutated in place — so
rendering it in order matters more than styling it.

Actions all move onto the incident: a/A acknowledge, s assign, z/Z
snooze, c note, d delete note, x archive, R resolve.

Two of those need care rather than a keybinding:

  - R, not r, resolves, and it asks first. The server treats a manual
    resolve as terminal: a later occurrence opens a new incident instead
    of reopening this one, and an alert that never stops firing leaves
    the incident closed for good. A stray keypress is not recoverable,
    so the prompt says what it means.
  - x refuses on an open incident rather than archiving it, since
    archiving unresolved work only hides it. Snooze is offered as the
    "not now" answer, and the client treats a snoozed_until in the past
    as not snoozed, matching the server, which sweeps nothing.

Statistics lead with MTTA and MTTR, neither of which was computable
before. The server sends null until something has actually been
acknowledged or resolved, and that renders as — rather than 0: no data
is a different claim from instant.

Alerts keep a tab of their own as the raw feed — useful for asking what
Alertmanager is really sending — with an Incident column replacing Ack
By, and i in the detail view jumping to the incident where something can
be done about it. Archived now holds archived incidents; archiving an
alert is server-side housekeeping and no longer a user action.

BREAKING CHANGE: requires terdut-server v0.4.0 or later. Against an
older server every incident request 404s. Use terdut-tui v0.3.x with
servers before v0.4.0.
This commit is contained in:
Niklas Ye
2026-07-30 21:54:54 +02:00
parent e04cfcf433
commit 1140d773f8
8 changed files with 1622 additions and 731 deletions
+146 -34
View File
@@ -63,6 +63,10 @@ func (c *Client) do(req *http.Request, out any) error {
// ListAlerts fetches alerts. status may be "firing", "resolved", or "" for all.
// Set archived=true to fetch only archived alerts; false returns only non-archived.
//
// Alerts are read-only on the server — there is nothing to acknowledge or
// archive here. This is the raw feed, useful for checking what Alertmanager is
// actually sending; the work queue is ListIncidents.
func (c *Client) ListAlerts(status string, archived bool, limit int) ([]Alert, error) {
q := url.Values{}
if status != "" {
@@ -87,23 +91,6 @@ func (c *Client) ListAlerts(status string, archived bool, limit int) ([]Alert, e
return alerts, c.do(req, &alerts)
}
func (c *Client) ArchiveAlert(id int64) (*Alert, error) {
req, err := c.newRequest(http.MethodPost, fmt.Sprintf("/api/alerts/%d/archive", id))
if err != nil {
return nil, err
}
var alert Alert
return &alert, c.do(req, &alert)
}
func (c *Client) UnarchiveAlert(id int64) error {
req, err := c.newRequest(http.MethodDelete, fmt.Sprintf("/api/alerts/%d/archive", id))
if err != nil {
return err
}
return c.do(req, nil)
}
// GetAlertStats fetches aggregate alert counts.
func (c *Client) GetAlertStats() (*AlertStats, error) {
req, err := c.newRequest(http.MethodGet, "/api/stats/alerts")
@@ -138,49 +125,172 @@ func (c *Client) GetAlert(id int64) (*Alert, error) {
return &alert, c.do(req, &alert)
}
func (c *Client) AcknowledgeAlert(id int64) (*Alert, error) {
req, err := c.newRequest(http.MethodPost, fmt.Sprintf("/api/alerts/%d/acknowledge", id))
// ── Incidents ──────────────────────────────────────────────────────────────
// ListIncidents fetches the work queue. status may be "triggered",
// "acknowledged", "resolved", or "" for the server default of open incidents
// only. archived and snoozed each switch the list to that set rather than
// adding to it, matching the server's filters.
func (c *Client) ListIncidents(status string, archived, snoozed bool, limit int) ([]Incident, error) {
q := url.Values{}
if status != "" {
q.Set("status", status)
}
if archived {
q.Set("archived", "true")
}
if snoozed {
q.Set("snoozed", "true")
}
if limit > 0 {
q.Set("limit", strconv.Itoa(limit))
}
path := "/api/incidents"
if len(q) > 0 {
path += "?" + q.Encode()
}
req, err := c.newRequest(http.MethodGet, path)
if err != nil {
return nil, err
}
var alert Alert
return &alert, c.do(req, &alert)
var incidents []Incident
return incidents, c.do(req, &incidents)
}
func (c *Client) UnacknowledgeAlert(id int64) error {
req, err := c.newRequest(http.MethodDelete, fmt.Sprintf("/api/alerts/%d/acknowledge", id))
// GetIncident returns one incident with its member alerts inline.
func (c *Client) GetIncident(id int64) (*Incident, error) {
req, err := c.newRequest(http.MethodGet, fmt.Sprintf("/api/incidents/%d", id))
if err != nil {
return nil, err
}
var incident Incident
return &incident, c.do(req, &incident)
}
func (c *Client) GetIncidentTimeline(id int64) ([]IncidentEvent, error) {
req, err := c.newRequest(http.MethodGet, fmt.Sprintf("/api/incidents/%d/timeline", id))
if err != nil {
return nil, err
}
var events []IncidentEvent
return events, c.do(req, &events)
}
func (c *Client) AcknowledgeIncident(id int64) (*Incident, error) {
req, err := c.newRequest(http.MethodPost, fmt.Sprintf("/api/incidents/%d/acknowledge", id))
if err != nil {
return nil, err
}
var incident Incident
return &incident, c.do(req, &incident)
}
func (c *Client) UnacknowledgeIncident(id int64) error {
req, err := c.newRequest(http.MethodDelete, fmt.Sprintf("/api/incidents/%d/acknowledge", id))
if err != nil {
return err
}
return c.do(req, nil)
}
func (c *Client) GetComments(alertID int64) ([]Comment, error) {
req, err := c.newRequest(http.MethodGet, fmt.Sprintf("/api/alerts/%d/comments", alertID))
// ResolveIncident closes an incident by hand. This is terminal on the server: a
// later occurrence in the same group opens a new incident rather than reopening
// this one, and if the alert underneath never stops firing the incident stays
// closed. Use SnoozeIncident for "not now".
func (c *Client) ResolveIncident(id int64) (*Incident, error) {
req, err := c.newRequest(http.MethodPost, fmt.Sprintf("/api/incidents/%d/resolve", id))
if err != nil {
return nil, err
}
var comments []Comment
return comments, c.do(req, &comments)
var incident Incident
return &incident, c.do(req, &incident)
}
func (c *Client) AddComment(alertID int64, content string) (*Comment, error) {
req, err := c.newRequestWithBody(http.MethodPost, fmt.Sprintf("/api/alerts/%d/comments", alertID), map[string]string{"content": content})
func (c *Client) AssignIncident(id, userID int64) (*Incident, error) {
body := struct {
UserID int64 `json:"user_id"`
}{UserID: userID}
req, err := c.newRequestWithBody(http.MethodPost, fmt.Sprintf("/api/incidents/%d/assign", id), body)
if err != nil {
return nil, err
}
var comment Comment
return &comment, c.do(req, &comment)
var incident Incident
return &incident, c.do(req, &incident)
}
func (c *Client) DeleteComment(alertID, commentID int64) error {
req, err := c.newRequest(http.MethodDelete, fmt.Sprintf("/api/alerts/%d/comments/%d", alertID, commentID))
// SnoozeIncident hides an incident from the default queue for a duration,
// without closing it.
func (c *Client) SnoozeIncident(id int64, duration string) (*Incident, error) {
body := struct {
Duration string `json:"duration"`
}{Duration: duration}
req, err := c.newRequestWithBody(http.MethodPost, fmt.Sprintf("/api/incidents/%d/snooze", id), body)
if err != nil {
return nil, err
}
var incident Incident
return &incident, c.do(req, &incident)
}
func (c *Client) UnsnoozeIncident(id int64) error {
req, err := c.newRequest(http.MethodDelete, fmt.Sprintf("/api/incidents/%d/snooze", id))
if err != nil {
return err
}
return c.do(req, nil)
}
func (c *Client) ArchiveIncident(id int64) (*Incident, error) {
req, err := c.newRequest(http.MethodPost, fmt.Sprintf("/api/incidents/%d/archive", id))
if err != nil {
return nil, err
}
var incident Incident
return &incident, c.do(req, &incident)
}
func (c *Client) UnarchiveIncident(id int64) error {
req, err := c.newRequest(http.MethodDelete, fmt.Sprintf("/api/incidents/%d/archive", id))
if err != nil {
return err
}
return c.do(req, nil)
}
// AddNote appends a note to the incident's timeline.
func (c *Client) AddNote(incidentID int64, content string) (*IncidentEvent, error) {
req, err := c.newRequestWithBody(http.MethodPost,
fmt.Sprintf("/api/incidents/%d/notes", incidentID), map[string]string{"content": content})
if err != nil {
return nil, err
}
var event IncidentEvent
return &event, c.do(req, &event)
}
// DeleteNote removes one of your own notes. Only notes are deletable — the rest
// of the timeline is a record of what happened.
func (c *Client) DeleteNote(incidentID, eventID int64) error {
req, err := c.newRequest(http.MethodDelete,
fmt.Sprintf("/api/incidents/%d/notes/%d", incidentID, eventID))
if err != nil {
return err
}
return c.do(req, nil)
}
func (c *Client) GetIncidentStats() (*IncidentStats, error) {
req, err := c.newRequest(http.MethodGet, "/api/stats/incidents")
if err != nil {
return nil, err
}
var stats IncidentStats
return &stats, c.do(req, &stats)
}
// ── Statistics ─────────────────────────────────────────────────────────────
func (c *Client) GetTopAlerts(limit int) ([]TopAlert, error) {
req, err := c.newRequest(http.MethodGet, fmt.Sprintf("/api/stats/alerts/top?limit=%d", limit))
if err != nil {
@@ -232,7 +342,9 @@ func (c *Client) GetCurrentOnCall() (*ScheduleEntry, error) {
return nil, nil
}
if resp.StatusCode >= 400 {
var e struct{ Error string `json:"error"` }
var e struct {
Error string `json:"error"`
}
_ = json.NewDecoder(resp.Body).Decode(&e)
if e.Error != "" {
return nil, fmt.Errorf("server returned %d: %s", resp.StatusCode, e.Error)
+111 -21
View File
@@ -2,21 +2,26 @@ package api
import "time"
// Alert is the server's record of what Alertmanager said. It is read-only:
// acknowledging, assigning, noting and resolving all happen on the Incident an
// alert belongs to.
type Alert struct {
ID int64 `json:"id"`
Fingerprint string `json:"fingerprint"`
Name string `json:"name"`
Status string `json:"status"`
Labels map[string]string `json:"labels"`
Annotations map[string]string `json:"annotations"`
StartsAt time.Time `json:"starts_at"`
EndsAt *time.Time `json:"ends_at"`
GeneratorURL string `json:"generator_url"`
ReceivedAt time.Time `json:"received_at"`
AcknowledgedByID *int64 `json:"acknowledged_by_id"`
AcknowledgedBy string `json:"acknowledged_by"`
AcknowledgedAt *time.Time `json:"acknowledged_at"`
ArchivedAt *time.Time `json:"archived_at,omitempty"`
ID int64 `json:"id"`
Fingerprint string `json:"fingerprint"`
Name string `json:"name"`
Status string `json:"status"`
Labels map[string]string `json:"labels"`
Annotations map[string]string `json:"annotations"`
StartsAt time.Time `json:"starts_at"`
EndsAt *time.Time `json:"ends_at"`
GeneratorURL string `json:"generator_url"`
ReceivedAt time.Time `json:"received_at"`
ArchivedAt *time.Time `json:"archived_at,omitempty"`
// IncidentID is the most recent incident this alert belongs to. An alert row
// is reused across occurrences of the same fingerprint, so it belongs to a
// series of incidents over its life and this is only the newest.
IncidentID *int64 `json:"incident_id,omitempty"`
// ResolutionSource records why a resolved alert left the firing state:
// "alertmanager" for a real resolved webhook, "expiry" when the server
@@ -24,19 +29,104 @@ type Alert struct {
ResolutionSource *string `json:"resolution_source,omitempty"`
}
// Incident statuses.
const (
StatusTriggered = "triggered"
StatusAcknowledged = "acknowledged"
StatusResolved = "resolved"
)
// Incident is the work item: what a person acknowledges, assigns, snoozes,
// discusses and resolves. Many alerts map to one incident, correlated by the
// groupKey Alertmanager computed from the operator's group_by configuration.
type Incident struct {
ID int64 `json:"id"`
GroupKey string `json:"group_key"`
Title string `json:"title"`
GroupLabels map[string]string `json:"group_labels"`
Status string `json:"status"`
// Severity is a high-water mark across the incident's alerts, never lowered,
// so a resolved incident still says how bad it got.
Severity string `json:"severity,omitempty"`
TriggeredAt time.Time `json:"triggered_at"`
AcknowledgedByID *int64 `json:"acknowledged_by_id,omitempty"`
AcknowledgedBy string `json:"acknowledged_by,omitempty"`
AcknowledgedAt *time.Time `json:"acknowledged_at,omitempty"`
AssignedToID *int64 `json:"assigned_to_id,omitempty"`
AssignedTo string `json:"assigned_to,omitempty"`
SnoozedUntil *time.Time `json:"snoozed_until,omitempty"`
ResolvedAt *time.Time `json:"resolved_at,omitempty"`
// ResolutionSource is "alerts" when every alert stopped firing, or "manual"
// when a person closed it. Treat the value set as open.
ResolutionSource *string `json:"resolution_source,omitempty"`
ArchivedAt *time.Time `json:"archived_at,omitempty"`
// Alerts is populated by GET /api/incidents/{id} only.
Alerts []Alert `json:"alerts,omitempty"`
}
// IsSnoozed reports whether the incident is currently quietened. A snooze
// expires by falling into the past; nothing on the server sweeps it.
func (i Incident) IsSnoozed() bool {
return i.SnoozedUntil != nil && i.SnoozedUntil.After(time.Now())
}
// IsOpen reports whether the incident is still work in progress.
func (i Incident) IsOpen() bool { return i.ResolvedAt == nil }
// Incident timeline event types written by the server. New ones may be added,
// so render unrecognised types generically rather than dropping them.
const (
EventTriggered = "triggered"
EventAlertAdded = "alert_added"
EventAlertResolved = "alert_resolved"
EventAcknowledged = "acknowledged"
EventUnacknowledged = "unacknowledged"
EventAssigned = "assigned"
EventSnoozed = "snoozed"
EventUnsnoozed = "unsnoozed"
EventResolved = "resolved"
EventNote = "note"
)
// IncidentEvent is one entry in an incident's timeline. An empty Username means
// the server acted rather than a person. On an "assigned" event the user is the
// assignee, not the actor.
type IncidentEvent struct {
ID int64 `json:"id"`
IncidentID int64 `json:"incident_id"`
Type string `json:"type"`
UserID *int64 `json:"user_id,omitempty"`
Username string `json:"username,omitempty"`
AlertID *int64 `json:"alert_id,omitempty"`
Detail string `json:"detail,omitempty"`
CreatedAt time.Time `json:"created_at"`
}
type AlertStats struct {
Total int `json:"total"`
Firing int `json:"firing"`
Resolved int `json:"resolved"`
}
type Comment struct {
ID int64 `json:"id"`
AlertID int64 `json:"alert_id"`
UserID int64 `json:"user_id"`
Username string `json:"username"`
Content string `json:"content"`
CreatedAt time.Time `json:"created_at"`
// IncidentStats carries the queue counts plus mean time to acknowledge and to
// resolve. Both averages are nil until something has actually been acknowledged
// or resolved — that is "no data", not zero.
type IncidentStats struct {
Total int `json:"total"`
Triggered int `json:"triggered"`
Acknowledged int `json:"acknowledged"`
Resolved int `json:"resolved"`
MTTASeconds *float64 `json:"mtta_seconds"`
MTTRSeconds *float64 `json:"mttr_seconds"`
}
type TopAlert struct {