Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| c4833067f0 | |||
| 2bc8f336a0 | |||
| a955356821 |
@@ -463,6 +463,10 @@ queue, the incident and the alert list, instead of leaving it in the title.
|
||||
An alert that is not grouped by `cluster` still shows the chip on the alert
|
||||
list, which reads the label from the alert itself.
|
||||
|
||||
The queue has a cluster dropdown once there are two or more values to choose
|
||||
between. It filters on the incident's `cluster` group label
|
||||
(`GET /api/incidents?cluster=...`), so it only sees incidents grouped by it.
|
||||
|
||||
### An incident opens only on a new occurrence
|
||||
|
||||
An incident opens when an alert **transitions into firing**: a fingerprint that
|
||||
@@ -945,7 +949,8 @@ the team gets the same `404` as anybody else.
|
||||
|
||||
| Method | Path | Description |
|
||||
|---|---|---|
|
||||
| `GET` | `/api/incidents` | List incidents. Filters: `?status=triggered\|acknowledged\|resolved`, `?severity=`, `?assigned_to=<user id>`, `?archived=true`, `?snoozed=true`, `?from=YYYY-MM-DD`, `?to=YYYY-MM-DD`, `?sort=severity`, `?limit=` (default 50, max 500) |
|
||||
| `GET` | `/api/incidents` | List incidents. Filters: `?status=triggered\|acknowledged\|resolved`, `?severity=`, `?assigned_to=<user id>`, `?archived=true`, `?snoozed=true`, `?from=YYYY-MM-DD`, `?to=YYYY-MM-DD`, `?sort=severity`, `?cluster=<value of the cluster group label>`, `?limit=` (default 50, max 500) |
|
||||
| `GET` | `/api/incidents/clusters` | The distinct `cluster` values on the caller's incidents from the last 90 days, sorted (`?team_id=` narrows it). An empty array when nothing carries the label |
|
||||
| `GET` | `/api/incidents/{id}` | Get single incident, with its alerts inline |
|
||||
| `GET` | `/api/incidents/{id}/alerts` | Alerts under this incident |
|
||||
| `GET` | `/api/incidents/{id}/timeline` | Full event history, chronological |
|
||||
|
||||
@@ -15,5 +15,5 @@ type: application
|
||||
# appVersion and image.tag in values.yaml no longer agree, and that is not an oversight:
|
||||
# image.tag stays "latest", which is what a local install actually pulls. appVersion is
|
||||
# metadata and drives nothing.
|
||||
version: 0.42.1
|
||||
appVersion: "v0.42.1"
|
||||
version: 0.43.0
|
||||
appVersion: "v0.43.0"
|
||||
|
||||
@@ -13,6 +13,52 @@ import (
|
||||
"github.com/go-chi/chi/v5"
|
||||
)
|
||||
|
||||
// handleListClusters answers GET /api/incidents/clusters: the distinct values of
|
||||
// the origin label across the caller's incidents from the last 90 days, sorted,
|
||||
// so the queue can offer them as a filter. Optional team_id narrows it to one
|
||||
// team. Empty when nothing carries the label, which is how the UI knows to show
|
||||
// no filter at all.
|
||||
func handleListClusters(db *sql.DB) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
args := &sqlArgs{}
|
||||
where := []string{
|
||||
"team_id = ANY(" + args.add(callerTeamIDs(r.Context())) + ")",
|
||||
"triggered_at >= " + args.add(time.Now().AddDate(0, 0, -90).Unix()),
|
||||
}
|
||||
if team := r.URL.Query().Get("team_id"); team != "" {
|
||||
if n, err := strconv.ParseInt(team, 10, 64); err == nil {
|
||||
where = append(where, "team_id = "+args.add(n))
|
||||
}
|
||||
}
|
||||
label := args.add(originLabel)
|
||||
|
||||
rows, err := db.QueryContext(r.Context(),
|
||||
fmt.Sprintf("SELECT DISTINCT group_labels->>%[1]s AS v FROM incidents WHERE %[2]s AND group_labels->>%[1]s <> '' ORDER BY v LIMIT 200",
|
||||
label, strings.Join(where, " AND ")),
|
||||
args.all()...)
|
||||
if err != nil {
|
||||
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
clusters := []string{}
|
||||
for rows.Next() {
|
||||
var v string
|
||||
if err := rows.Scan(&v); err != nil {
|
||||
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||
return
|
||||
}
|
||||
clusters = append(clusters, v)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||
return
|
||||
}
|
||||
respond(w, http.StatusOK, clusters)
|
||||
}
|
||||
}
|
||||
|
||||
func handleListIncidents(db *sql.DB) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
q := r.URL.Query()
|
||||
@@ -53,6 +99,12 @@ func handleListIncidents(db *sql.DB) http.HandlerFunc {
|
||||
if severity := q.Get("severity"); severity != "" {
|
||||
where = append(where, "i.severity = "+args.add(severity))
|
||||
}
|
||||
// Where it came from: the value of the origin label (originLabel, by
|
||||
// convention "cluster") among the incident's group labels. An incident has
|
||||
// it only when the label is in Alertmanager's group_by.
|
||||
if cluster := q.Get("cluster"); cluster != "" {
|
||||
where = append(where, "i.group_labels->>"+args.add(originLabel)+" = "+args.add(cluster))
|
||||
}
|
||||
if assignee := q.Get("assigned_to"); assignee != "" {
|
||||
if n, err := strconv.ParseInt(assignee, 10, 64); err == nil {
|
||||
where = append(where, "i.assigned_to = "+args.add(n))
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -972,3 +973,37 @@ func TestServiceAccount_AssignArchiveUnarchiveRecordActor(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The queue can be narrowed to one cluster, and the distinct clusters are
|
||||
// offered so the filter has something to list.
|
||||
func TestIncidents_FilterByCluster(t *testing.T) {
|
||||
s := newTS(t)
|
||||
|
||||
for _, c := range []string{"prod-eu", "prod-us"} {
|
||||
fireGroupedAs(t, s, "fp-"+c,
|
||||
map[string]string{"alertname": "PodRestarting", "cluster": c, "namespace": "n"},
|
||||
`{}:{alertname="PodRestarting",cluster="`+c+`",namespace="n"}`)
|
||||
}
|
||||
// A cluster-less incident exists too, and must never match a cluster filter.
|
||||
fireGroupedAs(t, s, "fp-none", map[string]string{"alertname": "DiskFull"}, `{}:{alertname="DiskFull"}`)
|
||||
|
||||
if got := len(listIncidents(t, s, "")); got != 3 {
|
||||
t.Fatalf("expected 3 open incidents, got %d", got)
|
||||
}
|
||||
eu := listIncidents(t, s, "?cluster=prod-eu")
|
||||
if len(eu) != 1 {
|
||||
t.Fatalf("expected 1 incident for prod-eu, got %d", len(eu))
|
||||
}
|
||||
if labels, _ := eu[0]["group_labels"].(map[string]any); labels["cluster"] != "prod-eu" {
|
||||
t.Errorf("filtered to the wrong cluster: %v", eu[0]["group_labels"])
|
||||
}
|
||||
if got := len(listIncidents(t, s, "?cluster=nowhere")); got != 0 {
|
||||
t.Errorf("an unknown cluster should match nothing, got %d", got)
|
||||
}
|
||||
|
||||
var clusters []string
|
||||
decode(t, s.req(t, http.MethodGet, "/api/incidents/clusters", nil), &clusters)
|
||||
if want := "prod-eu,prod-us"; strings.Join(clusters, ",") != want {
|
||||
t.Errorf("clusters = %v, want %s", clusters, want)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -168,10 +168,17 @@ func fireCritical(t *testing.T, s *ts) {
|
||||
// fireGrouped posts one critical alert whose Alertmanager group carries the
|
||||
// given labels, the way group_by puts them on the webhook.
|
||||
func fireGrouped(t *testing.T, s *ts, groupLabels map[string]string, groupKey string) {
|
||||
t.Helper()
|
||||
fireGroupedAs(t, s, "fp-grouped", groupLabels, groupKey)
|
||||
}
|
||||
|
||||
// fireGroupedAs is fireGrouped with its own alert fingerprint, for a test that
|
||||
// needs several alerts open at once.
|
||||
func fireGroupedAs(t *testing.T, s *ts, fingerprint string, groupLabels map[string]string, groupKey string) {
|
||||
t.Helper()
|
||||
payload := map[string]any{
|
||||
"version": "4", "status": "firing", "groupKey": groupKey, "groupLabels": groupLabels,
|
||||
"alerts": []map[string]any{amAlert("fp-grouped", "PodRestarting", "firing",
|
||||
"alerts": []map[string]any{amAlert(fingerprint, "PodRestarting", "firing",
|
||||
"2026-05-20T10:00:00Z", zeroTime, map[string]string{"severity": "critical"})},
|
||||
}
|
||||
data, _ := json.Marshal(payload)
|
||||
|
||||
@@ -145,6 +145,7 @@ func NewRouter(db *sql.DB, notify NotifyConfig, cfg config.Config, version strin
|
||||
r.Get("/api/alerts/{id}", handleGetAlert(db))
|
||||
|
||||
r.Get("/api/incidents", handleListIncidents(db))
|
||||
r.Get("/api/incidents/clusters", handleListClusters(db))
|
||||
r.Get("/api/incidents/{id}", handleGetIncident(db))
|
||||
r.Get("/api/incidents/{id}/alerts", handleIncidentAlerts(db))
|
||||
r.Get("/api/incidents/{id}/timeline", handleIncidentTimeline(db))
|
||||
|
||||
@@ -404,6 +404,16 @@ input:focus, textarea:focus { outline: none; border-color: var(--accent); box-sh
|
||||
mask-image: linear-gradient(90deg, #000 calc(100% - 28px), transparent);
|
||||
}
|
||||
|
||||
/* The queue's cluster filter: a pill-shaped select under the status chips. */
|
||||
.queue-origin { position: relative; display: flex; align-items: center; padding: 0 16px 8px; }
|
||||
.origin-select-icon { position: absolute; left: 28px; width: 16px; height: 16px; color: var(--muted); pointer-events: none; }
|
||||
.origin-select {
|
||||
min-height: 34px; max-width: 100%; padding: 0 12px 0 34px;
|
||||
border: 1px solid var(--border-strong); border-radius: 999px;
|
||||
background: var(--surface); color: var(--text);
|
||||
font: inherit; font-size: 13px; font-weight: 600; cursor: pointer;
|
||||
}
|
||||
|
||||
/* ---------- lists ---------- */
|
||||
|
||||
.list { padding: 0 16px 16px; display: grid; gap: 8px; }
|
||||
|
||||
@@ -154,6 +154,9 @@
|
||||
<section id="view-queue" class="view view-queue" data-view="queue">
|
||||
<div class="pane pane-list">
|
||||
<div class="chips" id="queue-filters" role="tablist" aria-label="Filter"></div>
|
||||
<!-- The cluster filter. Hidden until the queue has seen two or more
|
||||
clusters; queue.js fills it in. -->
|
||||
<div class="queue-origin" id="queue-origin" hidden></div>
|
||||
<div id="queue-list" class="list"></div>
|
||||
</div>
|
||||
<div class="pane pane-detail" id="detail" aria-live="polite"></div>
|
||||
|
||||
@@ -83,6 +83,7 @@ export const setNotifyTarget = (id, ntfyTopic) =>
|
||||
|
||||
// incidents
|
||||
export const incidents = (query, opts) => call('GET', '/incidents', { query, ...opts });
|
||||
export const clusters = (query) => call('GET', '/incidents/clusters', { query });
|
||||
export const incident = (id) => call('GET', `/incidents/${id}`);
|
||||
export const timeline = (id) => call('GET', `/incidents/${id}/timeline`);
|
||||
|
||||
|
||||
+102
-10
@@ -1,7 +1,7 @@
|
||||
// The incident queue: filter chips and a list of incident rows.
|
||||
|
||||
import * as api from './api.js';
|
||||
import { h, clear, badge, severityBadge, originChip, emptyState, spinner, fadeOnOverflow } from './ui.js';
|
||||
import { h, clear, icon, badge, severityBadge, originChip, emptyState, spinner, fadeOnOverflow } from './ui.js';
|
||||
import { ago, until, isFuture, severityClass, labelSummary, originOf, titleWithoutOrigin } from './format.js';
|
||||
import { state, myID, onTeamChange } from './state.js';
|
||||
import * as onboarding from './onboarding.js';
|
||||
@@ -30,8 +30,10 @@ onboarding.onRerender(() => renderList());
|
||||
// The queue used to keep its own team filter (a per-tab sessionStorage value,
|
||||
// out of step with team.js's own picker); both now defer to the global
|
||||
// selector's shared state, so re-render whenever it changes.
|
||||
onTeamChange(() => {
|
||||
onTeamChange(async () => {
|
||||
renderChips();
|
||||
await loadClusters(true);
|
||||
renderOrigin();
|
||||
refresh({ fresh: true });
|
||||
});
|
||||
|
||||
@@ -52,6 +54,63 @@ function loadFilter() {
|
||||
return 'open';
|
||||
}
|
||||
|
||||
// Which cluster the queue is narrowed to, or null for all of them. Kept in
|
||||
// localStorage like the selected team: it is a standing choice, not a
|
||||
// per-tab one.
|
||||
const ORIGIN_KEY = 'terdut.queue.cluster';
|
||||
let origin = loadOrigin();
|
||||
let clusters = []; // the clusters the server has seen, for the dropdown
|
||||
let clustersAt = 0;
|
||||
|
||||
function loadOrigin() {
|
||||
try {
|
||||
return localStorage.getItem(ORIGIN_KEY) || null;
|
||||
} catch {
|
||||
return null; // storage unavailable
|
||||
}
|
||||
}
|
||||
|
||||
function saveOrigin() {
|
||||
try {
|
||||
if (origin) localStorage.setItem(ORIGIN_KEY, origin);
|
||||
else localStorage.removeItem(ORIGIN_KEY);
|
||||
} catch {
|
||||
/* storage unavailable */
|
||||
}
|
||||
}
|
||||
|
||||
// The clusters to offer, from the server because the list on screen is only
|
||||
// the fifty most recent. Asked again at most every five minutes; a cluster
|
||||
// first met in an incident on screen is added at once (see noteClusters). A
|
||||
// remembered choice the server no longer knows would filter to nothing with no
|
||||
// hint why, so it is dropped.
|
||||
async function loadClusters(force = false) {
|
||||
if (!force && Date.now() - clustersAt < 5 * 60 * 1000) return;
|
||||
try {
|
||||
const query = state.selectedTeamID != null ? { team_id: state.selectedTeamID } : undefined;
|
||||
clusters = await api.clusters(query);
|
||||
clustersAt = Date.now();
|
||||
if (origin && !clusters.includes(origin)) {
|
||||
origin = null;
|
||||
saveOrigin();
|
||||
}
|
||||
} catch {
|
||||
/* the filter is a convenience: without the list it stays hidden */
|
||||
}
|
||||
}
|
||||
|
||||
function noteClusters(list) {
|
||||
let added = false;
|
||||
for (const inc of list || []) {
|
||||
const c = originOf(inc.group_labels);
|
||||
if (c && !clusters.includes(c)) {
|
||||
clusters = [...clusters, c].sort();
|
||||
added = true;
|
||||
}
|
||||
}
|
||||
return added;
|
||||
}
|
||||
|
||||
function saveFilter() {
|
||||
try {
|
||||
sessionStorage.setItem('terdut.queue.filter', filter);
|
||||
@@ -64,6 +123,7 @@ export function show(incidentID) {
|
||||
selected = incidentID;
|
||||
if (!built) {
|
||||
renderChips();
|
||||
renderOrigin();
|
||||
built = true;
|
||||
}
|
||||
renderList();
|
||||
@@ -73,15 +133,22 @@ export async function refresh({ fresh = false } = {}) {
|
||||
const f = FILTERS.find((x) => x.id === filter);
|
||||
const requested = filter;
|
||||
try {
|
||||
// First, so a remembered cluster the server no longer knows is dropped
|
||||
// before it is sent as a filter.
|
||||
await loadClusters();
|
||||
// The open list is already fetched for the badges; no need to ask twice.
|
||||
// The cached open queue covers every team, so it can only be reused when
|
||||
// no team filter is applied.
|
||||
const query = state.selectedTeamID != null ? { ...f.query, team_id: state.selectedTeamID } : f.query;
|
||||
const cached = filter === 'open' && !fresh && state.selectedTeamID == null;
|
||||
const query = { ...f.query };
|
||||
if (state.selectedTeamID != null) query.team_id = state.selectedTeamID;
|
||||
if (origin) query.cluster = origin;
|
||||
const cached = filter === 'open' && !fresh && state.selectedTeamID == null && !origin;
|
||||
const result = cached ? state.open : await api.incidents(query);
|
||||
await onboarding.load();
|
||||
if (requested !== filter) return;
|
||||
items = result;
|
||||
noteClusters(state.open);
|
||||
noteClusters(items);
|
||||
error = null;
|
||||
} catch (err) {
|
||||
if (requested !== filter) return;
|
||||
@@ -91,6 +158,7 @@ export async function refresh({ fresh = false } = {}) {
|
||||
// whichever caller updated it before calling here — app.js's poll, or the
|
||||
// `cached` branch above.
|
||||
renderChips();
|
||||
renderOrigin();
|
||||
renderList();
|
||||
}
|
||||
|
||||
@@ -109,9 +177,9 @@ function setFilter(id) {
|
||||
// for the badges — Snoozed/Resolved/Archived would need a request of their
|
||||
// own, so those chips stay count-less for now.
|
||||
function chipCount(id) {
|
||||
const open = state.selectedTeamID == null
|
||||
? state.open
|
||||
: state.open.filter((i) => i.team_id === state.selectedTeamID);
|
||||
const open = state.open.filter((i) =>
|
||||
(state.selectedTeamID == null || i.team_id === state.selectedTeamID)
|
||||
&& (!origin || originOf(i.group_labels) === origin));
|
||||
if (id === 'open') return open.length;
|
||||
if (id === 'triggered') return open.filter((i) => i.status === 'triggered').length;
|
||||
if (id === 'acknowledged') return open.filter((i) => i.status === 'acknowledged').length;
|
||||
@@ -144,6 +212,30 @@ function renderChips() {
|
||||
chipsFade();
|
||||
}
|
||||
|
||||
// The cluster dropdown, under the status chips. Not shown at all until there
|
||||
// are two clusters to choose between, the same rule the team selector follows.
|
||||
function renderOrigin() {
|
||||
const el = document.getElementById('queue-origin');
|
||||
if (clusters.length < 2) {
|
||||
el.hidden = true;
|
||||
return;
|
||||
}
|
||||
const select = h('select', { class: 'origin-select', 'aria-label': 'Cluster' },
|
||||
h('option', { value: '', text: 'All clusters', selected: !origin }),
|
||||
clusters.map((c) => h('option', { value: c, text: c, selected: c === origin })));
|
||||
select.addEventListener('change', () => {
|
||||
origin = select.value || null;
|
||||
saveOrigin();
|
||||
items = null;
|
||||
cursor = -1;
|
||||
renderChips();
|
||||
renderList();
|
||||
refresh({ fresh: true });
|
||||
});
|
||||
clear(el, icon('server', 'icon origin-select-icon'), select);
|
||||
el.hidden = false;
|
||||
}
|
||||
|
||||
function renderList() {
|
||||
const el = document.getElementById('queue-list');
|
||||
const checklist = onboarding.card();
|
||||
@@ -201,7 +293,7 @@ function row(inc, index) {
|
||||
? h('span', { class: 'row-team', text: inc.team_name })
|
||||
: null;
|
||||
|
||||
const origin = originOf(inc.group_labels);
|
||||
const rowOrigin = originOf(inc.group_labels);
|
||||
|
||||
return h('a', {
|
||||
class: `row ${severityClass(inc.severity)} ${resolved ? 'resolved' : ''} ${index === cursor ? 'kbd-focus' : ''}`,
|
||||
@@ -209,10 +301,10 @@ function row(inc, index) {
|
||||
'aria-current': inc.id === selected ? 'true' : null,
|
||||
dataset: { index: String(index) },
|
||||
},
|
||||
h('div', { class: 'row-title', text: titleWithoutOrigin(inc.title, origin) }),
|
||||
h('div', { class: 'row-title', text: titleWithoutOrigin(inc.title, rowOrigin) }),
|
||||
h('div', { class: 'row-age', title: inc.triggered_at, text: `Triggered ${ago(inc.triggered_at)}` }),
|
||||
h('div', { class: 'row-meta' },
|
||||
origin && originChip(origin),
|
||||
rowOrigin && !origin && originChip(rowOrigin),
|
||||
!impliedStatus && status,
|
||||
// The left-border colour alone doesn't say what it means; spell it out
|
||||
// too, same badge the incident detail page uses for severity.
|
||||
|
||||
Reference in New Issue
Block a user